From c2e4cd652f7afb8a149a042f3e596879599faaf2 Mon Sep 17 00:00:00 2001
From: unclecode <unclecode@kidocode.com>
Date: Thu, 25 Jun 2026 09:25:07 +0000
Subject: [PATCH 01/26] perf(content-filter): lxml-native
 PruningContentFilterLXML, ~10x faster pruning, made default

The bs4 PruningContentFilter recomputes get_text() and encode_contents()
on every node while pruning, each walking the whole subtree -> O(N*subtree),
which is super-linear on wide/deep pages (a 6000-card page spent ~2.2s
inside filter_content, dominated by repeated subtree serialization).

PruningContentFilterLXML computes every per-node metric (text length,
inner-HTML length, link text, word count) once in a single bottom-up pass
cached on a __slots__ object, then scores and prunes top-down -> O(N).

Output is identical to the original: scoring math, thresholds, tag weights
and the bs4 quirks (per-fragment stripping, separator-less word count,
ASCII whitespace-only collapse, void-element self-closing, 256-depth cap
lifted via huge_tree) are reproduced 1:1. Validated node-by-node against
bs4 on real and pretty-printed pages plus a battery of adversarial shapes,
and end-to-end (fit_markdown byte-identical).

Made it the default where the code defaulted to PruningContentFilter: the
Docker server FIT filter and the CLI pruning filter. Exported from the
package and registered in the (de)serialization allowlists so configs
round-trip.

Speed (pruning only): medium 134->13ms, cards200 67->7ms, cards6000 2200->260ms.
---
 crawl4ai/__init__.py                          |   2 +
 crawl4ai/async_configs.py                     |   4 +-
 crawl4ai/cli.py                               |   3 +-
 crawl4ai/content_filter_strategy_lxml.py      | 391 ++++++++++++++++++
 crawl4ai/types.py                             |   4 +
 deploy/docker/api.py                          |   3 +-
 tests/async/test_content_filter_prune_lxml.py | 209 ++++++++++
 7 files changed, 612 insertions(+), 4 deletions(-)
 create mode 100644 crawl4ai/content_filter_strategy_lxml.py
 create mode 100644 tests/async/test_content_filter_prune_lxml.py

diff --git a/crawl4ai/__init__.py b/crawl4ai/__init__.py
index 0d1071aec..c33e8b136 100644
--- a/crawl4ai/__init__.py
+++ b/crawl4ai/__init__.py
@@ -42,6 +42,7 @@
     LLMContentFilter,
     RelevantContentFilter,
 )
+from .content_filter_strategy_lxml import PruningContentFilterLXML
 from .models import CrawlResult, MarkdownGenerationResult, DisplayMode
 from .components.crawler_monitor import CrawlerMonitor
 from .link_preview import LinkPreview
@@ -176,6 +177,7 @@
     "LLMTableExtraction",
     "RelevantContentFilter",
     "PruningContentFilter",
+    "PruningContentFilterLXML",
     "BM25ContentFilter",
     "LLMContentFilter",
     "BaseDispatcher",
diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py
index 27320fd4b..76445c021 100644
--- a/crawl4ai/async_configs.py
+++ b/crawl4ai/async_configs.py
@@ -133,7 +133,7 @@ class MatchMode(Enum):
     "CosineStrategy", "RegexExtractionStrategy",
     # Markdown / content
     "DefaultMarkdownGenerator",
-    "PruningContentFilter", "BM25ContentFilter", "LLMContentFilter",
+    "PruningContentFilter", "PruningContentFilterLXML", "BM25ContentFilter", "LLMContentFilter",
     # Scraping
     "LXMLWebScrapingStrategy", "PDFContentScrapingStrategy",
     # Chunking
@@ -190,7 +190,7 @@ class UntrustedConfigError(ValueError):
     # non-LLM extraction / markdown / scraping / chunking strategies
     "JsonCssExtractionStrategy", "JsonXPathExtractionStrategy",
     "JsonLxmlExtractionStrategy", "RegexExtractionStrategy", "CosineStrategy",
-    "DefaultMarkdownGenerator", "PruningContentFilter", "BM25ContentFilter",
+    "DefaultMarkdownGenerator", "PruningContentFilter", "PruningContentFilterLXML", "BM25ContentFilter",
     "LXMLWebScrapingStrategy", "PDFContentScrapingStrategy",
     "RegexChunking",
     "DefaultTableExtraction", "NoTableExtraction",
diff --git a/crawl4ai/cli.py b/crawl4ai/cli.py
index 02b67155e..ec826f63c 100644
--- a/crawl4ai/cli.py
+++ b/crawl4ai/cli.py
@@ -25,6 +25,7 @@
     JsonXPathExtractionStrategy,
     BM25ContentFilter,
     PruningContentFilter,
+    PruningContentFilterLXML,
     BrowserProfiler,
     DefaultMarkdownGenerator,
     LLMConfig,
@@ -1094,7 +1095,7 @@ def crawl_cmd(url: str, browser_config: str, crawler_config: str, filter_config:
                 )
             elif filter_conf["type"] == "pruning":
                 crawler_cfg.markdown_generator = DefaultMarkdownGenerator(
-                    content_filter = PruningContentFilter(
+                    content_filter = PruningContentFilterLXML(
                         user_query=filter_conf.get("query"),
                         threshold=filter_conf.get("threshold", 0.48)
                     )
diff --git a/crawl4ai/content_filter_strategy_lxml.py b/crawl4ai/content_filter_strategy_lxml.py
new file mode 100644
index 000000000..24671945b
--- /dev/null
+++ b/crawl4ai/content_filter_strategy_lxml.py
@@ -0,0 +1,391 @@
+"""
+content_filter_strategy_lxml.py
+===============================
+
+An lxml-native, single-pass reimplementation of :class:`PruningContentFilter`.
+
+Why
+---
+The original (BeautifulSoup) ``PruningContentFilter`` recomputes
+``node.get_text(strip=True)`` and ``node.encode_contents()`` for *every* node it
+visits while pruning. Both calls walk the node's entire subtree, and in
+BeautifulSoup they are pure-Python and slow. Because pruning visits most nodes
+in the document, the total work is effectively ``O(N * average_subtree_size)``
+-- it degrades super-linearly on wide/deep pages. A 6,000-card page spent
+~2.1 s purely inside ``filter_content``, dominated by repeated subtree
+serialization (``str(node)`` / ``encode_contents``).
+
+How this version is different
+-----------------------------
+Every per-node metric the score needs -- visible-text length, inner-HTML
+length, direct-link text length and word count -- is computed **exactly once**
+in a single *bottom-up* pass and cached on a slotted metrics object. Each text
+fragment in the document is therefore touched a constant number of times
+total, not once per ancestor. Scoring + pruning then run in a single
+*top-down* pass over the cached metrics. Total work is ``O(N)`` plus
+``O(total_text_size)``, entirely on libxml2's C-backed tree.
+
+Fidelity
+--------
+BeautifulSoup's ``lxml`` tree-builder and ``lxml.html`` both parse through
+libxml2, so the parsed trees are structurally identical node-for-node. The
+scoring math, thresholds, tag weights and the original's edge-case quirks
+(the always-zero ``class_id`` contribution, the separator-less word count) are
+reproduced 1:1.
+
+Two details are needed to match BeautifulSoup exactly:
+
+* **Per-fragment stripping.** ``get_text(strip=True)`` strips each text
+  fragment independently. BeautifulSoup's ``decompose()`` leaves the text that
+  surrounded a removed tag as *separate* fragments, so each is stripped on its
+  own. We therefore compute metrics *before* physically removing comments and
+  excluded tags: a removed child contributes 0, but its trailing text (tail)
+  is still measured as its own stripped fragment -- identical to what
+  BeautifulSoup sees after ``decompose``.
+* **No depth cap.** libxml2's HTML parser caps tree depth at 256 by default;
+  ``huge_tree=True`` lifts that so deeply nested pages parse like they do under
+  BeautifulSoup.
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Dict, List
+
+from lxml import etree
+from lxml import html as lhtml
+
+from .content_filter_strategy import PruningContentFilter
+
+# HTML void elements, taken verbatim from BeautifulSoup's LXMLTreeBuilder
+# ``empty_element_tags``. These serialize self-closed (``<br/>``) with no end
+# tag, which we must account for when reproducing inner-HTML length.
+VOID_ELEMENTS = frozenset(
+    {
+        "area", "base", "basefont", "bgsound", "br", "col", "command", "embed",
+        "frame", "hr", "image", "img", "input", "isindex", "keygen", "link",
+        "menuitem", "meta", "nextid", "param", "source", "spacer", "track", "wbr",
+    }
+)
+
+# A single reused parser. ``huge_tree=True`` lifts libxml2's 256-level depth
+# cap (and other limits) so output matches BeautifulSoup on pathological pages.
+_HTML_PARSER = lhtml.HTMLParser(huge_tree=True)
+
+# libxml2 treats only these as "blank" characters (IS_BLANK_CH); a text node
+# made up entirely of them is collapsed to a single character by BeautifulSoup's
+# tree builder. Note this is ASCII-only -- U+00A0 (&nbsp;) is NOT collapsed,
+# even though Python's str.strip() would remove it.
+_ASCII_WS = " \t\n\r"
+
+
+def _esc_len(s: str) -> int:
+    """Serialized length of ``s`` after HTML entity escaping (``&``->``&amp;``,
+    ``<``->``&lt;``, ``>``->``&gt;``), matching BeautifulSoup's minimal formatter."""
+    return len(s) + (s.count("&") << 2) + 3 * (s.count("<") + s.count(">"))
+
+
+class _NodeMetrics:
+    """Cached, slotted metrics for a single element (one instance per node).
+
+    Attributes
+    ----------
+    text_len:
+        Length of the element's visible text, computed the way
+        BeautifulSoup's ``get_text(strip=True)`` does it: every text fragment
+        (``.text`` of the node + ``.tail`` of each child) is stripped
+        independently and concatenated with no separator.
+    inner_len:
+        Length of the element's serialized inner HTML, matching
+        ``encode_contents()`` (entity-escaped text, child tags including
+        attributes, void elements self-closed).
+    space_count:
+        Number of ASCII spaces in the stripped, concatenated visible text.
+        ``word_count`` is ``space_count + 1`` -- the original counts words by
+        ``text.count(" ") + 1`` on the separator-less concatenation, and we
+        reproduce that exactly.
+    string_len:
+        Length of the stripped BeautifulSoup ``.string`` for this node, or
+        ``-1`` when ``.string`` would be ``None``. Used only to evaluate
+        ``link_text_len`` over direct-child ``<a>`` elements.
+    """
+
+    __slots__ = ("text_len", "inner_len", "space_count", "string_len")
+
+    def __init__(self, text_len: int, inner_len: int, space_count: int, string_len: int):
+        self.text_len = text_len
+        self.inner_len = inner_len
+        self.space_count = space_count
+        self.string_len = string_len
+
+
+class PruningContentFilterLXML(PruningContentFilter):
+    """Drop-in, lxml-native replacement for :class:`PruningContentFilter`.
+
+    Same constructor and same ``filter_content`` contract (returns a list of
+    HTML block strings). Subclasses :class:`PruningContentFilter` so it shares
+    the exact tag weights, metric weights, thresholds and negative-class
+    patterns -- only the engine changes.
+    """
+
+    def __init__(
+        self,
+        user_query: str = None,
+        min_word_threshold: int = None,
+        threshold_type: str = "fixed",
+        threshold: float = 0.48,
+    ):
+        # Mirror PruningContentFilter's explicit signature (rather than
+        # *args/**kwargs) so config (de)serialization can introspect and
+        # round-trip these fields.
+        super().__init__(
+            user_query=user_query,
+            min_word_threshold=min_word_threshold,
+            threshold_type=threshold_type,
+            threshold=threshold,
+        )
+        # word count is only needed when a min-word threshold is active; skip
+        # the per-fragment space counting otherwise.
+        self._need_words = self.min_word_threshold is not None
+
+    # ------------------------------------------------------------------ #
+    # Public API
+    # ------------------------------------------------------------------ #
+    def filter_content(self, html: str, min_word_threshold: int = None) -> List[str]:
+        if not html or not isinstance(html, str):
+            return []
+
+        try:
+            root = lhtml.document_fromstring(html, parser=_HTML_PARSER)
+        except (etree.ParserError, etree.LxmlError, ValueError):
+            return []
+
+        body = root.find("body")
+        if body is None:
+            body = root
+
+        # Metrics are computed on the *un-stripped* tree so that text which
+        # surrounds soon-to-be-removed nodes is measured as separate fragments
+        # (matching BeautifulSoup's decompose semantics).
+        metrics = self._compute_metrics(body)
+
+        # Now physically drop comments and unwanted tags. ``with_tail=False``
+        # keeps the text that followed them (merged into the previous sibling /
+        # parent) so the serialized output still contains it, exactly as a
+        # BeautifulSoup decompose would leave it.
+        etree.strip_elements(body, etree.Comment, with_tail=False)
+        if self.excluded_tags:
+            etree.strip_elements(body, *self.excluded_tags, with_tail=False)
+
+        body_metrics = metrics.get(body)
+        if body_metrics is None:
+            return []
+
+        # The original scores the body itself first; if it fails, the whole
+        # body is decomposed and nothing is returned.
+        if self._should_remove(body, body_metrics, metrics):
+            return []
+
+        # Top-down pass: a node survives only if it and every ancestor up to
+        # body pass the threshold. Iterative (explicit stack) to avoid Python
+        # recursion limits on deeply nested documents.
+        stack = [body]
+        while stack:
+            node = stack.pop()
+            for child in list(node):
+                if not isinstance(child.tag, str):
+                    continue
+                child_metrics = metrics.get(child)
+                if child_metrics is None or self._should_remove(child, child_metrics, metrics):
+                    child.drop_tree()  # removes subtree, preserves following text
+                else:
+                    stack.append(child)
+
+        # Collect surviving direct children of body, in document order,
+        # skipping any that no longer carry visible text after pruning.
+        blocks: List[str] = []
+        for el in body:
+            if not isinstance(el.tag, str):
+                continue
+            if any(t.strip() for t in el.itertext()):
+                blocks.append(lhtml.tostring(el, encoding="unicode", with_tail=False))
+        return blocks
+
+    # ------------------------------------------------------------------ #
+    # Metrics (single bottom-up pass)
+    # ------------------------------------------------------------------ #
+    def _compute_metrics(self, body) -> Dict[object, _NodeMetrics]:
+        """Compute and cache metrics for ``body`` and every descendant element,
+        skipping the interior of excluded subtrees (they will be removed).
+
+        Nodes are visited in reverse document order so every parent sees its
+        children's metrics already cached. A parent is always appended to the
+        work list before its children, so ``reversed`` guarantees children are
+        processed first.
+        """
+        metrics: Dict[object, _NodeMetrics] = {}
+        need_words = self._need_words
+        excluded = self.excluded_tags
+
+        # Pre-order collection that does not descend into excluded subtrees.
+        nodes = []
+        stack = [body]
+        while stack:
+            el = stack.pop()
+            nodes.append(el)
+            for child in el:
+                ct = child.tag
+                if isinstance(ct, str) and ct not in excluded:
+                    stack.append(child)
+
+        for el in reversed(nodes):
+            txt = el.text
+            if txt:
+                stripped = txt.strip()
+                text_len = len(stripped)
+                # BeautifulSoup's tree builder collapses an ASCII-whitespace-only
+                # text node to a single character; anything else is kept verbatim.
+                if stripped or txt.strip(_ASCII_WS):
+                    inner_len = _esc_len(txt)
+                else:
+                    inner_len = 1
+                space_count = stripped.count(" ") if need_words else 0
+                frag_count = 1
+                string_len = text_len  # candidate, valid only if frag_count == 1
+            else:
+                text_len = inner_len = space_count = 0
+                frag_count = 0
+                string_len = -1
+
+            for child in el:
+                ct = child.tag
+                kept_element = isinstance(ct, str) and ct not in excluded
+
+                if kept_element:
+                    cm = metrics[child]
+                    text_len += cm.text_len
+                    inner_len += self._outer_len(child, cm.inner_len)
+                    if need_words:
+                        space_count += cm.space_count
+                    frag_count += 1
+                    string_len = cm.string_len  # recurse if this is the sole content
+                # else: comment / excluded element -> removed, contributes 0,
+                # but its trailing text is still a separate fragment below.
+
+                tail = child.tail
+                if tail:
+                    st = tail.strip()
+                    text_len += len(st)
+                    inner_len += _esc_len(tail) if (st or tail.strip(_ASCII_WS)) else 1
+                    if need_words:
+                        space_count += st.count(" ")
+                    frag_count += 1
+                    string_len = len(st)  # candidate, valid only if frag_count == 1
+
+            # BeautifulSoup ``Tag.string`` is non-None only when the element has
+            # exactly one content fragment.
+            if frag_count != 1:
+                string_len = -1
+
+            metrics[el] = _NodeMetrics(text_len, inner_len, space_count, string_len)
+
+        return metrics
+
+    def _outer_len(self, el, inner_len: int) -> int:
+        """Serialized outer-HTML length of ``el`` given its cached inner length.
+
+        Mirrors BeautifulSoup's default (minimal) serialization:
+        ``<tag attr="val">inner</tag>``, with void elements self-closed as
+        ``<tag/>`` and HTML entity escaping inside attribute values.
+        """
+        tag = el.tag
+        attrs_len = 0
+        for k, v in el.attrib.items():
+            # ' key="value"' -> 1 space + len(key) + '="' + escaped value + '"'
+            attrs_len += 4 + len(k) + _esc_len(v)
+        tag_len = len(tag)
+        if tag in VOID_ELEMENTS:
+            return 1 + tag_len + attrs_len + 2  # <tag ... />
+        # start "<tag ...>" = 2 + tag_len + attrs_len ; end "</tag>" = 3 + tag_len
+        return (2 + tag_len + attrs_len) + inner_len + (3 + tag_len)
+
+    # ------------------------------------------------------------------ #
+    # Scoring (top-down pass)
+    # ------------------------------------------------------------------ #
+    def _should_remove(self, node, m: _NodeMetrics, metrics: Dict[object, _NodeMetrics]) -> bool:
+        text_len = m.text_len
+        tag_len = m.inner_len
+
+        link_text_len = 0
+        for child in node:
+            if child.tag == "a":
+                cm = metrics.get(child)
+                if cm is not None and cm.string_len > 0:
+                    link_text_len += cm.string_len
+
+        score = self._composite_score(node, m, text_len, tag_len, link_text_len)
+
+        if self.threshold_type == "fixed":
+            return score < self.threshold
+
+        # dynamic threshold
+        tag_importance = self.tag_importance.get(node.tag, 0.7)
+        text_ratio = text_len / tag_len if tag_len > 0 else 0
+        link_ratio = link_text_len / text_len if text_len > 0 else 1
+
+        threshold = self.threshold
+        if tag_importance > 1:
+            threshold *= 0.8
+        if text_ratio > 0.4:
+            threshold *= 0.9
+        if link_ratio > 0.6:
+            threshold *= 1.2
+
+        return score < threshold
+
+    def _composite_score(self, node, m: _NodeMetrics, text_len: int, tag_len: int, link_text_len: int) -> float:
+        if self.min_word_threshold:
+            word_count = m.space_count + 1
+            if word_count < self.min_word_threshold:
+                return -1.0
+
+        cfg = self.metric_config
+        w = self.metric_weights
+        score = 0.0
+        total = 0.0
+
+        if cfg["text_density"]:
+            density = text_len / tag_len if tag_len > 0 else 0
+            score += w["text_density"] * density
+            total += w["text_density"]
+
+        if cfg["link_density"]:
+            link_density = 1 - (link_text_len / text_len if text_len > 0 else 0)
+            score += w["link_density"] * link_density
+            total += w["link_density"]
+
+        if cfg["tag_weight"]:
+            score += w["tag_weight"] * self.tag_weights.get(node.tag, 0.5)
+            total += w["tag_weight"]
+
+        if cfg["class_id_weight"]:
+            # Faithful to the original: _class_id_score is <= 0, so max(0, .)
+            # contributes nothing -- it only dilutes the denominator.
+            score += w["class_id_weight"] * max(0, self._class_id_score(node))
+            total += w["class_id_weight"]
+
+        if cfg["text_length"]:
+            score += w["text_length"] * math.log(text_len + 1)
+            total += w["text_length"]
+
+        return score / total if total > 0 else 0
+
+    def _class_id_score(self, node) -> float:
+        score = 0.0
+        cls = node.get("class")
+        if cls is not None and self.negative_patterns.match(cls):
+            score -= 0.5
+        node_id = node.get("id")
+        if node_id is not None and self.negative_patterns.match(node_id):
+            score -= 0.5
+        return score
diff --git a/crawl4ai/types.py b/crawl4ai/types.py
index 72a0828e9..b0cd4ff5b 100644
--- a/crawl4ai/types.py
+++ b/crawl4ai/types.py
@@ -49,6 +49,7 @@
 # Content filter types
 RelevantContentFilter = Union['RelevantContentFilterType']
 PruningContentFilter = Union['PruningContentFilterType']
+PruningContentFilterLXML = Union['PruningContentFilterLXMLType']
 BM25ContentFilter = Union['BM25ContentFilterType']
 LLMContentFilter = Union['LLMContentFilterType']
 
@@ -152,6 +153,9 @@
         BM25ContentFilter as BM25ContentFilterType,
         LLMContentFilter as LLMContentFilterType,
     )
+    from .content_filter_strategy_lxml import (
+        PruningContentFilterLXML as PruningContentFilterLXMLType,
+    )
     
     # Dispatcher imports
     from .async_dispatcher import (
diff --git a/deploy/docker/api.py b/deploy/docker/api.py
index 1756b925f..7fcf02e0c 100644
--- a/deploy/docker/api.py
+++ b/deploy/docker/api.py
@@ -69,6 +69,7 @@ def _attach_declarative_hooks(crawler, hooks_config: dict) -> dict:
     BM25ContentFilter,
     LLMContentFilter
 )
+from crawl4ai.content_filter_strategy_lxml import PruningContentFilterLXML
 from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
 from crawl4ai.content_scraping_strategy import LXMLWebScrapingStrategy
 
@@ -354,7 +355,7 @@ async def handle_markdown_request(
             from llm_broker import resolve_llm
             _llm = resolve_llm(config, provider)
             content_filter = {
-                FilterType.FIT: PruningContentFilter(),
+                FilterType.FIT: PruningContentFilterLXML(),
                 FilterType.BM25: BM25ContentFilter(user_query=query or ""),
                 FilterType.LLM: LLMContentFilter(
                     llm_config=LLMConfig(
diff --git a/tests/async/test_content_filter_prune_lxml.py b/tests/async/test_content_filter_prune_lxml.py
new file mode 100644
index 000000000..afc1226f4
--- /dev/null
+++ b/tests/async/test_content_filter_prune_lxml.py
@@ -0,0 +1,209 @@
+"""Tests for the lxml-native PruningContentFilterLXML.
+
+Two concerns:
+  1. The new filter satisfies the same behavioral contract as the original
+     (ported from test_content_filter_prune.py).
+  2. The new filter is output-equivalent to the original (bs4) implementation
+     across a battery of HTML shapes and configs -- this is the fidelity guard
+     for the lxml rewrite.
+"""
+import os
+import sys
+import time
+
+import pytest
+from bs4 import BeautifulSoup
+
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
+from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy_lxml import PruningContentFilterLXML
+
+
+# --------------------------------------------------------------------------- #
+# Fixtures
+# --------------------------------------------------------------------------- #
+@pytest.fixture
+def basic_html():
+    return """
+    <html><body><article>
+        <h1>Main Article</h1>
+        <p>This is a high-quality paragraph with substantial text content. It contains enough words to pass the threshold and has good text density without too many links. This kind of content should survive the pruning process.</p>
+        <div class="sidebar">Low quality sidebar content</div>
+        <div class="social-share">Share buttons</div>
+    </article></body></html>
+    """
+
+
+@pytest.fixture
+def link_heavy_html():
+    return """
+    <html><body><div class="content">
+        <p>Good content paragraph that should remain.</p>
+        <div class="links">
+            <a href="#">Link 1</a><a href="#">Link 2</a><a href="#">Link 3</a><a href="#">Link 4</a>
+        </div>
+    </div></body></html>
+    """
+
+
+@pytest.fixture
+def mixed_content_html():
+    return """
+    <html><body><article>
+        <h1>Article Title</h1>
+        <p class="summary">Short summary.</p>
+        <div class="content">
+            <p>Long high-quality paragraph with substantial content that should definitely survive the pruning process. This content has good text density and proper formatting which makes it valuable for retention.</p>
+        </div>
+        <div class="comments"><p>Short comment 1</p><p>Short comment 2</p></div>
+    </article></body></html>
+    """
+
+
+# Battery of shapes that stress the metric reproduction (void elements,
+# entities, nested/!direct anchors, text around removed tags, comments,
+# unicode, tables, deep nesting, attributes).
+EQUIVALENCE_CASES = {
+    "void_img": '<body><div><p>text with <img src="a.png?x=1&y=2" alt="hi"> image content here now</p></div></body>',
+    "void_mix": '<body><div>alpha<br>beta<hr><input name="q" value="z"> gamma content text</div></body>',
+    "entities_text": '<body><div><p>Tom &amp; Jerry &lt;tag&gt; &copy; 2024 &nbsp; end of paragraph text</p></div></body>',
+    "entities_attr": '<body><div><a href="/s?q=a&b=c&d=e">a very long anchor text node here</a> body text follows</div></body>',
+    "nested_a_span": '<body><div class="x"><a href="#"><span>nested link text inside span</span></a> tail text after anchor</div></body>',
+    "nested_a_multi": '<body><div><a href="#">part one <b>bold</b> part two</a> and more body text content</div></body>',
+    "a_not_direct": '<body><div><p>wrapper text <a href="#">deep link</a> more wrapper text content here</p></div></body>',
+    "tail_after_nav": '<body><div>before text content <nav>NAVIGATION MENU</nav> after text continues here now</div></body>',
+    "tail_after_script": '<body><div>start text <script>var x=1;</script> middle part <style>.a{}</style> end content text</div></body>',
+    "comment_interspersed": '<body><div>alpha text <!-- a comment --> beta text <!--another--> gamma delta content</div></body>',
+    "pre_whitespace": '<body><pre>  line one\n      line two\n   line three with content  </pre></body>',
+    "table": '<body><table><tr><td>cell one content</td><td>cell two content</td></tr><tr><td>three</td><td>four</td></tr></table></body>',
+    "unicode_emoji": '<body><div><p>Héllo wörld 你好 مرحبا 🚀🔥 emoji test paragraph content</p></div></body>',
+    "nbsp": '<body><div><p>&nbsp;&nbsp;&nbsp;</p><p>actual real content paragraph text goes here now today</p></div></body>',
+    "many_attrs": '<body><div id="main" class="a b c" data-x="1" data-y="2&3" role="main"><p>paragraph text content here for the win today</p></div></body>',
+    "header_footer_aside": '<body><header>HEAD</header><main><p>Main content paragraph that is substantial enough to keep around here.</p></main><aside>ASIDE JUNK</aside><footer>FOOT</footer></body>',
+    "links_vs_content": '<body><div class="links">' + ''.join(f'<a href="#">L{i}</a>' for i in range(20)) + '</div><div class="content"><p>Good paragraph with real content that should clearly survive pruning here today.</p></div></body>',
+    "deep_nested": '<body>' + '<div>' * 300 + 'deep content text that is long enough to matter here' + '</div>' * 300 + '</body>',
+    "malformed": "<div>Unclosed div<p>Nested<span>content that keeps going on here</div>",
+}
+
+CONFIGS = [
+    {},
+    {"threshold_type": "dynamic"},
+    {"threshold_type": "dynamic", "threshold": 0.45},
+    {"threshold": 0.3},
+    {"threshold": 0.7},
+    {"min_word_threshold": 3},
+    {"min_word_threshold": 10},
+]
+
+
+def _norm_block_texts(blocks):
+    """Visible text of each block, whitespace-normalized (serialization-agnostic)."""
+    return [" ".join(BeautifulSoup(b, "lxml").get_text(" ").split()) for b in blocks]
+
+
+# --------------------------------------------------------------------------- #
+# Behavioral contract (ported, run against the NEW class)
+# --------------------------------------------------------------------------- #
+class TestPruningContentFilterLXMLBehavior:
+    def test_basic_pruning(self, basic_html):
+        contents = PruningContentFilterLXML(min_word_threshold=5).filter_content(basic_html)
+        combined = " ".join(contents).lower()
+        assert "high-quality paragraph" in combined
+        assert "sidebar content" not in combined
+        assert "share buttons" not in combined
+
+    def test_min_word_threshold(self, mixed_content_html):
+        contents = PruningContentFilterLXML(min_word_threshold=10).filter_content(mixed_content_html)
+        combined = " ".join(contents).lower()
+        assert "short summary" not in combined
+        assert "long high-quality paragraph" in combined
+        assert "short comment" not in combined
+
+    def test_link_density_impact(self, link_heavy_html):
+        contents = PruningContentFilterLXML(threshold_type="dynamic").filter_content(link_heavy_html)
+        combined = " ".join(contents).lower()
+        assert "good content paragraph" in combined
+        assert len([c for c in contents if "href" in c]) < 2
+
+    def test_tag_importance(self, mixed_content_html):
+        contents = PruningContentFilterLXML(threshold_type="dynamic").filter_content(mixed_content_html)
+        assert any("article" in c.lower() for c in contents) or any("h1" in c.lower() for c in contents)
+
+    def test_empty_input(self):
+        f = PruningContentFilterLXML()
+        assert f.filter_content("") == []
+        assert f.filter_content(None) == []
+
+    def test_malformed_html(self):
+        contents = PruningContentFilterLXML().filter_content("<div>Unclosed div<p>Nested<span>content</div>")
+        assert isinstance(contents, list)
+
+    @pytest.mark.parametrize("threshold,expected_count", [(0.3, 4), (0.48, 2), (0.7, 1)])
+    def test_threshold_levels(self, mixed_content_html, threshold, expected_count):
+        contents = PruningContentFilterLXML(threshold_type="fixed", threshold=threshold).filter_content(mixed_content_html)
+        assert len(contents) <= expected_count
+
+    def test_consistent_output(self, basic_html):
+        f = PruningContentFilterLXML()
+        assert f.filter_content(basic_html) == f.filter_content(basic_html)
+
+
+# --------------------------------------------------------------------------- #
+# Fidelity: new output must match the original bs4 implementation
+# --------------------------------------------------------------------------- #
+class TestPruningEquivalence:
+    @pytest.mark.parametrize("name", list(EQUIVALENCE_CASES.keys()))
+    @pytest.mark.parametrize("config", CONFIGS)
+    def test_blocks_match_original(self, name, config):
+        html = EQUIVALENCE_CASES[name]
+        old = PruningContentFilter(**config).filter_content(html)
+        new = PruningContentFilterLXML(**config).filter_content(html)
+        assert _norm_block_texts(old) == _norm_block_texts(new), (
+            f"{name} {config}\n old={_norm_block_texts(old)}\n new={_norm_block_texts(new)}"
+        )
+
+    def test_fixture_files_match_original(self):
+        """If saved server fixtures exist, diff old vs new on them too (large)."""
+        fixtures = os.environ.get("PRUNE_FIXTURES_DIR")
+        if not fixtures or not os.path.isdir(fixtures):
+            pytest.skip("no PRUNE_FIXTURES_DIR set")
+        for fn in sorted(os.listdir(fixtures)):
+            if not fn.endswith(".html"):
+                continue
+            html = open(os.path.join(fixtures, fn)).read()
+            for config in CONFIGS:
+                old = PruningContentFilter(**config).filter_content(html)
+                new = PruningContentFilterLXML(**config).filter_content(html)
+                assert _norm_block_texts(old) == _norm_block_texts(new), f"{fn} {config}"
+
+
+# --------------------------------------------------------------------------- #
+# Performance: new must be substantially faster on a non-trivial page
+# --------------------------------------------------------------------------- #
+def test_performance_faster_than_original():
+    # ~600 repeated cards -> a few hundred KB of HTML
+    html = "<body>" + "".join(
+        f'<div class="card"><h3>Card {i}</h3>'
+        f'<p>Some descriptive paragraph text for card number {i} with enough words to score.</p>'
+        f'<div class="meta"><a href="#">tag-a</a><a href="#">tag-b</a></div></div>'
+        for i in range(600)
+    ) + "</body>"
+
+    def timed(cls):
+        f = cls()
+        t0 = time.perf_counter()
+        out = f.filter_content(html)
+        return time.perf_counter() - t0, out
+
+    old_t, old_out = timed(PruningContentFilter)
+    new_t, new_out = timed(PruningContentFilterLXML)
+
+    assert _norm_block_texts(old_out) == _norm_block_texts(new_out)
+    # Expect a large margin; assert a conservative 2x to avoid flakiness.
+    assert new_t < old_t / 2, f"new={new_t*1000:.1f}ms not <2x faster than old={old_t*1000:.1f}ms"
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-q"])

From 42ad7ecd49dfce8144314de073ecaa6d3fbeb37c Mon Sep 17 00:00:00 2001
From: unclecode <unclecode@kidocode.com>
Date: Thu, 25 Jun 2026 09:36:48 +0000
Subject: [PATCH 02/26] deprecate PruningContentFilter; make
 PruningContentFilterLXML the documented default

- PruningContentFilter now emits a DeprecationWarning on direct instantiation
  (suppressed when the PruningContentFilterLXML subclass calls super().__init__),
  pointing users to the lxml class and explaining the alias roadmap.
- Lazily re-export PruningContentFilterLXML from content_filter_strategy via a
  module __getattr__ so existing import paths keep working with no circular import.
- Update all active docs + example scripts to PruningContentFilterLXML and add
  deprecation/migration callouts to fit-markdown, markdown-generation and the SDK
  reference. Historical release notes (0.4.0) left unchanged.
---
 crawl4ai/content_filter_strategy.py           | 28 +++++++++
 docs/examples/browser_optimization_example.py |  4 +-
 docs/examples/demo_multi_config_clean.py      |  4 +-
 docs/examples/docker/demo_docker_api.py       |  2 +-
 docs/examples/docker_config_obj.py            |  8 +--
 .../extraction_strategies_examples.py         |  4 +-
 docs/examples/hello_world.py                  |  4 +-
 docs/examples/hello_world_undetected.py       |  4 +-
 docs/examples/quickstart.py                   |  4 +-
 docs/examples/quickstart_examples_set_1.py    |  4 +-
 docs/examples/quickstart_examples_set_2.py    |  4 +-
 docs/examples/serp_api_project_11_feb.py      |  4 +-
 docs/examples/session_id_example.py           |  2 +-
 .../bbc_sport_research_assistant.py           |  4 +-
 .../url_seeder/tutorial_url_seeder.md         |  4 +-
 .../url_seeder/url_seeder_quick_demo.py       |  4 +-
 docs/md_v2/advanced/multi-url-crawling.md     |  4 +-
 docs/md_v2/advanced/undetected-browser.md     |  2 +-
 docs/md_v2/api/arun_many.md                   |  4 +-
 docs/md_v2/api/crawl-result.md                |  4 +-
 docs/md_v2/complete-sdk-reference.md          | 59 +++++++++++--------
 docs/md_v2/core/crawler-result.md             |  4 +-
 docs/md_v2/core/fit-markdown.md               | 31 +++++++---
 docs/md_v2/core/markdown-generation.md        | 34 ++++++-----
 docs/md_v2/core/quickstart.md                 |  8 +--
 docs/md_v2/core/simple-crawling.md            |  2 +-
 docs/md_v2/extraction/llm-strategies.md       |  2 +-
 tests/async/test_content_filter_prune_lxml.py |  2 +
 28 files changed, 152 insertions(+), 92 deletions(-)

diff --git a/crawl4ai/content_filter_strategy.py b/crawl4ai/content_filter_strategy.py
index 0909be33d..04e521dbd 100644
--- a/crawl4ai/content_filter_strategy.py
+++ b/crawl4ai/content_filter_strategy.py
@@ -1,6 +1,7 @@
 import inspect
 import re
 import time
+import warnings
 from bs4 import BeautifulSoup, Tag
 from typing import List, Tuple, Dict, Optional
 from rank_bm25 import BM25Okapi
@@ -580,6 +581,19 @@ def __init__(
             threshold (float): Fixed threshold value (default: 0.48).
         """
         super().__init__(None)
+        # Deprecation: the BeautifulSoup engine is superseded by the lxml one.
+        # Only warn for direct use of this class, not when PruningContentFilterLXML
+        # (a subclass) calls super().__init__.
+        if type(self) is PruningContentFilter:
+            warnings.warn(
+                "PruningContentFilter (BeautifulSoup-based) is deprecated in favor of "
+                "PruningContentFilterLXML, an lxml reimplementation that is ~10x faster "
+                "with identical output. Switch to PruningContentFilterLXML now. In an "
+                "upcoming release PruningContentFilter will become an alias for the lxml "
+                "implementation, and PruningContentFilterLXML will be kept as a legacy alias.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
         self.min_word_threshold = min_word_threshold
         self.threshold_type = threshold_type
         self.threshold = threshold
@@ -1088,3 +1102,17 @@ def show_usage(self) -> None:
                     f"{i:<10} {usage.completion_tokens:>12,} "
                     f"{usage.prompt_tokens:>12,} {usage.total_tokens:>12,}"
                 )
+
+
+def __getattr__(name):
+    """Lazily re-export PruningContentFilterLXML from this module.
+
+    The lxml engine lives in ``content_filter_strategy_lxml`` (which imports
+    from this module). Exposing it here via PEP 562 lazy attribute access lets
+    ``from crawl4ai.content_filter_strategy import PruningContentFilterLXML``
+    work without creating an import-time circular dependency.
+    """
+    if name == "PruningContentFilterLXML":
+        from .content_filter_strategy_lxml import PruningContentFilterLXML
+        return PruningContentFilterLXML
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
diff --git a/docs/examples/browser_optimization_example.py b/docs/examples/browser_optimization_example.py
index cf5e13075..255388035 100644
--- a/docs/examples/browser_optimization_example.py
+++ b/docs/examples/browser_optimization_example.py
@@ -34,7 +34,7 @@ async def crawl_sequential(urls: List[str]):
     # Configure crawl settings
     crawl_config = CrawlerRunConfig(
         markdown_generator=DefaultMarkdownGenerator(
-            #  content_filter=PruningContentFilter(), In case you need fit_markdown
+            #  content_filter=PruningContentFilterLXML(), In case you need fit_markdown
         ),
     )
 
@@ -71,7 +71,7 @@ async def crawl_parallel(urls: List[str], max_concurrent: int = 3):
 
     crawl_config = CrawlerRunConfig(
         markdown_generator=DefaultMarkdownGenerator(
-            #  content_filter=PruningContentFilter(), In case you need fit_markdown
+            #  content_filter=PruningContentFilterLXML(), In case you need fit_markdown
         ),
     )
 
diff --git a/docs/examples/demo_multi_config_clean.py b/docs/examples/demo_multi_config_clean.py
index 09df71ce8..085ad33cb 100644
--- a/docs/examples/demo_multi_config_clean.py
+++ b/docs/examples/demo_multi_config_clean.py
@@ -16,7 +16,7 @@
 )
 from crawl4ai.processors.pdf import PDFContentScrapingStrategy
 from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
 from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
 
 
@@ -171,7 +171,7 @@ async def demo_part2_practical_crawling():
         CrawlerRunConfig(
             url_matcher=["*/blog/*", "*/article/*", "*python.org*"],
             markdown_generator=DefaultMarkdownGenerator(
-                content_filter=PruningContentFilter(threshold=0.48)
+                content_filter=PruningContentFilterLXML(threshold=0.48)
             )
         ),
         
diff --git a/docs/examples/docker/demo_docker_api.py b/docs/examples/docker/demo_docker_api.py
index 0a3d51af1..e6c3b58db 100644
--- a/docs/examples/docker/demo_docker_api.py
+++ b/docs/examples/docker/demo_docker_api.py
@@ -362,7 +362,7 @@ async def demo_markdown_pruning(client: httpx.AsyncClient):
                     "type": "DefaultMarkdownGenerator",
                     "params": {
                         "content_filter": {
-                            "type": "PruningContentFilter",
+                            "type": "PruningContentFilterLXML",
                             "params": {
                                 "threshold": 0.6,
                                 "threshold_type": "relative"
diff --git a/docs/examples/docker_config_obj.py b/docs/examples/docker_config_obj.py
index 6ddf157a3..2cff64fca 100644
--- a/docs/examples/docker_config_obj.py
+++ b/docs/examples/docker_config_obj.py
@@ -1,4 +1,4 @@
-from crawl4ai import BrowserConfig, CrawlerRunConfig, PruningContentFilter, DefaultMarkdownGenerator
+from crawl4ai import BrowserConfig, CrawlerRunConfig, PruningContentFilterLXML, DefaultMarkdownGenerator
 from crawl4ai.deep_crawling.filters import ContentTypeFilter, DomainFilter
 from crawl4ai.deep_crawling.scorers import KeywordRelevanceScorer, PathDepthScorer
 from crawl4ai.cache_context import CacheMode
@@ -42,7 +42,7 @@ async def part1_basic_config():
     crawler_config = CrawlerRunConfig(
         word_count_threshold=200,
         markdown_generator=DefaultMarkdownGenerator(
-            content_filter=PruningContentFilter(threshold=0.5)
+            content_filter=PruningContentFilterLXML(threshold=0.5)
         )
     )
     
@@ -90,7 +90,7 @@ async def part3_complex_structures():
     config = CrawlerRunConfig(
         cache_mode=CacheMode.BYPASS,
         markdown_generator=DefaultMarkdownGenerator(
-            content_filter=PruningContentFilter()
+            content_filter=PruningContentFilterLXML()
         ),
         deep_crawl_strategy=BFSDeepCrawlStrategy(
             max_depth=5,
@@ -172,7 +172,7 @@ async def part5_direct_api():
                     "type": "DefaultMarkdownGenerator",
                     "params": {
                         "content_filter": {
-                            "type": "PruningContentFilter",
+                            "type": "PruningContentFilterLXML",
                             "params": {
                                 "threshold": 0.48,
                                 "threshold_type": "fixed"
diff --git a/docs/examples/extraction_strategies_examples.py b/docs/examples/extraction_strategies_examples.py
index 5f072f888..6717d89c0 100644
--- a/docs/examples/extraction_strategies_examples.py
+++ b/docs/examples/extraction_strategies_examples.py
@@ -17,7 +17,7 @@
     JsonCssExtractionStrategy,
     JsonXPathExtractionStrategy,
 )
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
 from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
 
 
@@ -29,7 +29,7 @@ async def run_extraction(crawler: AsyncWebCrawler, url: str, strategy, name: str
             cache_mode=CacheMode.BYPASS,
             extraction_strategy=strategy,
             markdown_generator=DefaultMarkdownGenerator(
-                content_filter=PruningContentFilter()  # For fit_markdown support
+                content_filter=PruningContentFilterLXML()  # For fit_markdown support
             ),
         )
 
diff --git a/docs/examples/hello_world.py b/docs/examples/hello_world.py
index 2ba2e852b..605c88855 100644
--- a/docs/examples/hello_world.py
+++ b/docs/examples/hello_world.py
@@ -4,7 +4,7 @@
     BrowserConfig,
     CrawlerRunConfig,
     DefaultMarkdownGenerator,
-    PruningContentFilter,
+    PruningContentFilterLXML,
     CrawlResult
 )
 
@@ -17,7 +17,7 @@ async def main():
     async with AsyncWebCrawler(config=browser_config) as crawler:
         crawler_config = CrawlerRunConfig(
             markdown_generator=DefaultMarkdownGenerator(
-                content_filter=PruningContentFilter()
+                content_filter=PruningContentFilterLXML()
             ),
         )
         result: CrawlResult = await crawler.arun(
diff --git a/docs/examples/hello_world_undetected.py b/docs/examples/hello_world_undetected.py
index 83ce51ef6..ae9d4a345 100644
--- a/docs/examples/hello_world_undetected.py
+++ b/docs/examples/hello_world_undetected.py
@@ -4,7 +4,7 @@
     BrowserConfig,
     CrawlerRunConfig,
     DefaultMarkdownGenerator,
-    PruningContentFilter,
+    PruningContentFilterLXML,
     CrawlResult,
     UndetectedAdapter
 )
@@ -35,7 +35,7 @@ async def main():
         # Configure the crawl
         crawler_config = CrawlerRunConfig(
             markdown_generator=DefaultMarkdownGenerator(
-                content_filter=PruningContentFilter()
+                content_filter=PruningContentFilterLXML()
             ),
             capture_console_messages=True,  # Enable console capture to test adapter
         )
diff --git a/docs/examples/quickstart.py b/docs/examples/quickstart.py
index 9992a6bf0..fbedcecca 100644
--- a/docs/examples/quickstart.py
+++ b/docs/examples/quickstart.py
@@ -15,7 +15,7 @@
 from pydantic import BaseModel, Field
 from crawl4ai import AsyncWebCrawler, CacheMode, BrowserConfig, CrawlerRunConfig
 from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
 from crawl4ai import (
     JsonCssExtractionStrategy,
     LLMExtractionStrategy,
@@ -48,7 +48,7 @@ async def clean_content():
         excluded_tags=["nav", "footer", "aside"],
         remove_overlay_elements=True,
         markdown_generator=DefaultMarkdownGenerator(
-            content_filter=PruningContentFilter(
+            content_filter=PruningContentFilterLXML(
                 threshold=0.48, threshold_type="fixed", min_word_threshold=0
             ),
             options={"ignore_links": True},
diff --git a/docs/examples/quickstart_examples_set_1.py b/docs/examples/quickstart_examples_set_1.py
index 078d1c4a9..d9dabc2b3 100644
--- a/docs/examples/quickstart_examples_set_1.py
+++ b/docs/examples/quickstart_examples_set_1.py
@@ -10,7 +10,7 @@
 from crawl4ai import RoundRobinProxyStrategy
 from crawl4ai import JsonCssExtractionStrategy, LLMExtractionStrategy
 from crawl4ai import LLMConfig
-from crawl4ai import PruningContentFilter, BM25ContentFilter
+from crawl4ai import PruningContentFilterLXML, BM25ContentFilter
 from crawl4ai import DefaultMarkdownGenerator
 from crawl4ai import BFSDeepCrawlStrategy, DomainFilter, FilterChain
 from crawl4ai import BrowserConfig
@@ -69,7 +69,7 @@ async def demo_fit_markdown():
             url = "https://en.wikipedia.org/wiki/Python_(programming_language)",
             config=CrawlerRunConfig(
                 markdown_generator=DefaultMarkdownGenerator(
-                    content_filter=PruningContentFilter()
+                    content_filter=PruningContentFilterLXML()
                 )
             ),
         )
diff --git a/docs/examples/quickstart_examples_set_2.py b/docs/examples/quickstart_examples_set_2.py
index b12b084c2..e78d5701b 100644
--- a/docs/examples/quickstart_examples_set_2.py
+++ b/docs/examples/quickstart_examples_set_2.py
@@ -15,7 +15,7 @@
 from pydantic import BaseModel, Field
 from crawl4ai import AsyncWebCrawler, CacheMode, BrowserConfig, CrawlerRunConfig
 from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
 from crawl4ai import (
     JsonCssExtractionStrategy,
     LLMExtractionStrategy,
@@ -48,7 +48,7 @@ async def clean_content():
         excluded_tags=["nav", "footer", "aside"],
         remove_overlay_elements=True,
         markdown_generator=DefaultMarkdownGenerator(
-            content_filter=PruningContentFilter(
+            content_filter=PruningContentFilterLXML(
                 threshold=0.48, threshold_type="fixed", min_word_threshold=0
             ),
             options={"ignore_links": True},
diff --git a/docs/examples/serp_api_project_11_feb.py b/docs/examples/serp_api_project_11_feb.py
index 8f9312f73..2241bde0f 100644
--- a/docs/examples/serp_api_project_11_feb.py
+++ b/docs/examples/serp_api_project_11_feb.py
@@ -14,7 +14,7 @@
     CrawlerHub,
     CrawlResult,
     DefaultMarkdownGenerator,
-    PruningContentFilter,
+    PruningContentFilterLXML,
 )
 from pathlib import Path
 from pydantic import BaseModel
@@ -35,7 +35,7 @@ async def hello_web():
         crawler_config = CrawlerRunConfig(
             cache_mode=CacheMode.BYPASS,
             markdown_generator=DefaultMarkdownGenerator(
-                content_filter=PruningContentFilter(
+                content_filter=PruningContentFilterLXML(
                     threshold=0.48, threshold_type="fixed", min_word_threshold=0
                 )
             ),        
diff --git a/docs/examples/session_id_example.py b/docs/examples/session_id_example.py
index e49b7819b..4781b7053 100644
--- a/docs/examples/session_id_example.py
+++ b/docs/examples/session_id_example.py
@@ -4,7 +4,7 @@
     BrowserConfig,
     CrawlerRunConfig,
     DefaultMarkdownGenerator,
-    PruningContentFilter,
+    PruningContentFilterLXML,
     CrawlResult
 )
 
diff --git a/docs/examples/url_seeder/bbc_sport_research_assistant.py b/docs/examples/url_seeder/bbc_sport_research_assistant.py
index 51bcf179b..8e797d1d2 100644
--- a/docs/examples/url_seeder/bbc_sport_research_assistant.py
+++ b/docs/examples/url_seeder/bbc_sport_research_assistant.py
@@ -53,7 +53,7 @@
     SeedingConfig,
     AsyncLogger
 )
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
 from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
 
 # LiteLLM for AI communication
@@ -343,7 +343,7 @@ async def crawl_selected_urls(urls: List[str], query: str, config: ResearchConfi
                 
         # Configure markdown generator with content filter
         md_generator = DefaultMarkdownGenerator(
-            content_filter=PruningContentFilter(
+            content_filter=PruningContentFilterLXML(
                 threshold=0.48,
                 threshold_type="dynamic",
                 min_word_threshold=10
diff --git a/docs/examples/url_seeder/tutorial_url_seeder.md b/docs/examples/url_seeder/tutorial_url_seeder.md
index 4b9a2201d..0cda65bb8 100644
--- a/docs/examples/url_seeder/tutorial_url_seeder.md
+++ b/docs/examples/url_seeder/tutorial_url_seeder.md
@@ -73,7 +73,7 @@ from crawl4ai import (
     SeedingConfig,
     AsyncLogger
 )
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
 from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
 
 # LiteLLM for AI capabilities
@@ -357,7 +357,7 @@ async def crawl_selected_urls(
     # Configure intelligent content extraction
     # This removes ads, navigation, and other noise
     md_generator = DefaultMarkdownGenerator(
-        content_filter=PruningContentFilter(
+        content_filter=PruningContentFilterLXML(
             threshold=0.48,  # Content relevance threshold
             threshold_type="dynamic",  # Adapts to page structure
             min_word_threshold=10  # Ignore tiny text blocks
diff --git a/docs/examples/url_seeder/url_seeder_quick_demo.py b/docs/examples/url_seeder/url_seeder_quick_demo.py
index f48c5eec7..acc73ef0b 100644
--- a/docs/examples/url_seeder/url_seeder_quick_demo.py
+++ b/docs/examples/url_seeder/url_seeder_quick_demo.py
@@ -6,7 +6,7 @@
 """
 import asyncio, os
 from crawl4ai import AsyncUrlSeeder, AsyncWebCrawler, SeedingConfig, CrawlerRunConfig, AsyncLogger, DefaultMarkdownGenerator 
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
 
 CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
 
@@ -39,7 +39,7 @@ async def discover_and_crawl():
     async with AsyncWebCrawler() as crawler:
         config = CrawlerRunConfig(
             markdown_generator=DefaultMarkdownGenerator(
-                content_filter=PruningContentFilter(  # Smart filtering!
+                content_filter=PruningContentFilterLXML(  # Smart filtering!
                     threshold=0.48,  # Remove fluff
                     threshold_type="fixed",
                 ),
diff --git a/docs/md_v2/advanced/multi-url-crawling.md b/docs/md_v2/advanced/multi-url-crawling.md
index 2c924efff..994fbfcf3 100644
--- a/docs/md_v2/advanced/multi-url-crawling.md
+++ b/docs/md_v2/advanced/multi-url-crawling.md
@@ -418,7 +418,7 @@ When crawling diverse content types, you often need different configurations for
 from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, MatchMode
 from crawl4ai.processors.pdf import PDFContentScrapingStrategy
 from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
 from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
 
 async def crawl_mixed_content():
@@ -434,7 +434,7 @@ async def crawl_mixed_content():
         CrawlerRunConfig(
             url_matcher=["*/blog/*", "*/article/*"],
             markdown_generator=DefaultMarkdownGenerator(
-                content_filter=PruningContentFilter(threshold=0.48)
+                content_filter=PruningContentFilterLXML(threshold=0.48)
             )
         ),
         
diff --git a/docs/md_v2/advanced/undetected-browser.md b/docs/md_v2/advanced/undetected-browser.md
index 32b73fc5f..c709602fb 100644
--- a/docs/md_v2/advanced/undetected-browser.md
+++ b/docs/md_v2/advanced/undetected-browser.md
@@ -216,7 +216,7 @@ async def main():
         # Configure the crawl
         crawler_config = CrawlerRunConfig(
             markdown_generator=DefaultMarkdownGenerator(
-                content_filter=PruningContentFilter()
+                content_filter=PruningContentFilterLXML()
             ),
             capture_console_messages=True,  # Test adapter console capture
         )
diff --git a/docs/md_v2/api/arun_many.md b/docs/md_v2/api/arun_many.md
index 740dcc24a..96324ae24 100644
--- a/docs/md_v2/api/arun_many.md
+++ b/docs/md_v2/api/arun_many.md
@@ -105,7 +105,7 @@ Instead of using one config for all URLs, provide a list of configs with `url_ma
 from crawl4ai import CrawlerRunConfig, MatchMode
 from crawl4ai.processors.pdf import PDFContentScrapingStrategy
 from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
 from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
 
 # PDF files - specialized extraction
@@ -118,7 +118,7 @@ pdf_config = CrawlerRunConfig(
 blog_config = CrawlerRunConfig(
     url_matcher=["*/blog/*", "*/article/*", "*python.org*"],
     markdown_generator=DefaultMarkdownGenerator(
-        content_filter=PruningContentFilter(threshold=0.48)
+        content_filter=PruningContentFilterLXML(threshold=0.48)
     )
 )
 
diff --git a/docs/md_v2/api/crawl-result.md b/docs/md_v2/api/crawl-result.md
index 8890a7043..f061836c9 100644
--- a/docs/md_v2/api/crawl-result.md
+++ b/docs/md_v2/api/crawl-result.md
@@ -159,7 +159,7 @@ print(result.markdown.raw_markdown[:200])
 print(result.markdown.fit_markdown)
 print(result.markdown.fit_html)
 ```
-**Important**: "Fit" content (in `fit_markdown`/`fit_html`) exists in result.markdown, only if you used a **filter** (like **PruningContentFilter** or **BM25ContentFilter**) within a `MarkdownGenerationStrategy`.
+**Important**: "Fit" content (in `fit_markdown`/`fit_html`) exists in result.markdown, only if you used a **filter** (like **PruningContentFilterLXML** or **BM25ContentFilter**) within a `MarkdownGenerationStrategy`.
 
 ---
 
@@ -414,7 +414,7 @@ async def handle_result(result: CrawlResult):
    - `fit_markdown` and `fit_html` - No longer top-level properties in v0.5. Use `result.markdown.fit_markdown` and `result.markdown.fit_html`.
 
 2. **Fit Content**  
-   - **`fit_markdown`** and **`fit_html`** appear in MarkdownGenerationResult, only if you used a content filter (like **PruningContentFilter** or **BM25ContentFilter**) inside your **MarkdownGenerationStrategy** or set them directly.  
+   - **`fit_markdown`** and **`fit_html`** appear in MarkdownGenerationResult, only if you used a content filter (like **PruningContentFilterLXML** or **BM25ContentFilter**) inside your **MarkdownGenerationStrategy** or set them directly.  
    - If no filter is used, they remain `None`.
 
 3. **References & Citations**  
diff --git a/docs/md_v2/complete-sdk-reference.md b/docs/md_v2/complete-sdk-reference.md
index aa0517b2d..938111dcd 100644
--- a/docs/md_v2/complete-sdk-reference.md
+++ b/docs/md_v2/complete-sdk-reference.md
@@ -151,15 +151,15 @@ if __name__ == "__main__":
 ## 4. Generating Markdown Output
 - **`result.markdown`**:  
 - **`result.markdown.fit_markdown`**:  
-  The same content after applying any configured **content filter** (e.g., `PruningContentFilter`).
+  The same content after applying any configured **content filter** (e.g., `PruningContentFilterLXML`).
 ### Example: Using a Filter with `DefaultMarkdownGenerator`
 ```python
 from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
 from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
 
 md_generator = DefaultMarkdownGenerator(
-    content_filter=PruningContentFilter(threshold=0.4, threshold_type="fixed")
+    content_filter=PruningContentFilterLXML(threshold=0.4, threshold_type="fixed")
 )
 
 config = CrawlerRunConfig(
@@ -172,7 +172,7 @@ async with AsyncWebCrawler() as crawler:
     print("Raw Markdown length:", len(result.markdown.raw_markdown))
     print("Fit Markdown length:", len(result.markdown.fit_markdown))
 ```
-**Note**: If you do **not** specify a content filter or markdown generator, you’ll typically see only the raw Markdown. `PruningContentFilter` may adds around `50ms` in processing time. We’ll dive deeper into these strategies in a dedicated **Markdown Generation** tutorial.
+**Note**: If you do **not** specify a content filter or markdown generator, you’ll typically see only the raw Markdown. `PruningContentFilterLXML` may adds around `50ms` in processing time. We’ll dive deeper into these strategies in a dedicated **Markdown Generation** tutorial.
 ## 5. Simple Data Extraction (CSS-based)
 ```python
 from crawl4ai import JsonCssExtractionStrategy
@@ -1006,7 +1006,7 @@ Instead of using one config for all URLs, provide a list of configs with `url_ma
 from crawl4ai import CrawlerRunConfig, MatchMode
 from crawl4ai.processors.pdf import PDFContentScrapingStrategy
 from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
 from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
 
 # PDF files - specialized extraction
@@ -1019,7 +1019,7 @@ pdf_config = CrawlerRunConfig(
 blog_config = CrawlerRunConfig(
     url_matcher=["*/blog/*", "*/article/*", "*python.org*"],
     markdown_generator=DefaultMarkdownGenerator(
-        content_filter=PruningContentFilter(threshold=0.48)
+        content_filter=PruningContentFilterLXML(threshold=0.48)
     )
 )
 
@@ -1171,7 +1171,7 @@ print(result.markdown.raw_markdown[:200])
 print(result.markdown.fit_markdown)
 print(result.markdown.fit_html)
 ```
-**Important**: "Fit" content (in `fit_markdown`/`fit_html`) exists in result.markdown, only if you used a **filter** (like **PruningContentFilter** or **BM25ContentFilter**) within a `MarkdownGenerationStrategy`.
+**Important**: "Fit" content (in `fit_markdown`/`fit_html`) exists in result.markdown, only if you used a **filter** (like **PruningContentFilterLXML** or **BM25ContentFilter**) within a `MarkdownGenerationStrategy`.
 ## 4. Media & Links
 ### 4.1 **`media`** *(Dict[str, List[Dict]])*  
 **What**: Contains info about discovered images, videos, or audio. Typically keys: `"images"`, `"videos"`, `"audios"`.  
@@ -1363,7 +1363,7 @@ async def handle_result(result: CrawlResult):
    - `markdown_v2` - Removed in v0.5. Accessing it raises `AttributeError`. Use `markdown`.
    - `fit_markdown` and `fit_html` - Removed as top-level `CrawlResult` properties in v0.5. Use `result.markdown.fit_markdown` and `result.markdown.fit_html`.
 2. **Fit Content**  
-   - **`fit_markdown`** and **`fit_html`** appear in MarkdownGenerationResult, only if you used a content filter (like **PruningContentFilter** or **BM25ContentFilter**) inside your **MarkdownGenerationStrategy** or set them directly.  
+   - **`fit_markdown`** and **`fit_html`** appear in MarkdownGenerationResult, only if you used a content filter (like **PruningContentFilterLXML** or **BM25ContentFilter**) inside your **MarkdownGenerationStrategy** or set them directly.  
    - If no filter is used, they remain `None`.
 3. **References & Citations**  
    - If you enable link citations in your `DefaultMarkdownGenerator` (`options={"citations": True}`), you’ll see `markdown_with_citations` plus a **`references_markdown`** block. This helps large language models or academic-like referencing.
@@ -2038,7 +2038,7 @@ The `arun()` method returns a `CrawlResult` object with several useful propertie
 ```python
 config = CrawlerRunConfig(
     markdown_generator=DefaultMarkdownGenerator(
-        content_filter=PruningContentFilter(threshold=0.6),
+        content_filter=PruningContentFilterLXML(threshold=0.6),
         options={"ignore_links": True}
     )
 )
@@ -2293,12 +2293,12 @@ config = CrawlerRunConfig(markdown_generator=md_generator)
 - **`bm25_threshold`**: Raise it to keep fewer blocks; lower it to keep more.  
 - **`use_stemming`** *(default `True`)*: Whether to apply stemming to the query and content.
 - **`language (str)`**: Language for stemming (default: 'english').
-### 5.2 PruningContentFilter
-If you **don’t** have a specific query, or if you just want a robust “junk remover,” use `PruningContentFilter`. It analyzes text density, link density, HTML structure, and known patterns (like “nav,” “footer”) to systematically prune extraneous or repetitive sections.
+### 5.2 PruningContentFilterLXML
+If you **don’t** have a specific query, or if you just want a robust “junk remover,” use `PruningContentFilterLXML`. It analyzes text density, link density, HTML structure, and known patterns (like “nav,” “footer”) to systematically prune extraneous or repetitive sections.
 ```python
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
 
-prune_filter = PruningContentFilter(
+prune_filter = PruningContentFilterLXML(
     threshold=0.5,
     threshold_type="fixed",  # or "dynamic"
     min_word_threshold=50
@@ -2383,12 +2383,12 @@ When a content filter is active, the library produces two forms of markdown insi
 import asyncio
 from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
 from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
 
 async def main():
     config = CrawlerRunConfig(
         markdown_generator=DefaultMarkdownGenerator(
-            content_filter=PruningContentFilter(threshold=0.6),
+            content_filter=PruningContentFilterLXML(threshold=0.6),
             options={"ignore_links": True}
         )
     )
@@ -2424,14 +2424,14 @@ print("FIT:\n", md_obj.fit_markdown)
 - Or feed `fit_markdown` into a vector database to reduce token usage.  
 - `references_markdown` can help you keep track of link provenance.
 ## 8. Combining Filters (BM25 + Pruning) in Two Passes
-You might want to **prune out** noisy boilerplate first (with `PruningContentFilter`), and then **rank what’s left** against a user query (with `BM25ContentFilter`). You don’t have to crawl the page twice. Instead:
-1. **First pass**: Apply `PruningContentFilter` directly to the raw HTML from `result.html` (the crawler’s downloaded HTML).  
+You might want to **prune out** noisy boilerplate first (with `PruningContentFilterLXML`), and then **rank what’s left** against a user query (with `BM25ContentFilter`). You don’t have to crawl the page twice. Instead:
+1. **First pass**: Apply `PruningContentFilterLXML` directly to the raw HTML from `result.html` (the crawler’s downloaded HTML).  
 2. **Second pass**: Take the pruned HTML (or text) from step 1, and feed it into `BM25ContentFilter`, focusing on a user query.
 ### Two-Pass Example
 ```python
 import asyncio
 from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
-from crawl4ai.content_filter_strategy import PruningContentFilter, BM25ContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML, BM25ContentFilter
 from bs4 import BeautifulSoup
 
 async def main():
@@ -2450,8 +2450,8 @@ async def main():
 
         raw_html = result.html
 
-        # 2. First pass: PruningContentFilter on raw HTML
-        pruning_filter = PruningContentFilter(threshold=0.5, min_word_threshold=50)
+        # 2. First pass: PruningContentFilterLXML on raw HTML
+        pruning_filter = PruningContentFilterLXML(threshold=0.5, min_word_threshold=50)
 
         # filter_content returns a list of "text chunks" or cleaned HTML sections
         pruned_chunks = pruning_filter.filter_content(raw_html)
@@ -2518,17 +2518,24 @@ In **`CrawlerRunConfig`**, you can specify a **`content_filter`** to shape how c
 - **`result.markdown.fit_markdown`** (filtered or “fit” version)
 - **`result.markdown.fit_html`** (the corresponding HTML snippet that produced `fit_markdown`)
 ### 1.2 Common Filters
-## 2. PruningContentFilter
+## 2. PruningContentFilterLXML
+
+!!! warning "Deprecation"
+    Use `PruningContentFilterLXML` (lxml engine, **~10× faster**, identical output). The legacy
+    `PruningContentFilter` (BeautifulSoup) still works but emits a `DeprecationWarning`. In an
+    upcoming release `PruningContentFilter` will become an alias for the lxml engine, with
+    `PruningContentFilterLXML` kept as a legacy alias.
+
 ### 2.1 Usage Example
 ```python
 import asyncio
 from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
 from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
 
 async def main():
     # Step 1: Create a pruning filter
-    prune_filter = PruningContentFilter(
+    prune_filter = PruningContentFilterLXML(
         # Lower → more content retained, higher → more content pruned
         threshold=0.45,           
         # "fixed" or "dynamic"
@@ -2623,7 +2630,7 @@ If the content filter is **BM25**, you might see additional logic or references
 ## 5. Code Patterns Recap
 ### 5.1 Pruning
 ```python
-prune_filter = PruningContentFilter(
+prune_filter = PruningContentFilterLXML(
     threshold=0.5,
     threshold_type="fixed",
     min_word_threshold=10
@@ -2647,7 +2654,7 @@ config = CrawlerRunConfig(
     excluded_tags=["nav", "footer", "header"],
     exclude_external_links=True,
     markdown_generator=DefaultMarkdownGenerator(
-        content_filter=PruningContentFilter(threshold=0.5)
+        content_filter=PruningContentFilterLXML(threshold=0.5)
     )
 )
 ```
@@ -4698,7 +4705,7 @@ To keep context continuous across chunks, we can overlap them. E.g., `overlap_ra
 ## 7. Input Format
 By default, **LLMExtractionStrategy** uses `input_format="markdown"`, meaning the **crawler’s final markdown** is fed to the LLM. You can change to:
 - **`html`**: The cleaned HTML or raw HTML (depending on your crawler config) goes into the LLM.  
-- **`fit_markdown`**: If you used, for instance, `PruningContentFilter`, the “fit” version of the markdown is used. This can drastically reduce tokens if you trust the filter.  
+- **`fit_markdown`**: If you used, for instance, `PruningContentFilterLXML`, the “fit” version of the markdown is used. This can drastically reduce tokens if you trust the filter.  
 - **`markdown`**: Standard markdown output from the crawler’s `markdown_generator`.
 This setting is crucial: if the LLM instructions rely on HTML tags, pick `"html"`. If you prefer a text-based approach, pick `"markdown"`.
 ```python
diff --git a/docs/md_v2/core/crawler-result.md b/docs/md_v2/core/crawler-result.md
index a531e043e..79f723185 100644
--- a/docs/md_v2/core/crawler-result.md
+++ b/docs/md_v2/core/crawler-result.md
@@ -141,7 +141,7 @@ print(md_res.markdown_with_citations)
 print(md_res.references_markdown)
 ```
 
-**Note**: If you use a filter like `PruningContentFilter`, you’ll get `fit_markdown` and `fit_html` as well.
+**Note**: If you use a filter like `PruningContentFilterLXML`, you’ll get `fit_markdown` and `fit_html` as well.
 
 ---
 
@@ -336,7 +336,7 @@ else:
 ## 7. Next Steps
 
 - **Markdown Generation**: Dive deeper into how to configure `DefaultMarkdownGenerator` and various filters.  
-- **Content Filtering**: Learn how to use `BM25ContentFilter` and `PruningContentFilter`.
+- **Content Filtering**: Learn how to use `BM25ContentFilter` and `PruningContentFilterLXML`.
 - **Session & Hooks**: If you want to manipulate the page or preserve state across multiple `arun()` calls, see the hooking or session docs.  
 - **LLM Extraction**: For complex or unstructured content requiring AI-driven parsing, check the LLM-based strategies doc.
 
diff --git a/docs/md_v2/core/fit-markdown.md b/docs/md_v2/core/fit-markdown.md
index 1684a7790..149496ca1 100644
--- a/docs/md_v2/core/fit-markdown.md
+++ b/docs/md_v2/core/fit-markdown.md
@@ -1,5 +1,22 @@
 # Fit Markdown with Pruning & BM25
 
+!!! warning "Deprecation: `PruningContentFilter` → `PruningContentFilterLXML`"
+    The original `PruningContentFilter` (BeautifulSoup-based) is **deprecated** in favor of
+    **`PruningContentFilterLXML`**, an lxml reimplementation that is **~10× faster** on large
+    pages with **identical output**. It is already the default for the `crwl` CLI and the
+    Docker server's fit-markdown endpoint.
+
+    **Migrate** by swapping the class name (every parameter is unchanged):
+
+    ```python
+    from crawl4ai import PruningContentFilterLXML
+    prune_filter = PruningContentFilterLXML(threshold=0.48, threshold_type="dynamic")
+    ```
+
+    Instantiating the old `PruningContentFilter` now emits a `DeprecationWarning`. **Roadmap:**
+    in an upcoming release `PruningContentFilter` will become an alias for the lxml engine, and
+    `PruningContentFilterLXML` will be kept as a legacy alias.
+
 **Fit Markdown** is a specialized **filtered** version of your page’s markdown, focusing on the most relevant content. By default, Crawl4AI converts the entire HTML into a broad **raw_markdown**. With fit markdown, we apply a **content filter** algorithm (e.g., **Pruning** or **BM25**) to remove or rank low-value sections—such as repetitive sidebars, shallow text blocks, or irrelevancies—leaving a concise textual “core.”
 
 ---
@@ -17,12 +34,12 @@ In **`CrawlerRunConfig`**, you can specify a **`content_filter`** to shape how c
 
 ### 1.2 Common Filters
 
-1. **PruningContentFilter** – Scores each node by text density, link density, and tag importance, discarding those below a threshold.  
+1. **PruningContentFilterLXML** – Scores each node by text density, link density, and tag importance, discarding those below a threshold.  
 2. **BM25ContentFilter** – Focuses on textual relevance using BM25 ranking, especially useful if you have a specific user query (e.g., “machine learning” or “food nutrition”).
 
 ---
 
-## 2. PruningContentFilter
+## 2. PruningContentFilterLXML
 
 **Pruning** discards less relevant nodes based on **text density, link density, and tag importance**. It’s a heuristic-based approach—if certain sections appear too “thin” or too “spammy,” they’re pruned.
 
@@ -31,12 +48,12 @@ In **`CrawlerRunConfig`**, you can specify a **`content_filter`** to shape how c
 ```python
 import asyncio
 from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
 from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
 
 async def main():
     # Step 1: Create a pruning filter
-    prune_filter = PruningContentFilter(
+    prune_filter = PruningContentFilterLXML(
         # Lower → more content retained, higher → more content pruned
         threshold=0.45,           
         # "fixed" or "dynamic"
@@ -159,7 +176,7 @@ If the content filter is **BM25**, you might see additional logic or references
 ### 5.1 Pruning
 
 ```python
-prune_filter = PruningContentFilter(
+prune_filter = PruningContentFilterLXML(
     threshold=0.5,
     threshold_type="fixed",
     min_word_threshold=10
@@ -191,7 +208,7 @@ config = CrawlerRunConfig(
     excluded_tags=["nav", "footer", "header"],
     exclude_external_links=True,
     markdown_generator=DefaultMarkdownGenerator(
-        content_filter=PruningContentFilter(threshold=0.5)
+        content_filter=PruningContentFilterLXML(threshold=0.5)
     )
 )
 ```
@@ -235,7 +252,7 @@ class MyCustomFilter(RelevantContentFilter):
 - **AI Pipelines**: Filter out boilerplate so LLM-based extraction or summarization runs on denser text.
 
 **Key Points**:
-- **PruningContentFilter**: Great if you just want the “meatiest” text without a user query.  
+- **PruningContentFilterLXML**: Great if you just want the “meatiest” text without a user query.  
 - **BM25ContentFilter**: Perfect for query-based extraction or searching.  
 - Combine with **`excluded_tags`, `exclude_external_links`, `word_count_threshold`** to refine your final “fit” text.  
 - Fit markdown ends up in **`result.markdown.fit_markdown`**; eventually **`result.markdown.fit_markdown`** in future versions.
diff --git a/docs/md_v2/core/markdown-generation.md b/docs/md_v2/core/markdown-generation.md
index af9b35b5b..6d231a217 100644
--- a/docs/md_v2/core/markdown-generation.md
+++ b/docs/md_v2/core/markdown-generation.md
@@ -205,14 +205,20 @@ config = CrawlerRunConfig(markdown_generator=md_generator)
 
 **No query provided?** BM25 tries to glean a context from page metadata, or you can simply treat it as a scorched-earth approach that discards text with low generic score. Realistically, you want to supply a query for best results.
 
-### 5.2 PruningContentFilter
+### 5.2 PruningContentFilterLXML
 
-If you **don’t** have a specific query, or if you just want a robust “junk remover,” use `PruningContentFilter`. It analyzes text density, link density, HTML structure, and known patterns (like “nav,” “footer”) to systematically prune extraneous or repetitive sections.
+!!! warning "Use `PruningContentFilterLXML` (the old `PruningContentFilter` is deprecated)"
+    `PruningContentFilterLXML` is the lxml-based engine: **~10× faster** on large pages with
+    **identical output**. The legacy `PruningContentFilter` still works but emits a
+    `DeprecationWarning`; in an upcoming release `PruningContentFilter` will alias the lxml
+    engine and `PruningContentFilterLXML` will remain as a legacy alias.
+
+If you **don’t** have a specific query, or if you just want a robust “junk remover,” use `PruningContentFilterLXML`. It analyzes text density, link density, HTML structure, and known patterns (like “nav,” “footer”) to systematically prune extraneous or repetitive sections.
 
 ```python
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
 
-prune_filter = PruningContentFilter(
+prune_filter = PruningContentFilterLXML(
     threshold=0.5,
     threshold_type="fixed",  # or "dynamic"
     min_word_threshold=50
@@ -225,7 +231,7 @@ prune_filter = PruningContentFilter(
     - `"dynamic"`: The filter adjusts threshold in a data-driven manner.  
 - **`min_word_threshold`**: Discard blocks under N words as likely too short or unhelpful.
 
-**When to Use PruningContentFilter**  
+**When to Use PruningContentFilterLXML**  
 - You want a broad cleanup without a user query.  
 - The page has lots of repeated sidebars, footers, or disclaimers that hamper text extraction.
 
@@ -320,12 +326,12 @@ When a content filter is active, the library produces two forms of markdown insi
 import asyncio
 from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
 from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
 
 async def main():
     config = CrawlerRunConfig(
         markdown_generator=DefaultMarkdownGenerator(
-            content_filter=PruningContentFilter(threshold=0.6),
+            content_filter=PruningContentFilterLXML(threshold=0.6),
             options={"ignore_links": True}
         )
     )
@@ -379,9 +385,9 @@ Below is a **revised section** under “Combining Filters (BM25 + Pruning)” th
 
 ## 8. Combining Filters (BM25 + Pruning) in Two Passes
 
-You might want to **prune out** noisy boilerplate first (with `PruningContentFilter`), and then **rank what’s left** against a user query (with `BM25ContentFilter`). You don’t have to crawl the page twice. Instead:
+You might want to **prune out** noisy boilerplate first (with `PruningContentFilterLXML`), and then **rank what’s left** against a user query (with `BM25ContentFilter`). You don’t have to crawl the page twice. Instead:
 
-1. **First pass**: Apply `PruningContentFilter` directly to the raw HTML from `result.html` (the crawler’s downloaded HTML).  
+1. **First pass**: Apply `PruningContentFilterLXML` directly to the raw HTML from `result.html` (the crawler’s downloaded HTML).  
 2. **Second pass**: Take the pruned HTML (or text) from step 1, and feed it into `BM25ContentFilter`, focusing on a user query.
 
 ### Two-Pass Example
@@ -389,7 +395,7 @@ You might want to **prune out** noisy boilerplate first (with `PruningContentFil
 ```python
 import asyncio
 from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
-from crawl4ai.content_filter_strategy import PruningContentFilter, BM25ContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML, BM25ContentFilter
 from bs4 import BeautifulSoup
 
 async def main():
@@ -408,8 +414,8 @@ async def main():
         
         raw_html = result.html
         
-        # 2. First pass: PruningContentFilter on raw HTML
-        pruning_filter = PruningContentFilter(threshold=0.5, min_word_threshold=50)
+        # 2. First pass: PruningContentFilterLXML on raw HTML
+        pruning_filter = PruningContentFilterLXML(threshold=0.5, min_word_threshold=50)
         
         # filter_content returns a list of "text chunks" or cleaned HTML sections
         pruned_chunks = pruning_filter.filter_content(raw_html)
@@ -449,7 +455,7 @@ if __name__ == "__main__":
 ### What’s Happening?
 
 1. **Raw HTML**: We crawl once and store the raw HTML in `result.html`.  
-2. **PruningContentFilter**: Takes HTML + optional parameters. It extracts blocks of text or partial HTML, removing headings/sections deemed “noise.” It returns a **list of text chunks**.  
+2. **PruningContentFilterLXML**: Takes HTML + optional parameters. It extracts blocks of text or partial HTML, removing headings/sections deemed “noise.” It returns a **list of text chunks**.  
 3. **Combine or Transform**: We join these pruned chunks back into a single HTML-like string. (Alternatively, you could store them in a list for further logic—whatever suits your pipeline.)  
 4. **BM25ContentFilter**: We feed the pruned string into `BM25ContentFilter` with a user query. This second pass further narrows the content to chunks relevant to “machine learning.”
 
@@ -495,7 +501,7 @@ In this **Markdown Generation Basics** tutorial, you learned to:
 
 - Configure the **DefaultMarkdownGenerator** with HTML-to-text options.  
 - Select different HTML sources using the `content_source` parameter.  
-- Use **BM25ContentFilter** for query-specific extraction or **PruningContentFilter** for general noise removal.  
+- Use **BM25ContentFilter** for query-specific extraction or **PruningContentFilterLXML** for general noise removal.  
 - Distinguish between raw and filtered markdown (`fit_markdown`).  
 - Leverage the `MarkdownGenerationResult` object to handle different forms of output (citations, references, etc.).
 
diff --git a/docs/md_v2/core/quickstart.md b/docs/md_v2/core/quickstart.md
index 78209efde..063c93c6d 100644
--- a/docs/md_v2/core/quickstart.md
+++ b/docs/md_v2/core/quickstart.md
@@ -92,17 +92,17 @@ By default, Crawl4AI automatically generates Markdown from each crawled page. Ho
 - **`result.markdown`**:  
   The direct HTML-to-Markdown conversion.  
 - **`result.markdown.fit_markdown`**:  
-  The same content after applying any configured **content filter** (e.g., `PruningContentFilter`).
+  The same content after applying any configured **content filter** (e.g., `PruningContentFilterLXML`).
 
 ### Example: Using a Filter with `DefaultMarkdownGenerator`
 
 ```python
 from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode
-from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.content_filter_strategy import PruningContentFilterLXML
 from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
 
 md_generator = DefaultMarkdownGenerator(
-    content_filter=PruningContentFilter(threshold=0.4, threshold_type="fixed")
+    content_filter=PruningContentFilterLXML(threshold=0.4, threshold_type="fixed")
 )
 
 config = CrawlerRunConfig(
@@ -116,7 +116,7 @@ async with AsyncWebCrawler() as crawler:
     print("Fit Markdown length:", len(result.markdown.fit_markdown))
 ```
 
-**Note**: If you do **not** specify a content filter or markdown generator, you’ll typically see only the raw Markdown. `PruningContentFilter` may adds around `50ms` in processing time. We’ll dive deeper into these strategies in a dedicated **Markdown Generation** tutorial.
+**Note**: If you do **not** specify a content filter or markdown generator, you’ll typically see only the raw Markdown. `PruningContentFilterLXML` may adds around `50ms` in processing time. We’ll dive deeper into these strategies in a dedicated **Markdown Generation** tutorial.
 
 ---
 
diff --git a/docs/md_v2/core/simple-crawling.md b/docs/md_v2/core/simple-crawling.md
index dd31bc973..c9b785336 100644
--- a/docs/md_v2/core/simple-crawling.md
+++ b/docs/md_v2/core/simple-crawling.md
@@ -33,7 +33,7 @@ The `arun()` method returns a `CrawlResult` object with several useful propertie
 ```python
 config = CrawlerRunConfig(
     markdown_generator=DefaultMarkdownGenerator(
-        content_filter=PruningContentFilter(threshold=0.6),
+        content_filter=PruningContentFilterLXML(threshold=0.6),
         options={"ignore_links": True}
     )
 )
diff --git a/docs/md_v2/extraction/llm-strategies.md b/docs/md_v2/extraction/llm-strategies.md
index cba4d6e4f..197d2ba26 100644
--- a/docs/md_v2/extraction/llm-strategies.md
+++ b/docs/md_v2/extraction/llm-strategies.md
@@ -175,7 +175,7 @@ By chunking, you can potentially process multiple chunks in parallel (depending
 By default, **LLMExtractionStrategy** uses `input_format="markdown"`, meaning the **crawler’s final markdown** is fed to the LLM. You can change to:
 
 - **`html`**: The cleaned HTML or raw HTML (depending on your crawler config) goes into the LLM.  
-- **`fit_markdown`**: If you used, for instance, `PruningContentFilter`, the “fit” version of the markdown is used. This can drastically reduce tokens if you trust the filter.  
+- **`fit_markdown`**: If you used, for instance, `PruningContentFilterLXML`, the “fit” version of the markdown is used. This can drastically reduce tokens if you trust the filter.  
 - **`markdown`**: Standard markdown output from the crawler’s `markdown_generator`.
 
 This setting is crucial: if the LLM instructions rely on HTML tags, pick `"html"`. If you prefer a text-based approach, pick `"markdown"`.
diff --git a/tests/async/test_content_filter_prune_lxml.py b/tests/async/test_content_filter_prune_lxml.py
index afc1226f4..889387ebb 100644
--- a/tests/async/test_content_filter_prune_lxml.py
+++ b/tests/async/test_content_filter_prune_lxml.py
@@ -153,6 +153,7 @@ def test_consistent_output(self, basic_html):
 # --------------------------------------------------------------------------- #
 # Fidelity: new output must match the original bs4 implementation
 # --------------------------------------------------------------------------- #
+@pytest.mark.filterwarnings("ignore::DeprecationWarning")  # deliberately exercises the deprecated bs4 filter
 class TestPruningEquivalence:
     @pytest.mark.parametrize("name", list(EQUIVALENCE_CASES.keys()))
     @pytest.mark.parametrize("config", CONFIGS)
@@ -182,6 +183,7 @@ def test_fixture_files_match_original(self):
 # --------------------------------------------------------------------------- #
 # Performance: new must be substantially faster on a non-trivial page
 # --------------------------------------------------------------------------- #
+@pytest.mark.filterwarnings("ignore::DeprecationWarning")  # compares against the deprecated bs4 filter
 def test_performance_faster_than_original():
     # ~600 repeated cards -> a few hundred KB of HTML
     html = "<body>" + "".join(

From a59f50e4a21e7100abaf500f8af912e06abda8ee Mon Sep 17 00:00:00 2001
From: Danilo Alonso <damusix@gmail.com>
Date: Fri, 28 Aug 2026 08:36:04 -0400
Subject: [PATCH 03/26] Fix: Make the untrusted timeout ceiling configurable

page_timeout, wait_for_timeout and body_visibility_timeout are clamped to
60s for any config arriving over HTTP, and the value was a module literal
with no env var, no config.yml key, and no way for an operator to change
it. A deployment that is not public could not crawl a page that
legitimately takes longer.

CRAWL4AI_MAX_TIMEOUT_MS now sets the ceiling, defaulting to the same
60000ms. A value that is not a positive integer warns and keeps the
default, since a typo would otherwise silently widen a DoS bound.

Read per call rather than captured at import, so the setting applies
wherever the process picked its environment up.

Closes #2211
---
 crawl4ai/async_configs.py     | 43 +++++++++++++++++--
 deploy/docker/MIGRATION.md    | 19 +++++++++
 tests/test_config_defaults.py | 80 ++++++++++++++++++++++++++++++++++-
 3 files changed, 138 insertions(+), 4 deletions(-)

diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py
index 3efb0d380..92ae71ed6 100644
--- a/crawl4ai/async_configs.py
+++ b/crawl4ai/async_configs.py
@@ -270,11 +270,46 @@ class UntrustedConfigError(ValueError):
 }
 
 # Upper bounds applied to attacker-influenced quantities after filtering.
-_MAX_TIMEOUT_MS = 60_000
+_DEFAULT_MAX_TIMEOUT_MS = 60_000
 _MAX_SCROLL_STEPS = 1000
 _MAX_VIEWPORT = 4000
 
 
+def _max_timeout_ms() -> int:
+    """Ceiling for the untrusted timeout fields, in milliseconds.
+
+    60s is the right bound for a server reachable by untrusted callers, and
+    stays the default. An operator whose deployment is not public — a crawler
+    on a private network fetching pages that legitimately take minutes — can
+    raise it with CRAWL4AI_MAX_TIMEOUT_MS, or lower it to tighten the bound.
+
+    Read per call rather than captured at import so the setting applies
+    wherever the process picked its environment up, and so a test can set it
+    without reloading the module. A value that is not a positive integer is
+    refused loudly and the default kept: a typo here would silently widen a
+    DoS bound, which is the one outcome worse than the timeout being fixed.
+    """
+    raw = os.getenv("CRAWL4AI_MAX_TIMEOUT_MS")
+
+    if raw is None or raw == "":
+        return _DEFAULT_MAX_TIMEOUT_MS
+
+    try:
+        ceiling = int(raw)
+    except ValueError:
+        ceiling = 0
+
+    if ceiling <= 0:
+        warnings.warn(
+            f"CRAWL4AI_MAX_TIMEOUT_MS={raw!r} is not a positive integer; "
+            f"keeping the {_DEFAULT_MAX_TIMEOUT_MS}ms default.",
+            stacklevel=2,
+        )
+        return _DEFAULT_MAX_TIMEOUT_MS
+
+    return ceiling
+
+
 def _filter_untrusted_fields(type_name: str, params: dict) -> dict:
     """Drop non-allowlisted fields and raise on forbidden (power) fields."""
     forbidden = UNTRUSTED_FORBIDDEN_FIELDS.get(type_name, set())
@@ -293,11 +328,13 @@ def _filter_untrusted_fields(type_name: str, params: dict) -> dict:
 
 def _clamp_untrusted(type_name: str, params: dict) -> dict:
     """Clamp attacker-influenced quantities to safe upper bounds."""
+    ceiling = _max_timeout_ms()
+
     def _cap_timeout(v):
         # 0 historically meant "no timeout"; treat as the cap, never unbounded.
         if not isinstance(v, (int, float)) or v <= 0:
-            return _MAX_TIMEOUT_MS
-        return min(int(v), _MAX_TIMEOUT_MS)
+            return ceiling
+        return min(int(v), ceiling)
 
     if type_name == "CrawlerRunConfig":
         for f in ("page_timeout", "wait_for_timeout", "body_visibility_timeout"):
diff --git a/deploy/docker/MIGRATION.md b/deploy/docker/MIGRATION.md
index bcd2097c7..c508e220f 100644
--- a/deploy/docker/MIGRATION.md
+++ b/deploy/docker/MIGRATION.md
@@ -170,6 +170,25 @@ limits:
 
 To keep the previous behavior exactly, set the caps you don't want to `0`.
 
+### Timeouts from a request are capped at 60s
+
+`page_timeout`, `wait_for_timeout`, and `body_visibility_timeout` arriving in a
+request body are clamped to 60000ms, so a client asking for more is given 60s
+and its crawl fails with `Page.goto: Timeout 60000ms exceeded`.
+
+That bound is right for a server reachable by untrusted callers. A deployment
+that is not public — a crawler on a private network fetching pages that
+legitimately take minutes — can raise it:
+
+```bash
+CRAWL4AI_MAX_TIMEOUT_MS=300000
+```
+
+A request still only gets the timeout it asks for; this sets the ceiling, and
+a smaller value tightens it. A value that is not a positive integer is refused
+with a warning and the 60000ms default kept, so a typo cannot silently widen
+the bound.
+
 ### Error responses are generic
 
 5xx responses return `{"error": "Internal server error", "correlation_id": "…"}`.
diff --git a/tests/test_config_defaults.py b/tests/test_config_defaults.py
index 3a7fba641..ad86f92f0 100644
--- a/tests/test_config_defaults.py
+++ b/tests/test_config_defaults.py
@@ -1,11 +1,12 @@
 """Tests for BrowserConfig.set_defaults / CrawlerRunConfig.set_defaults."""
 
+import warnings
 from types import SimpleNamespace
 from unittest.mock import AsyncMock, MagicMock
 
 import pytest
 
-from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig, Provenance
 from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
 
 
@@ -310,3 +311,80 @@ def test_independent_reset(self):
         BrowserConfig.reset_defaults()
         assert BrowserConfig.get_defaults() == {}
         assert CrawlerRunConfig.get_defaults() == {"verbose": False}
+
+
+# ── Untrusted timeout ceiling ──────────────────────────────────────────
+
+
+class TestMaxTimeoutCeiling:
+    """CRAWL4AI_MAX_TIMEOUT_MS raises (or lowers) the untrusted clamp."""
+
+    TIMEOUT_FIELDS = ("page_timeout", "wait_for_timeout", "body_visibility_timeout")
+
+    @pytest.mark.parametrize("field", TIMEOUT_FIELDS)
+    def test_defaults_to_60s_when_unset(self, monkeypatch, field):
+        monkeypatch.delenv("CRAWL4AI_MAX_TIMEOUT_MS", raising=False)
+
+        config = CrawlerRunConfig.load(
+            {field: 500_000}, provenance=Provenance.UNTRUSTED
+        )
+
+        assert getattr(config, field) == 60_000
+
+    @pytest.mark.parametrize("field", TIMEOUT_FIELDS)
+    def test_env_raises_the_ceiling(self, monkeypatch, field):
+        monkeypatch.setenv("CRAWL4AI_MAX_TIMEOUT_MS", "300000")
+
+        config = CrawlerRunConfig.load(
+            {field: 300_000}, provenance=Provenance.UNTRUSTED
+        )
+
+        assert getattr(config, field) == 300_000
+
+    def test_a_request_over_the_raised_ceiling_is_still_clamped(self, monkeypatch):
+        monkeypatch.setenv("CRAWL4AI_MAX_TIMEOUT_MS", "300000")
+
+        config = CrawlerRunConfig.load(
+            {"page_timeout": 900_000}, provenance=Provenance.UNTRUSTED
+        )
+
+        assert config.page_timeout == 300_000
+
+    def test_env_can_tighten_the_ceiling(self, monkeypatch):
+        monkeypatch.setenv("CRAWL4AI_MAX_TIMEOUT_MS", "5000")
+
+        config = CrawlerRunConfig.load(
+            {"page_timeout": 30_000}, provenance=Provenance.UNTRUSTED
+        )
+
+        assert config.page_timeout == 5_000
+
+    # A typo must not silently widen a DoS bound, so the default is kept and
+    # the operator is told rather than left to find out under load.
+    @pytest.mark.parametrize("value", ["", "abc", "0", "-1", "60_000", "1e5"])
+    def test_a_non_positive_integer_keeps_the_default(self, monkeypatch, value):
+        monkeypatch.setenv("CRAWL4AI_MAX_TIMEOUT_MS", value)
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+            config = CrawlerRunConfig.load(
+                {"page_timeout": 500_000}, provenance=Provenance.UNTRUSTED
+            )
+
+        assert config.page_timeout == 60_000
+
+    @pytest.mark.parametrize("value", ["abc", "0", "-1"])
+    def test_a_bad_value_warns(self, monkeypatch, value):
+        monkeypatch.setenv("CRAWL4AI_MAX_TIMEOUT_MS", value)
+
+        with pytest.warns(UserWarning, match="CRAWL4AI_MAX_TIMEOUT_MS"):
+            CrawlerRunConfig.load(
+                {"page_timeout": 1_000}, provenance=Provenance.UNTRUSTED
+            )
+
+    def test_trusted_config_is_never_clamped(self, monkeypatch):
+        monkeypatch.delenv("CRAWL4AI_MAX_TIMEOUT_MS", raising=False)
+
+        config = CrawlerRunConfig(page_timeout=900_000)
+
+        assert config.page_timeout == 900_000

From 07c70d8f82bef142278e7a300cd4d9480407ddfb Mon Sep 17 00:00:00 2001
From: Soham Kukreti <kukretisoham@gmail.com>
Date: Tue, 1 Sep 2026 13:19:58 +0530
Subject: [PATCH 04/26] fix(docker): skip /config/dump pre-flight for md/llm
 endpoints in playground

md/llm runs died before their request was sent: the pre-flight sends the legacy 'code' field, which 0.9.3 rejects on untrusted requests (fixes #2222).
---
 deploy/docker/static/playground/index.html | 46 +++++++++++-----------
 1 file changed, 24 insertions(+), 22 deletions(-)

diff --git a/deploy/docker/static/playground/index.html b/deploy/docker/static/playground/index.html
index 49d27d436..e874da472 100644
--- a/deploy/docker/static/playground/index.html
+++ b/deploy/docker/static/playground/index.html
@@ -686,29 +686,31 @@ <h2 class="font-medium text-accent">🔥 Stress Test</h2>
         async function runCrawl() {
             const endpoint = document.getElementById('endpoint').value;
             const urls = document.getElementById('urls').value.trim().split(/\n/).filter(u => u);
-            // 1) grab python from CodeMirror, validate via /config/dump
+            // 1) grab python from CodeMirror, validate via /config/dump.
+            // Advanced Config is /crawl-only, so md/llm skip this entirely.
             let advConfig = {};
-            try {
-                const cfgJson = await pyConfigToJson(); // may throw
-                if (Object.keys(cfgJson).length) {
-                    const cfgType = document.getElementById('cfg-type').value;
-                    advConfig = cfgType === 'CrawlerRunConfig'
-                        ? { crawler_config: cfgJson }
-                        : { browser_config: cfgJson };
-                }
-            } catch (err) {
-                const codeText = cm.getValue();
-                const streamFlag = /stream\s*=\s*True/i.test(codeText);
-                const isCrawlEndpoint = document.getElementById('endpoint').value === 'crawl';
-                if (isCrawlEndpoint && streamFlag) {
-                    // Fallback: minimal dump-shaped config so shouldUseStream + server load agree
-                    advConfig = { crawler_config: { type: 'CrawlerRunConfig', params: { stream: true } } };
-                } else {
-                    updateStatus('error');
-                    document.querySelector('#response-content code').textContent =
-                        JSON.stringify({ error: err.message }, null, 2);
-                    forceHighlightElement(document.querySelector('#response-content code'));
-                    return; // stop run
+            if (endpoint === 'crawl' || endpoint === 'crawl_stream') {
+                try {
+                    const cfgJson = await pyConfigToJson(); // may throw
+                    if (Object.keys(cfgJson).length) {
+                        const cfgType = document.getElementById('cfg-type').value;
+                        advConfig = cfgType === 'CrawlerRunConfig'
+                            ? { crawler_config: cfgJson }
+                            : { browser_config: cfgJson };
+                    }
+                } catch (err) {
+                    const codeText = cm.getValue();
+                    const streamFlag = /stream\s*=\s*True/i.test(codeText);
+                    if (streamFlag) {
+                        // Fallback: minimal dump-shaped config so shouldUseStream + server load agree
+                        advConfig = { crawler_config: { type: 'CrawlerRunConfig', params: { stream: true } } };
+                    } else {
+                        updateStatus('error');
+                        document.querySelector('#response-content code').textContent =
+                            JSON.stringify({ error: err.message }, null, 2);
+                        forceHighlightElement(document.querySelector('#response-content code'));
+                        return; // stop run
+                    }
                 }
             }
 

From f56c69e9051857f6a1705e8f0b6be2aa10a1c533 Mon Sep 17 00:00:00 2001
From: Krzysztof Olipra <contact@krzysztofolipra.com>
Date: Sun, 6 Sep 2026 01:35:19 +0200
Subject: [PATCH 05/26] fix: preventing disallow: /*? from blocking the whole
 website.

---
 crawl4ai/utils.py                     | 22 ++++++-
 tests/general/test_robot_parser.py    | 38 +++++++++++
 tests/unit/test_robots_query_rules.py | 92 +++++++++++++++++++++++++++
 3 files changed, 150 insertions(+), 2 deletions(-)
 create mode 100644 tests/unit/test_robots_query_rules.py

diff --git a/crawl4ai/utils.py b/crawl4ai/utils.py
index 279c27708..f4cb7a62c 100644
--- a/crawl4ai/utils.py
+++ b/crawl4ai/utils.py
@@ -249,6 +249,22 @@ def needs_update(self):
         return installed is None or installed < current
 
 
+def _preserve_bare_query(rules_text: str) -> str:
+    """Append '*' to Allow/Disallow values ending in a bare '?' (e.g. '/*?').
+
+    '/*?' and '/*?*' allow/deny exactly the same URLs; the rewrite only
+    survives parsers that would otherwise drop the trailing '?'.
+    """
+    fixed = []
+    for raw_line in rules_text.splitlines():
+        body, _, _ = raw_line.partition("#")
+        key, sep, value = body.partition(":")
+        if sep and key.strip().lower() in ("allow", "disallow") and value.strip().endswith("?"):
+            raw_line = f"{key.strip()}: {value.strip()}*"
+        fixed.append(raw_line)
+    return "\n".join(fixed)
+
+
 class RobotsParser:
     # Default 7 days cache TTL
     CACHE_TTL = 7 * 24 * 60 * 60
@@ -355,8 +371,10 @@ async def can_fetch(self, url: str, user_agent: str = "*") -> bool:
             return True
 
         # Create parser for this check
-        parser = RobotFileParser() 
-        parser.parse(rules.splitlines())
+        parser = RobotFileParser()
+        # Old Pythons drop a trailing '?' from rules, so '/*?' becomes
+        # '/*' and blocks the whole site. '/*?*' matches the same URLs.
+        parser.parse(_preserve_bare_query(rules).splitlines())
         
         # If parser can't read rules, allow access
         if not parser.mtime():
diff --git a/tests/general/test_robot_parser.py b/tests/general/test_robot_parser.py
index a2fc30f1a..6ed20fbc9 100644
--- a/tests/general/test_robot_parser.py
+++ b/tests/general/test_robot_parser.py
@@ -123,6 +123,44 @@ async def giant_robots(request):
         finally:
             await runner.cleanup()
 
+        # 4b. Test query-string disallow (Disallow: /*?) on a separate host port.
+        # Plain URLs must stay crawlable while query URLs are denied (RFC 9309).
+        async def start_query_server():
+            query_app = web.Application()
+
+            async def query_robots(request):
+                return web.Response(text="User-agent: *\nDisallow: /*?\n")
+
+            query_app.router.add_get('/robots.txt', query_robots)
+            query_runner = web.AppRunner(query_app)
+            await query_runner.setup()
+            query_site = web.TCPSite(query_runner, 'localhost', 8081)
+            await query_site.start()
+            return query_runner
+
+        query_runner = await start_query_server()
+        try:
+            print("\n4b. Testing query-string robots.txt rules...")
+            query_base = "http://localhost:8081"
+
+            result = await parser.can_fetch(f"{query_base}/", "bot")
+            print(f"Plain root (/): {'allowed' if result else 'denied'}")
+            assert result, "Plain root should be allowed with Disallow: /*?"
+
+            result = await parser.can_fetch(f"{query_base}/article", "bot")
+            print(f"Plain page (/article): {'allowed' if result else 'denied'}")
+            assert result, "Plain page should be allowed with Disallow: /*?"
+
+            result = await parser.can_fetch(f"{query_base}/?page=2", "bot")
+            print(f"Query root (/?page=2): {'allowed' if result else 'denied'}")
+            assert not result, "Query root should be denied with Disallow: /*?"
+
+            result = await parser.can_fetch(f"{query_base}/article?ref=x", "bot")
+            print(f"Query page (/article?ref=x): {'allowed' if result else 'denied'}")
+            assert not result, "Query page should be denied with Disallow: /*?"
+        finally:
+            await query_runner.cleanup()
+
         # 5. Cache manipulation
         print("\n5. Testing cache manipulation...")
         
diff --git a/tests/unit/test_robots_query_rules.py b/tests/unit/test_robots_query_rules.py
new file mode 100644
index 000000000..90b1c61bf
--- /dev/null
+++ b/tests/unit/test_robots_query_rules.py
@@ -0,0 +1,92 @@
+"""Unit tests for robots.txt rules ending in a bare '?' (e.g. 'Disallow: /*?').
+
+urllib drops the trailing '?', which the wildcard patch in crawl4ai.utils turns
+into the regex '^/.*' - disallowing the whole site. _preserve_bare_query rewrites
+the rule to the equivalent '/*?*', which survives the round trip.
+"""
+
+import asyncio
+
+import pytest
+
+from crawl4ai.utils import RobotsParser, _preserve_bare_query
+
+QUERY_RULES = "User-agent: *\nDisallow: /*?\n"
+
+
+@pytest.mark.parametrize(
+    "line, expected",
+    [
+        # A bare trailing '?' gains an explicit '*'
+        ("Disallow: /*?", "Disallow: /*?*"),
+        ("Allow: /*?", "Allow: /*?*"),
+        ("Disallow: /search?", "Disallow: /search?*"),
+        # Case and spacing are normalised, not required
+        ("disallow: /*?", "disallow: /*?*"),
+        ("DISALLOW:/*?", "DISALLOW: /*?*"),
+        ("Disallow:   /*?   ", "Disallow: /*?*"),
+        # Already explicit, or no trailing '?': left alone
+        ("Disallow: /*?*", "Disallow: /*?*"),
+        ("Disallow: /private/", "Disallow: /private/"),
+        ("Allow: /public/", "Allow: /public/"),
+        # Non-rule directives are never rewritten, even ending in '?'
+        ("User-agent: *", "User-agent: *"),
+        ("Sitemap: https://example.com/sitemap.xml?", "Sitemap: https://example.com/sitemap.xml?"),
+        ("", ""),
+        ("# just a comment", "# just a comment"),
+    ],
+)
+def test_preserve_bare_query_line_rewriting(line, expected):
+    assert _preserve_bare_query(line) == expected
+
+
+def test_preserve_bare_query_keeps_document_structure():
+    """Untouched lines, blank lines and ordering survive verbatim.
+
+    The rewrite is splitlines()-based, so a trailing newline is not preserved.
+    That is harmless: the only caller re-splits the result immediately.
+    """
+    source = "User-agent: *\nDisallow: /private/\n\nDisallow: /*?\nAllow: /public/\n"
+    assert _preserve_bare_query(source) == (
+        "User-agent: *\nDisallow: /private/\n\nDisallow: /*?*\nAllow: /public/"
+    )
+    # What the caller actually consumes is unaffected by the missing newline.
+    assert _preserve_bare_query(source).splitlines() == [
+        "User-agent: *", "Disallow: /private/", "", "Disallow: /*?*", "Allow: /public/",
+    ]
+
+
+def test_preserve_bare_query_is_idempotent():
+    once = _preserve_bare_query(QUERY_RULES)
+    assert _preserve_bare_query(once) == once
+
+
+def _can_fetch(rules, path, tmp_path):
+    """Answer can_fetch for a host whose rules are pre-seeded in the cache.
+
+    Nothing listens on the host, and can_fetch falls back to 'allowed' whenever a
+    fetch fails, so any denial below can only have come from the cached rules.
+    """
+    host = "localhost:8098"
+    parser = RobotsParser(cache_dir=str(tmp_path))
+    parser._cache_rules(host, rules)
+    assert parser._get_cached_rules(host)[1], "seeded rules should be fresh"
+    return asyncio.run(parser.can_fetch(f"http://{host}{path}", "bot"))
+
+
+@pytest.mark.parametrize("path", ["/", "/article", "/a/b/c"])
+def test_query_disallow_keeps_plain_urls_crawlable(path, tmp_path):
+    """'Disallow: /*?' must not take the whole site down."""
+    assert _can_fetch(QUERY_RULES, path, tmp_path) is True
+
+
+@pytest.mark.parametrize("path", ["/?page=2", "/article?ref=x", "/a/b?x=1&y=2"])
+def test_query_disallow_denies_query_urls(path, tmp_path):
+    assert _can_fetch(QUERY_RULES, path, tmp_path) is False
+
+
+def test_ordinary_rules_still_apply(tmp_path):
+    """The rewrite must not disturb rules that never had a trailing '?'."""
+    rules = "User-agent: *\nDisallow: /private/\nAllow: /public/\n"
+    assert _can_fetch(rules, "/public/page", tmp_path) is True
+    assert _can_fetch(rules, "/private/secret", tmp_path) is False

From 07873f1e226fd726ad6cb1f98d6a2b589e326597 Mon Sep 17 00:00:00 2001
From: ntohidi <nasrin@kidocode.com>
Date: Mon, 7 Sep 2026 11:56:27 +0200
Subject: [PATCH 06/26] fix(security): route the seeder and robots.txt fetches
 through the egress proxy

Two coordinated-disclosure reports, one root cause: a library HTTP client that
fetches a caller-influenced URL without going through the egress broker. The
browser, webhook and PDF paths were covered; these two channels were missed,
and both are reachable from an untrusted Docker API body (check_robots_txt and
link_preview_config are on the untrusted-config allowlist).

- AsyncUrlSeeder built a plain httpx client. Nine request sites, three of them
  following redirects blind, and the fetched page's parsed <head> came back to
  the caller in result.links[*].head_data - so this one exfiltrates, it is not
  blind SSRF.
- RobotsParser.can_fetch fetched robots.txt on a bare aiohttp session with
  ssl=False and redirects auto-followed, and cached the body for 7 days.

Both now take proxy=proxy_url() from the new crawl4ai/egress_policy module,
which deploy/docker/server.py points at the PinningProxy it already starts for
Chromium. A proxy rather than a set_url_validator hook because a validator
resolves the name, checks it, then throws the address away and lets the client
re-resolve at connect time - the rebinding window egress_broker.py exists to
close. The proxy dials the address it validated and re-checks every redirect
hop, since each hop is a fresh proxied request.

Unset (plain-library use) means proxy=None, the default on both httpx and
aiohttp: no behaviour change, and HTTP/2 survives because the kwarg is proxy=
and not transport=.

Also here:
- robots.txt no longer disables TLS verification. The except below fails open,
  so a bad certificate now means robots is ignored rather than respected.
- LinkPreviewConfig gets untrusted clamps (max_links, concurrency, timeout).
  Each link is a separate outbound fetch and all three were unbounded.
- egress_proxy sends Connection: close on the direct branch too, so a
  keep-alive client cannot put an absolute-form request line on the spliced
  origin socket. Same host either way, so this is correctness, not SSRF.
---
 crawl4ai/async_configs.py                     |  15 ++
 crawl4ai/async_url_seeder.py                  |   9 +-
 crawl4ai/egress_policy.py                     |  35 +++
 crawl4ai/utils.py                             |  13 +-
 deploy/docker/egress_proxy.py                 |  18 +-
 deploy/docker/server.py                       |  17 +-
 .../docker/tests/test_security_ssrf_seeder.py | 233 ++++++++++++++++++
 tests/unit/test_egress_policy.py              | 159 ++++++++++++
 8 files changed, 486 insertions(+), 13 deletions(-)
 create mode 100644 crawl4ai/egress_policy.py
 create mode 100644 deploy/docker/tests/test_security_ssrf_seeder.py
 create mode 100644 tests/unit/test_egress_policy.py

diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py
index 383079fef..a6691e657 100644
--- a/crawl4ai/async_configs.py
+++ b/crawl4ai/async_configs.py
@@ -298,6 +298,9 @@ class UntrustedConfigError(ValueError):
 _MAX_VIEWPORT = 4000
 _MAX_PDF_BYTES = 100 * 1024 * 1024
 _MAX_PDF_PAGES = 2000
+_MAX_LINK_PREVIEW_LINKS = 100      # the class default
+_MAX_LINK_PREVIEW_CONCURRENCY = 10  # the class default
+_MAX_LINK_PREVIEW_TIMEOUT_S = 10    # seconds here, not ms
 
 
 def _filter_untrusted_fields(type_name: str, params: dict) -> dict:
@@ -345,6 +348,18 @@ def _cap_timeout(v):
         # Rasterizing every page is the most expensive thing this strategy can
         # do, and nothing about untrusted crawling needs it.
         params["extract_images"] = False
+    elif type_name == "LinkPreviewConfig":
+        # Also no field allowlist, and each link is a separate outbound fetch:
+        # unclamped, one request body can order millions of them at any
+        # concurrency it likes.
+        for f, cap in (
+            ("max_links", _MAX_LINK_PREVIEW_LINKS),
+            ("concurrency", _MAX_LINK_PREVIEW_CONCURRENCY),
+            ("timeout", _MAX_LINK_PREVIEW_TIMEOUT_S),
+        ):
+            if f in params:
+                v = params[f]
+                params[f] = cap if not isinstance(v, int) or v <= 0 else min(v, cap)
     return params
 
 
diff --git a/crawl4ai/async_url_seeder.py b/crawl4ai/async_url_seeder.py
index 22fa8f630..7f9fedfb4 100644
--- a/crawl4ai/async_url_seeder.py
+++ b/crawl4ai/async_url_seeder.py
@@ -53,6 +53,7 @@
 # You might need to adjust this import based on your exact file structure
 # Import AsyncLogger for default if needed
 from .async_logger import AsyncLoggerBase, AsyncLogger
+from .egress_policy import proxy_url
 
 # Import SeedingConfig for type hints
 from typing import TYPE_CHECKING
@@ -303,7 +304,11 @@ def __init__(
     ):
         self.ttl = ttl
         self._owns_client = client is None  # Track if we created the client
-        self.client = client or httpx.AsyncClient(http2=True, timeout=20, headers={
+        # proxy=None unless an embedder installed one (the Docker server does).
+        # Every request in this class goes through self.client, so this single
+        # kwarg puts the seeder's whole fan-out -- sitemaps, robots, link heads,
+        # and each redirect hop -- behind the egress policy.
+        self.client = client or httpx.AsyncClient(http2=True, timeout=20, proxy=proxy_url(), headers={
             "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) +AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36"
         })
         self.logger = logger  # Store the logger instance
@@ -1772,7 +1777,7 @@ async def _latest_index(self) -> str:
         self._log("info", "Fetching latest Common Crawl index from {url}",
                   params={"url": COLLINFO_URL}, tag="URL_SEED")
         try:
-            async with httpx.AsyncClient() as c:
+            async with httpx.AsyncClient(proxy=proxy_url()) as c:
                 j = await c.get(COLLINFO_URL, timeout=10)
                 j.raise_for_status()  # Raise an exception for bad status codes
                 idx = j.json()[0]["id"]
diff --git a/crawl4ai/egress_policy.py b/crawl4ai/egress_policy.py
new file mode 100644
index 000000000..c9b47cbfd
--- /dev/null
+++ b/crawl4ai/egress_policy.py
@@ -0,0 +1,35 @@
+"""Process-wide egress proxy for the library's own HTTP clients.
+
+The library deliberately has no egress policy of its own: as a plain library
+the caller already chooses the URL, so there is nothing to defend against.
+It matters when the URL comes from an untrusted API client, which is the
+Docker server's situation -- deploy/docker/server.py starts a pinning forward
+proxy (egress_proxy.PinningProxy) and registers it here at boot, so the
+seeder and robots.txt fetches go through the same resolve-and-pin rule the
+browser path already gets.
+
+A proxy rather than a validator hook (the shape used by
+crawl4ai/processors/pdf) because a validator resolves the name, checks it and
+then throws the address away -- the client re-resolves when it dials, which is
+the DNS-rebinding window egress_broker.py exists to close. The proxy resolves,
+pins and dials the same address, and re-validates every redirect hop because
+each hop is a fresh proxied request.
+
+Unset (the plain-library case) means proxy_url() is None, which is the default
+value of the `proxy=` kwarg on both httpx and aiohttp: no behaviour change.
+"""
+
+from typing import Optional
+
+_proxy_url: Optional[str] = None
+
+
+def set_egress_proxy(url: Optional[str]) -> None:
+    """Route the library's own HTTP clients through `url` (None = direct)."""
+    global _proxy_url
+    _proxy_url = url
+
+
+def proxy_url() -> Optional[str]:
+    """The installed egress proxy, or None for a direct connection."""
+    return _proxy_url
diff --git a/crawl4ai/utils.py b/crawl4ai/utils.py
index 279c27708..d78b3c35e 100644
--- a/crawl4ai/utils.py
+++ b/crawl4ai/utils.py
@@ -36,6 +36,7 @@
 
 from packaging import version
 from . import __version__
+from .egress_policy import proxy_url
 from typing import Sequence
 
 from itertools import chain
@@ -340,8 +341,16 @@ async def can_fetch(self, url: str, user_agent: str = "*") -> bool:
                 scheme = parsed.scheme or 'http'
                 robots_url = f"{scheme}://{domain}/robots.txt"
                 
-                async with aiohttp.ClientSession() as session:
-                    async with session.get(robots_url, timeout=2, ssl=False) as response:
+                # proxy=None unless an embedder installed one (the Docker
+                # server does). Without it this fetch is a caller-chosen URL on
+                # an unguarded client: it reached internal hosts directly and
+                # followed redirects into them.
+                # ssl=False is gone: a bad certificate now raises, and the
+                # except below fails open, so robots is ignored rather than
+                # respected -- more crawling, never a broken crawl.
+                timeout = aiohttp.ClientTimeout(total=2)
+                async with aiohttp.ClientSession(timeout=timeout) as session:
+                    async with session.get(robots_url, proxy=proxy_url()) as response:
                         if response.status == 200:
                             rules = await response.text()
                             self._cache_rules(domain, rules)
diff --git a/deploy/docker/egress_proxy.py b/deploy/docker/egress_proxy.py
index 807bb80a1..7a1dfdaeb 100644
--- a/deploy/docker/egress_proxy.py
+++ b/deploy/docker/egress_proxy.py
@@ -103,6 +103,14 @@ def _bracket(ip: str) -> str:
     return f"[{ip}]" if ":" in ip else ip
 
 
+def _drop_connection_header(headers: bytes) -> bytes:
+    """Strip any Connection: header so the caller can set its own."""
+    return b"".join(
+        ln + b"\r\n" for ln in headers.split(b"\r\n")
+        if ln and not ln.lower().startswith(b"connection:")
+    )
+
+
 class PinningProxy:
     """Async HTTP forward-proxy that connects only to pinned, global IPs."""
 
@@ -219,6 +227,11 @@ async def _handle_absolute(self, method, target, request_line, client_reader, cl
         # upstream proxy (which then needs no DNS lookup of its own).
         if upstream is None:
             out = f"{method} {path} HTTP/1.1\r\n".encode("latin-1")
+            # One validated request per origin connection. After this the bytes
+            # are spliced raw, and a keep-alive client would put its next
+            # request on the wire in absolute form (it still thinks it is
+            # talking to a proxy), which some origins answer with 400.
+            headers = _drop_connection_header(headers) + b"Connection: close\r\n"
         else:
             out = f"{method} http://{_bracket(pin.ip)}:{port}{path} HTTP/1.1\r\n".encode("latin-1")
             if upstream[2]:
@@ -226,10 +239,7 @@ async def _handle_absolute(self, method, target, request_line, client_reader, cl
             # One validated request per upstream connection: only this first
             # request is pinned/rewritten, so force close to keep a reused
             # client connection from smuggling unvalidated requests upstream.
-            headers = b"".join(
-                ln + b"\r\n" for ln in headers.split(b"\r\n")
-                if ln and not ln.lower().startswith(b"connection:")
-            ) + b"Connection: close\r\n"
+            headers = _drop_connection_header(headers) + b"Connection: close\r\n"
         out += b"Host: " + sp.hostname.encode("latin-1")
         if sp.port:
             out += f":{sp.port}".encode("latin-1")
diff --git a/deploy/docker/server.py b/deploy/docker/server.py
index 3636efb93..c232a3e5e 100644
--- a/deploy/docker/server.py
+++ b/deploy/docker/server.py
@@ -211,11 +211,18 @@ async def lifespan(_: FastAPI):
     from egress_proxy import PinningProxy
     from egress_broker import set_egress_proxy
     app.state.egress_proxy = PinningProxy()
-    set_egress_proxy(await app.state.egress_proxy.start())
-
-    # The pinning proxy only covers Chromium. PDFContentScrapingStrategy fetches
-    # with requests on its own, so hand the library the same destination policy
-    # or that path stays an unguarded SSRF hole.
+    _proxy_url = await app.state.egress_proxy.start()
+    set_egress_proxy(_proxy_url)
+
+    # Chromium is only one of the clients that fetch a caller-chosen URL. The
+    # library's own HTTP clients -- the URL seeder (link previews, sitemaps) and
+    # RobotsParser -- take a proxy kwarg, so point them at the same pinning
+    # proxy or those paths stay unguarded SSRF holes.
+    from crawl4ai.egress_policy import set_egress_proxy as set_library_egress_proxy
+    set_library_egress_proxy(_proxy_url)
+
+    # PDFContentScrapingStrategy fetches with requests, which has no proxy hook
+    # we can rely on here, so it gets the same destination policy by injection.
     _install_pdf_egress_policy()
 
     # Bounded background-job queue (per-principal quotas optional).
diff --git a/deploy/docker/tests/test_security_ssrf_seeder.py b/deploy/docker/tests/test_security_ssrf_seeder.py
new file mode 100644
index 000000000..b0a83eff1
--- /dev/null
+++ b/deploy/docker/tests/test_security_ssrf_seeder.py
@@ -0,0 +1,233 @@
+"""SSRF on the library's own HTTP clients (2026-09 reports), end to end.
+
+Two channels fetched a caller-influenced URL without going through the egress
+broker: AsyncUrlSeeder (link previews, sitemaps) and RobotsParser. The seeder
+one was not blind -- the internal page's parsed <head> came back to the API
+caller in result.links[*].head_data.
+
+These run the real PinningProxy over real loopback sockets with the real
+resolve_and_pin rule, so 127.0.0.1 and 169.254.169.254 are refused because they
+genuinely are not global. Only "public.example" is stubbed, to stand in for a
+global host without touching the network.
+"""
+
+import asyncio
+import sqlite3
+
+import pytest
+from aiohttp import web
+
+import egress_proxy
+from egress_broker import PinnedTarget, resolve_and_pin
+from egress_proxy import PinningProxy
+
+from crawl4ai import egress_policy
+from crawl4ai.async_url_seeder import AsyncUrlSeeder
+from crawl4ai.utils import RobotsParser
+
+pytestmark = pytest.mark.posture
+
+INTERNAL_PAGE = (
+    '<html><head><title>INTERNAL-MARKER</title>'
+    '<meta name="secret" content="internal-only"></head><body>x</body></html>'
+)
+
+_PROXY_ENV = (
+    "CRAWL4AI_UPSTREAM_PROXY", "HTTP_PROXY", "http_proxy",
+    "HTTPS_PROXY", "https_proxy", "NO_PROXY", "no_proxy",
+)
+
+
+@pytest.fixture(autouse=True)
+def _clear_proxy_env(monkeypatch):
+    for name in _PROXY_ENV:
+        monkeypatch.delenv(name, raising=False)
+
+
+@pytest.fixture(autouse=True)
+def _reset_library_proxy():
+    yield
+    egress_policy.set_egress_proxy(None)
+
+
+async def _serve(handler):
+    """Start a loopback HTTP server; return (runner, port)."""
+    app = web.Application()
+    app.router.add_get("/{tail:.*}", handler)
+    runner = web.AppRunner(app)
+    await runner.setup()
+    site = web.TCPSite(runner, "127.0.0.1", 0)
+    await site.start()
+    return runner, site._server.sockets[0].getsockname()[1]
+
+
+async def _start_proxy():
+    proxy = PinningProxy()
+    egress_policy.set_egress_proxy(await proxy.start())
+    return proxy
+
+
+def _treat_port_as_public(monkeypatch, port):
+    """Let one loopback port through the proxy as if it were a global host.
+
+    Everything else keeps the real not-is_global rule, so the "internal"
+    service on its own port is still refused. Using a loopback port rather
+    than a made-up hostname matters: the unfixed code must be able to reach
+    the first hop, or the test would pass for the wrong reason.
+    """
+    def fake_pin(url):
+        if f":{port}" in url:
+            return PinnedTarget("http", "127.0.0.1", port, "127.0.0.1")
+        return resolve_and_pin(url)
+    monkeypatch.setattr(egress_proxy, "resolve_and_pin", fake_pin)
+
+
+@pytest.mark.asyncio
+class TestSeederEgress:
+    async def test_seeder_head_resolution_to_loopback_is_blocked(self):
+        """_resolve_head is a separate sink from the head fetch."""
+        hits = []
+
+        async def internal(req):
+            hits.append(req.path)
+            return web.Response(text=INTERNAL_PAGE, content_type="text/html")
+
+        runner, port = await _serve(internal)
+        proxy = await _start_proxy()
+        seeder = AsyncUrlSeeder()
+        try:
+            assert await seeder._resolve_head(f"http://127.0.0.1:{port}/") is None
+            assert hits == []
+        finally:
+            await seeder.client.aclose()
+            await proxy.stop()
+            await runner.cleanup()
+
+    async def test_seeder_direct_loopback_is_blocked(self):
+        hits = []
+
+        async def internal(req):
+            hits.append(req.path)
+            return web.Response(text=INTERNAL_PAGE, content_type="text/html")
+
+        runner, port = await _serve(internal)
+        proxy = await _start_proxy()
+        seeder = AsyncUrlSeeder()
+        try:
+            res = await seeder.extract_head_for_urls(
+                [f"http://127.0.0.1:{port}/secret"]
+            )
+            assert hits == [], "the internal service was reached"
+            assert not (res[0].get("head_data") or {}).get("title")
+        finally:
+            await seeder.client.aclose()
+            await proxy.stop()
+            await runner.cleanup()
+
+    async def test_seeder_redirect_into_internal_is_blocked(self, monkeypatch):
+        """A public first hop must not become a free pass for the second."""
+        hits = []
+
+        async def internal(req):
+            hits.append(req.path)
+            return web.Response(text=INTERNAL_PAGE, content_type="text/html")
+
+        irunner, iport = await _serve(internal)
+
+        async def public(req):
+            raise web.HTTPFound(f"http://127.0.0.1:{iport}/secret")
+
+        prunner, pport = await _serve(public)
+        _treat_port_as_public(monkeypatch, pport)
+
+        proxy = await _start_proxy()
+        seeder = AsyncUrlSeeder()
+        try:
+            ok, html, final = await seeder._fetch_head(
+                f"http://127.0.0.1:{pport}/page", timeout=5
+            )
+            assert hits == [], "the redirect reached the internal service"
+            assert "INTERNAL-MARKER" not in html, "the internal page came back"
+        finally:
+            await seeder.client.aclose()
+            await proxy.stop()
+            await prunner.cleanup()
+            await irunner.cleanup()
+
+    async def test_seeder_public_host_still_works(self, monkeypatch):
+        """The fix must not simply block everything."""
+        async def public(req):
+            return web.Response(
+                text='<html><head><title>PUBLIC-OK</title></head><body>x</body></html>',
+                content_type="text/html",
+            )
+
+        runner, port = await _serve(public)
+        _treat_port_as_public(monkeypatch, port)
+
+        proxy = await _start_proxy()
+        seeder = AsyncUrlSeeder()
+        try:
+            res = await seeder.extract_head_for_urls(
+                [f"http://127.0.0.1:{port}/page"]
+            )
+            assert (res[0].get("head_data") or {}).get("title") == "PUBLIC-OK"
+        finally:
+            await seeder.client.aclose()
+            await proxy.stop()
+            await runner.cleanup()
+
+
+@pytest.mark.asyncio
+class TestRobotsEgress:
+    async def test_robots_fetch_to_loopback_is_blocked(self, tmp_path):
+        hits = []
+
+        async def internal(req):
+            hits.append(req.path)
+            return web.Response(text="User-agent: *\nDisallow: /secret\n")
+
+        runner, port = await _serve(internal)
+        proxy = await _start_proxy()
+        parser = RobotsParser(cache_dir=str(tmp_path))
+        try:
+            # Unchanged contract: every failure on this path fails open.
+            assert await parser.can_fetch(
+                f"http://127.0.0.1:{port}/secret", "bot"
+            ) is True
+            assert hits == [], "the internal service was reached"
+            with sqlite3.connect(parser.db_path) as conn:
+                rows = conn.execute("SELECT domain FROM robots_cache").fetchall()
+            # The load-bearing assertion: nothing internal was fetched, so
+            # nothing internal got persisted for the 7-day cache TTL.
+            assert rows == []
+        finally:
+            await proxy.stop()
+            await runner.cleanup()
+
+    async def test_robots_fetch_follows_redirect_into_internal_no_further(
+        self, monkeypatch, tmp_path
+    ):
+        hits = []
+
+        async def internal(req):
+            hits.append(req.path)
+            return web.Response(text="User-agent: *\nDisallow: /secret\n")
+
+        irunner, iport = await _serve(internal)
+
+        async def public(req):
+            raise web.HTTPFound(f"http://127.0.0.1:{iport}/robots.txt")
+
+        prunner, pport = await _serve(public)
+        _treat_port_as_public(monkeypatch, pport)
+
+        proxy = await _start_proxy()
+        parser = RobotsParser(cache_dir=str(tmp_path))
+        try:
+            await parser.can_fetch(f"http://127.0.0.1:{pport}/secret", "bot")
+            assert hits == [], "robots fetch followed the redirect into loopback"
+        finally:
+            await proxy.stop()
+            await prunner.cleanup()
+            await irunner.cleanup()
diff --git a/tests/unit/test_egress_policy.py b/tests/unit/test_egress_policy.py
new file mode 100644
index 000000000..91202fe4b
--- /dev/null
+++ b/tests/unit/test_egress_policy.py
@@ -0,0 +1,159 @@
+"""The library's own HTTP clients honour the installed egress proxy.
+
+Two SSRF reports (2026-09) came from the same root cause: AsyncUrlSeeder and
+RobotsParser build HTTP clients that never go through the Docker server's
+egress broker. The fix is one `proxy=` kwarg per client, fed from
+crawl4ai.egress_policy. These tests pin both halves of that: the proxy is used
+when installed, and nothing changes for plain-library users when it is not.
+
+No network and no server: everything here is client construction.
+"""
+
+import asyncio
+
+import aiohttp
+import pytest
+
+from crawl4ai import egress_policy
+from crawl4ai.async_url_seeder import AsyncUrlSeeder
+from crawl4ai.utils import RobotsParser
+
+PROXY = "http://127.0.0.1:9"  # discard port; nothing here connects
+
+
+@pytest.fixture(autouse=True)
+def _reset_proxy():
+    """The proxy URL is a module-level global; leaking it breaks later tests."""
+    yield
+    egress_policy.set_egress_proxy(None)
+
+
+def _proxy_transport(client):
+    """The transport httpx would use for an outbound URL, or None if direct."""
+    import httpx
+
+    transport = client._transport_for_url(httpx.URL("http://example.com/"))
+    return transport if transport is not client._transport else None
+
+
+# ── seeder ─────────────────────────────────────────────────────────────
+
+
+def test_seeder_client_has_no_proxy_by_default():
+    """Plain-library use must behave exactly as it did before the fix."""
+    seeder = AsyncUrlSeeder()
+    try:
+        assert seeder.client._mounts == {}
+        assert _proxy_transport(seeder.client) is None
+    finally:
+        asyncio.run(seeder.client.aclose())
+
+
+def test_seeder_client_uses_installed_proxy():
+    egress_policy.set_egress_proxy(PROXY)
+    seeder = AsyncUrlSeeder()
+    try:
+        assert _proxy_transport(seeder.client) is not None
+    finally:
+        asyncio.run(seeder.client.aclose())
+
+
+def test_seeder_keeps_http2_through_proxy():
+    """The proxy must not cost the seeder HTTP/2.
+
+    Passing `transport=` to httpx.AsyncClient silently drops http2=True; the
+    `proxy=` kwarg does not, and httpcore negotiates h2 inside the CONNECT
+    tunnel. This test is what stops that regression sneaking back in.
+    """
+    egress_policy.set_egress_proxy(PROXY)
+    seeder = AsyncUrlSeeder()
+    try:
+        transport = _proxy_transport(seeder.client)
+        assert transport._pool._http2 is True
+    finally:
+        asyncio.run(seeder.client.aclose())
+
+
+def test_collinfo_client_uses_installed_proxy():
+    """The throwaway client in _latest_index is a second construction site."""
+    import inspect
+
+    src = inspect.getsource(AsyncUrlSeeder._latest_index)
+    assert "httpx.AsyncClient(proxy=proxy_url())" in src
+
+
+# ── robots.txt ─────────────────────────────────────────────────────────
+
+
+class _FakeResponse:
+    status = 404
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, *exc):
+        return False
+
+
+def _record_robots_get(monkeypatch):
+    """Capture the kwargs RobotsParser hands aiohttp, without any network."""
+    calls = []
+
+    def fake_get(self, url, **kwargs):
+        calls.append((url, kwargs))
+        return _FakeResponse()
+
+    monkeypatch.setattr(aiohttp.ClientSession, "get", fake_get)
+    return calls
+
+
+def test_robots_fetch_passes_proxy(monkeypatch, tmp_path):
+    calls = _record_robots_get(monkeypatch)
+    egress_policy.set_egress_proxy(PROXY)
+
+    parser = RobotsParser(cache_dir=str(tmp_path))
+    asyncio.run(parser.can_fetch("http://example.com/x", "bot"))
+
+    assert len(calls) == 1
+    url, kwargs = calls[0]
+    assert url == "http://example.com/robots.txt"
+    assert kwargs["proxy"] == PROXY
+
+
+def test_robots_fetch_is_direct_by_default(monkeypatch, tmp_path):
+    calls = _record_robots_get(monkeypatch)
+
+    parser = RobotsParser(cache_dir=str(tmp_path))
+    asyncio.run(parser.can_fetch("http://example.com/x", "bot"))
+
+    assert calls[0][1]["proxy"] is None
+
+
+def test_robots_fetch_no_longer_disables_tls_verification(monkeypatch, tmp_path):
+    """ssl=False made every robots fetch accept any certificate."""
+    calls = _record_robots_get(monkeypatch)
+
+    parser = RobotsParser(cache_dir=str(tmp_path))
+    asyncio.run(parser.can_fetch("https://example.com/x", "bot"))
+
+    assert calls[0][1].get("ssl", None) is None
+
+
+# ── untrusted LinkPreviewConfig clamps ─────────────────────────────────
+
+
+@pytest.mark.parametrize(
+    "field,asked,expected",
+    [
+        ("max_links", 10_000_000, 100),
+        ("max_links", 0, 100),
+        ("concurrency", 10_000, 10),
+        ("timeout", 3600, 10),
+        ("max_links", 5, 5),  # a reasonable value survives
+    ],
+)
+def test_link_preview_untrusted_values_are_clamped(field, asked, expected):
+    from crawl4ai.async_configs import _enforce_untrusted
+
+    out = _enforce_untrusted("LinkPreviewConfig", {field: asked})
+    assert out[field] == expected

From 6ccf610bcb44328815cf521a6c528dc807670ae8 Mon Sep 17 00:00:00 2001
From: ntohidi <nasrin@kidocode.com>
Date: Mon, 7 Sep 2026 13:04:58 +0200
Subject: [PATCH 07/26] fix(docker): recycle pooled browser contexts by pages
 served (#2231)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Pooled browsers get slower with sustained use and the janitor never
recycles them, because it only closes browsers that have been *idle*
past a TTL — and a server under continuous load never has an idle one.

Measured where the slowdown actually lives: after 1000 real page loads
on one pooled browser, a fresh context inside the SAME chromium process
navigated as fast as a brand-new process (7.1ms vs 7.0ms), while the
worn context took 32.0ms. Wiping that context's cookies/localStorage/
service workers in place recovered half of it (16.3ms). A control run
with pages that leave no state behind stayed flat over 2000 pages.

So the rot is context state, not the browser process, and the mechanism
to bound it already exists in browser_manager (version-based recycling,
which replaces the context under load without waiting for a quiet
moment). It was simply never enabled for the Docker server.

Enabling it is one config line — no second lifetime policy in the pool's
janitor, which would put the same rule in two places. Measured effect at
max_pages_before_recycle=200: penalty drops from ~30ms to ~10-14ms. It
bounds the damage to one recycle window rather than removing it, so the
threshold is the knob.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019EQGr8ey8riKFr4inp9B8f
---
 deploy/docker/config.yml                 |  8 +++++
 tests/docker/test_pool_recycle_config.py | 42 ++++++++++++++++++++++++
 2 files changed, 50 insertions(+)
 create mode 100644 tests/docker/test_pool_recycle_config.py

diff --git a/deploy/docker/config.yml b/deploy/docker/config.yml
index 7aabc6814..6040ed1ca 100644
--- a/deploy/docker/config.yml
+++ b/deploy/docker/config.yml
@@ -88,6 +88,14 @@ crawler:
     kwargs:
       headless: true
       text_mode: true
+      # Pooled browsers are long-lived, and their browser context is reused for
+      # every crawl that shares a config. Real sites leave state in that context
+      # (cookies, localStorage, service workers) which is never cleared, and
+      # navigation gets measurably slower as it builds up. Recycling by pages
+      # served bounds that: the context is replaced inside the same Chromium
+      # process, old contexts drain on their own, and it works under sustained
+      # load (unlike the janitor, which only closes *idle* browsers). See #2231.
+      max_pages_before_recycle: 200
     extra_args:
       # - "--single-process"
       # SECURITY: --no-sandbox disables the Chromium renderer sandbox (a renderer
diff --git a/tests/docker/test_pool_recycle_config.py b/tests/docker/test_pool_recycle_config.py
new file mode 100644
index 000000000..c51a81b0c
--- /dev/null
+++ b/tests/docker/test_pool_recycle_config.py
@@ -0,0 +1,42 @@
+"""The Docker pool must recycle its browser context by pages served (#2231).
+
+The pool's janitor only closes *idle* browsers, so a server under sustained
+load never recycles one. Recycling by page count is what bounds the context
+state (cookies/localStorage/service workers) that slows navigation down.
+"""
+import sys
+from pathlib import Path
+
+import pytest
+import yaml
+
+from crawl4ai import BrowserConfig
+
+CONFIG_PATH = Path(__file__).resolve().parents[2] / "deploy" / "docker" / "config.yml"
+
+
+@pytest.fixture(scope="module")
+def browser_section():
+    return yaml.safe_load(CONFIG_PATH.read_text())["crawler"]["browser"]
+
+
+def test_recycle_is_enabled(browser_section):
+    kwargs = browser_section.get("kwargs", {})
+    assert kwargs.get("max_pages_before_recycle", 0) > 0, (
+        "deploy/docker/config.yml must set crawler.browser.kwargs."
+        "max_pages_before_recycle > 0, otherwise pooled browser contexts are "
+        "never recycled under sustained load (issue #2231)."
+    )
+
+
+def test_kwargs_reach_browser_config(browser_section):
+    """server.py/api.py splat these kwargs straight into BrowserConfig."""
+    cfg = BrowserConfig(
+        extra_args=browser_section.get("extra_args", []),
+        **browser_section.get("kwargs", {}),
+    )
+    assert cfg.max_pages_before_recycle > 0
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main([__file__, "-v"]))

From 47ea8d7a817e1062d6a9ca4bab1d2b4111c3a5ed Mon Sep 17 00:00:00 2001
From: ntohidi <nasrin@kidocode.com>
Date: Mon, 7 Sep 2026 13:14:13 +0200
Subject: [PATCH 08/26] fix(docker): apply the recycle policy in the pool, not
 in config.yml kwargs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The previous commit put max_pages_before_recycle in config.yml's
crawler.browser.kwargs, which does not reach the endpoints in the report:
/crawl and /crawl/stream build their BrowserConfig from the request body
(api.py:687, api.py:903) and nothing merges the server's browser kwargs
into it. Only the handful of endpoints that call
get_default_browser_config() would have picked it up.

Move the setting to crawler.pool (it is a pool policy, next to max_pages
and idle_ttl_sec) and apply it in get_crawler()/init_permanent(), which
every endpoint goes through. Pooling is what makes a browser long-lived,
so the pool is the right owner of its recycling policy.

Applied before _sig() so all requests still share one signature and
pooling is unchanged; an explicit per-request value wins.

Tests cover the call site too, not only the helper — dropping
_apply_pool_defaults() from get_crawler() now fails.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019EQGr8ey8riKFr4inp9B8f
---
 deploy/docker/config.yml                 | 16 ++---
 deploy/docker/crawler_pool.py            | 21 ++++++-
 tests/docker/test_pool_recycle_config.py | 74 ++++++++++++++++++------
 3 files changed, 82 insertions(+), 29 deletions(-)

diff --git a/deploy/docker/config.yml b/deploy/docker/config.yml
index 6040ed1ca..094209899 100644
--- a/deploy/docker/config.yml
+++ b/deploy/docker/config.yml
@@ -84,18 +84,18 @@ crawler:
   pool:
     max_pages: 40                          # ← GLOBAL_SEM permits
     idle_ttl_sec: 300                     # ← 30 min janitor cutoff
+    # Pooled browsers are long-lived and reuse one browser context per config.
+    # Real sites leave state in that context (cookies, localStorage, service
+    # workers) which is never cleared, and navigation gets measurably slower as
+    # it builds up. Recycling by pages served bounds it: the context is replaced
+    # inside the same Chromium process, old contexts drain on their own, and it
+    # works under sustained load — unlike the janitor, which only closes *idle*
+    # browsers and so never fires on a busy server. 0 disables. See #2231.
+    max_pages_before_recycle: 200
   browser:
     kwargs:
       headless: true
       text_mode: true
-      # Pooled browsers are long-lived, and their browser context is reused for
-      # every crawl that shares a config. Real sites leave state in that context
-      # (cookies, localStorage, service workers) which is never cleared, and
-      # navigation gets measurably slower as it builds up. Recycling by pages
-      # served bounds that: the context is replaced inside the same Chromium
-      # process, old contexts drain on their own, and it works under sustained
-      # load (unlike the janitor, which only closes *idle* browsers). See #2231.
-      max_pages_before_recycle: 200
     extra_args:
       # - "--single-process"
       # SECURITY: --no-sandbox disables the Chromium renderer sandbox (a renderer
diff --git a/deploy/docker/crawler_pool.py b/deploy/docker/crawler_pool.py
index 516d9562a..cdde8bba1 100644
--- a/deploy/docker/crawler_pool.py
+++ b/deploy/docker/crawler_pool.py
@@ -20,9 +20,26 @@
 # Config
 MEM_LIMIT = CONFIG.get("crawler", {}).get("memory_threshold_percent", 95.0)
 BASE_IDLE_TTL = CONFIG.get("crawler", {}).get("pool", {}).get("idle_ttl_sec", 300)
+RECYCLE_PAGES = CONFIG.get("crawler", {}).get("pool", {}).get("max_pages_before_recycle", 0)
 DEFAULT_CONFIG_SIG = None  # Cached sig for default config
 
 
+def _apply_pool_defaults(cfg: BrowserConfig) -> BrowserConfig:
+    """Apply pool-owned policy to a browser config before it is pooled.
+
+    Browsers handed out by this pool are long-lived, and the endpoints build
+    their BrowserConfig from the request body (see api.py handle_crawl_request),
+    so config.yml's browser kwargs never reach them. Pooling is what makes a
+    browser long-lived, so the pool is where its recycling policy belongs.
+
+    Applied before _sig() so every request shares one signature and pooling is
+    unaffected. An explicit per-request value wins. See #2231.
+    """
+    if RECYCLE_PAGES and not cfg.max_pages_before_recycle:
+        cfg.max_pages_before_recycle = RECYCLE_PAGES
+    return cfg
+
+
 def get_pool_snapshot() -> dict:
     """Return a point-in-time snapshot of pool state for monitoring.
 
@@ -54,7 +71,7 @@ def _is_default_config(sig: str) -> bool:
 
 async def get_crawler(cfg: BrowserConfig) -> AsyncWebCrawler:
     """Get crawler from pool with tiered strategy."""
-    sig = _sig(cfg)
+    sig = _sig(_apply_pool_defaults(cfg))
     async with LOCK:
         # Check permanent browser for default config
         if PERMANENT and _is_default_config(sig):
@@ -135,7 +152,7 @@ async def init_permanent(cfg: BrowserConfig):
     async with LOCK:
         if PERMANENT:
             return
-        DEFAULT_CONFIG_SIG = _sig(cfg)
+        DEFAULT_CONFIG_SIG = _sig(_apply_pool_defaults(cfg))
         logger.info("🔥 Creating permanent default browser")
         PERMANENT = AsyncWebCrawler(config=cfg, thread_safe=False)
         await PERMANENT.start()
diff --git a/tests/docker/test_pool_recycle_config.py b/tests/docker/test_pool_recycle_config.py
index c51a81b0c..c89f48273 100644
--- a/tests/docker/test_pool_recycle_config.py
+++ b/tests/docker/test_pool_recycle_config.py
@@ -1,8 +1,12 @@
-"""The Docker pool must recycle its browser context by pages served (#2231).
+"""Pooled browsers must recycle their context by pages served (#2231).
 
 The pool's janitor only closes *idle* browsers, so a server under sustained
 load never recycles one. Recycling by page count is what bounds the context
 state (cookies/localStorage/service workers) that slows navigation down.
+
+The policy has to be applied by the pool, not by config.yml's browser kwargs:
+/crawl and /crawl/stream build their BrowserConfig from the request body
+(api.py handle_crawl_request), so those kwargs never reach them.
 """
 import sys
 from pathlib import Path
@@ -12,30 +16,62 @@
 
 from crawl4ai import BrowserConfig
 
-CONFIG_PATH = Path(__file__).resolve().parents[2] / "deploy" / "docker" / "config.yml"
-
+ROOT = Path(__file__).resolve().parents[2]
+DOCKER_DIR = ROOT / "deploy" / "docker"
+sys.path.insert(0, str(DOCKER_DIR))
 
-@pytest.fixture(scope="module")
-def browser_section():
-    return yaml.safe_load(CONFIG_PATH.read_text())["crawler"]["browser"]
+pool = pytest.importorskip("crawler_pool", reason="docker server deps not installed")
 
 
-def test_recycle_is_enabled(browser_section):
-    kwargs = browser_section.get("kwargs", {})
-    assert kwargs.get("max_pages_before_recycle", 0) > 0, (
-        "deploy/docker/config.yml must set crawler.browser.kwargs."
-        "max_pages_before_recycle > 0, otherwise pooled browser contexts are "
-        "never recycled under sustained load (issue #2231)."
+def test_config_enables_recycling():
+    cfg = yaml.safe_load((DOCKER_DIR / "config.yml").read_text())
+    assert cfg["crawler"]["pool"].get("max_pages_before_recycle", 0) > 0, (
+        "config.yml must set crawler.pool.max_pages_before_recycle > 0, else "
+        "pooled contexts are never recycled under sustained load (#2231)."
     )
 
 
-def test_kwargs_reach_browser_config(browser_section):
-    """server.py/api.py splat these kwargs straight into BrowserConfig."""
-    cfg = BrowserConfig(
-        extra_args=browser_section.get("extra_args", []),
-        **browser_section.get("kwargs", {}),
-    )
-    assert cfg.max_pages_before_recycle > 0
+def test_pool_applies_recycling_to_a_request_supplied_config():
+    """A config off the wire carries no recycle setting; the pool must add it."""
+    from_request = BrowserConfig()
+    assert from_request.max_pages_before_recycle == 0  # guard the premise
+
+    pool._apply_pool_defaults(from_request)
+    assert from_request.max_pages_before_recycle == pool.RECYCLE_PAGES > 0
+
+
+def test_explicit_caller_value_wins():
+    cfg = BrowserConfig(max_pages_before_recycle=7)
+    pool._apply_pool_defaults(cfg)
+    assert cfg.max_pages_before_recycle == 7
+
+
+@pytest.mark.asyncio
+async def test_get_crawler_applies_it(monkeypatch):
+    """The call site matters, not just the helper: a config handed to
+    get_crawler() must come back carrying the pool's recycle policy."""
+    class _FakeCrawler:
+        def __init__(self, config, **kw):
+            self.config = config
+
+        async def start(self):
+            return self
+
+    monkeypatch.setattr(pool, "AsyncWebCrawler", _FakeCrawler)
+    monkeypatch.setattr(pool, "COLD_POOL", {})
+    monkeypatch.setattr(pool, "HOT_POOL", {})
+    monkeypatch.setattr(pool, "PERMANENT", None)
+    monkeypatch.setattr(pool, "get_container_memory_percent", lambda: 10.0)
+
+    cfg = BrowserConfig()
+    await pool.get_crawler(cfg)
+    assert cfg.max_pages_before_recycle == pool.RECYCLE_PAGES > 0
+
+
+def test_signature_is_stable_across_requests():
+    """Applying the default must not split the pool into two signatures."""
+    a, b = BrowserConfig(), BrowserConfig()
+    assert pool._sig(pool._apply_pool_defaults(a)) == pool._sig(pool._apply_pool_defaults(b))
 
 
 if __name__ == "__main__":

From 7b02a2c0d05612b7fb0dfe170ce459dd18b353a4 Mon Sep 17 00:00:00 2001
From: Soham Kukreti <kukretisoham@gmail.com>
Date: Tue, 8 Sep 2026 19:19:39 +0530
Subject: [PATCH 09/26] fix(docker): build permanent browser with
 egress-hardened default config so its pool signature matches

---
 deploy/docker/server.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/deploy/docker/server.py b/deploy/docker/server.py
index 3636efb93..dd96fd78b 100644
--- a/deploy/docker/server.py
+++ b/deploy/docker/server.py
@@ -232,10 +232,7 @@ async def lifespan(_: FastAPI):
     monitor_module.monitor_stats.start_persistence_worker()
 
     # Initialize browser pool
-    await init_permanent(BrowserConfig(
-        extra_args=_browser_extra_args(),
-        **config["crawler"]["browser"].get("kwargs", {}),
-    ))
+    await init_permanent(get_default_browser_config())
 
     # Start background tasks
     app.state.janitor = asyncio.create_task(janitor())

From 5ee5110e292bb8e1e1393e76d3335ec7695487b9 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 9 Sep 2026 01:52:15 +0000
Subject: [PATCH 10/26] fix(browser): drop OptimizationHints from
 --disable-features

Chrome-for-Testing SEGV_ACCERRs under --headless=new on macOS arm64 when
OptimizationHints is disabled. Keep MediaRouter and DialMediaRouteProvider.

Fixes #2239

Co-authored-by: Zsanz3 <Zsanz3@users.noreply.github.com>
---
 crawl4ai/browser_manager.py                 | 12 +++--
 tests/async/test_browser_memory.py          | 20 ++++---
 tests/regression/test_reg_browser.py        | 45 ++++++++++++++++
 tests/unit/test_browser_disable_features.py | 59 +++++++++++++++++++++
 4 files changed, 126 insertions(+), 10 deletions(-)
 create mode 100644 tests/unit/test_browser_disable_features.py

diff --git a/crawl4ai/browser_manager.py b/crawl4ai/browser_manager.py
index f4ab0aa32..17d6f5eb3 100644
--- a/crawl4ai/browser_manager.py
+++ b/crawl4ai/browser_manager.py
@@ -85,8 +85,10 @@ def build_browser_flags(config: BrowserConfig) -> List[str]:
             "--force-color-profile=srgb",
             "--mute-audio",
             "--disable-background-timer-throttling",
-            # Memory-saving flags: disable unused Chrome features
-            "--disable-features=OptimizationHints,MediaRouter,DialMediaRouteProvider",
+            # Memory-saving flags: disable unused Chrome features.
+            # Do not disable OptimizationHints — it SEGV_ACCERRs Chrome-for-Testing
+            # under --headless=new on macOS arm64 (issue #2239).
+            "--disable-features=MediaRouter,DialMediaRouteProvider",
             "--disable-component-update",
             "--disable-domain-reliability",
         ]
@@ -1102,8 +1104,10 @@ def _build_browser_args(self) -> dict:
             "--force-color-profile=srgb",
             "--mute-audio",
             "--disable-background-timer-throttling",
-            # Memory-saving flags: disable unused Chrome features
-            "--disable-features=OptimizationHints,MediaRouter,DialMediaRouteProvider",
+            # Memory-saving flags: disable unused Chrome features.
+            # Do not disable OptimizationHints — it SEGV_ACCERRs Chrome-for-Testing
+            # under --headless=new on macOS arm64 (issue #2239).
+            "--disable-features=MediaRouter,DialMediaRouteProvider",
             "--disable-component-update",
             "--disable-domain-reliability",
             # "--single-process",
diff --git a/tests/async/test_browser_memory.py b/tests/async/test_browser_memory.py
index cd1685d04..ee6ab58e1 100644
--- a/tests/async/test_browser_memory.py
+++ b/tests/async/test_browser_memory.py
@@ -187,8 +187,11 @@ async def test_memory_saving_flags_applied(test_server):
         assert any("max-old-space-size" in a for a in args_list), (
             "memory_saving_mode=True should add V8 heap cap"
         )
-        # Always-on flags should be present regardless
-        assert any("OptimizationHints" in a for a in args_list)
+        # Always-on flags should be present regardless.
+        # OptimizationHints must stay off the disable-list (issue #2239).
+        assert any("MediaRouter" in a for a in args_list)
+        assert any("DialMediaRouteProvider" in a for a in args_list)
+        assert not any("OptimizationHints" in a for a in args_list)
 
     async with AsyncWebCrawler(config=config_off) as crawler:
         bm = _bm(crawler)
@@ -201,7 +204,9 @@ async def test_memory_saving_flags_applied(test_server):
             "memory_saving_mode=False should NOT add V8 heap cap"
         )
         # Always-on flags should still be there
-        assert any("OptimizationHints" in a for a in args_list)
+        assert any("MediaRouter" in a for a in args_list)
+        assert any("DialMediaRouteProvider" in a for a in args_list)
+        assert not any("OptimizationHints" in a for a in args_list)
 
 
 # ===========================================================================
@@ -210,15 +215,18 @@ async def test_memory_saving_flags_applied(test_server):
 
 @pytest.mark.asyncio
 async def test_always_on_flags_present(test_server):
-    """The 3 always-on memory flags should appear in _build_browser_args
-    even with default BrowserConfig."""
+    """The always-on memory flags should appear in _build_browser_args
+    even with default BrowserConfig. OptimizationHints must not be disabled
+    (issue #2239 / Chrome-for-Testing SEGV_ACCERR under --headless=new)."""
     config = BrowserConfig(headless=True, verbose=False)
     async with AsyncWebCrawler(config=config) as crawler:
         browser_args = _bm(crawler)._build_browser_args()
         args_list = browser_args.get("args", browser_args) if isinstance(browser_args, dict) else browser_args
         assert any("disable-component-update" in a for a in args_list)
         assert any("disable-domain-reliability" in a for a in args_list)
-        assert any("OptimizationHints" in a for a in args_list)
+        assert any("MediaRouter" in a for a in args_list)
+        assert any("DialMediaRouteProvider" in a for a in args_list)
+        assert not any("OptimizationHints" in a for a in args_list)
 
 
 # ===========================================================================
diff --git a/tests/regression/test_reg_browser.py b/tests/regression/test_reg_browser.py
index dac55a841..cab2b28ed 100644
--- a/tests/regression/test_reg_browser.py
+++ b/tests/regression/test_reg_browser.py
@@ -131,6 +131,51 @@ async def test_browser_context_manager(local_server):
     # If we get here without exception, cleanup succeeded
 
 
+# ---------------------------------------------------------------------------
+# Launch flags
+# ---------------------------------------------------------------------------
+
+
+def _disable_feature_names(args):
+    """Every name listed in any --disable-features switch (not last-wins)."""
+    names = []
+    for arg in args:
+        if arg.startswith("--disable-features="):
+            names.extend(
+                name.strip()
+                for name in arg.split("=", 1)[1].split(",")
+                if name.strip()
+            )
+    return names
+
+
+def test_disable_features_omits_optimization_hints():
+    """Issue #2239: OptimizationHints in --disable-features crashes
+    Chrome-for-Testing under --headless=new on macOS arm64 (SEGV_ACCERR).
+
+    MediaRouter and DialMediaRouteProvider must remain. #2219 merges
+    repeated --disable-features lists and previously asserted
+    OptimizationHints as a default that must survive; after this fix the
+    expected defaults are MediaRouter and DialMediaRouteProvider only.
+    """
+    from crawl4ai.browser_manager import BrowserManager, ManagedBrowser
+
+    for light_mode in (False, True):
+        config = BrowserConfig(headless=True, light_mode=light_mode)
+        for flags in (
+            ManagedBrowser.build_browser_flags(config),
+            BrowserManager(browser_config=config)._build_browser_args()["args"],
+        ):
+            names = _disable_feature_names(flags)
+            assert "OptimizationHints" not in names, (
+                f"light_mode={light_mode}: OptimizationHints must not be disabled"
+            )
+            for expected in ("MediaRouter", "DialMediaRouteProvider"):
+                assert expected in names, (
+                    f"light_mode={light_mode} lost {expected}"
+                )
+
+
 # ---------------------------------------------------------------------------
 # Viewport configuration
 # ---------------------------------------------------------------------------
diff --git a/tests/unit/test_browser_disable_features.py b/tests/unit/test_browser_disable_features.py
new file mode 100644
index 000000000..1b99249f4
--- /dev/null
+++ b/tests/unit/test_browser_disable_features.py
@@ -0,0 +1,59 @@
+"""Issue #2239: OptimizationHints must not appear in built --disable-features.
+
+Chrome-for-Testing 151+ SEGV_ACCERRs under --headless=new on macOS arm64 when
+OptimizationHints is disabled. MediaRouter and DialMediaRouteProvider stay.
+
+These are flag-construction checks only — no browser or network required.
+
+#2219 merges repeated --disable-features switches and currently expects
+OptimizationHints to survive that merge. After this change the surviving
+defaults are MediaRouter and DialMediaRouteProvider only.
+"""
+
+from crawl4ai import BrowserConfig
+from crawl4ai.browser_manager import BrowserManager, ManagedBrowser
+
+
+def _disable_feature_names(args):
+    """Every name listed in any --disable-features switch (not last-wins)."""
+    names = []
+    for arg in args:
+        if arg.startswith("--disable-features="):
+            names.extend(
+                name.strip()
+                for name in arg.split("=", 1)[1].split(",")
+                if name.strip()
+            )
+    return names
+
+
+def test_build_browser_flags_omits_optimization_hints():
+    config = BrowserConfig(headless=True)
+    names = _disable_feature_names(ManagedBrowser.build_browser_flags(config))
+    assert "OptimizationHints" not in names
+    assert "MediaRouter" in names
+    assert "DialMediaRouteProvider" in names
+
+
+def test_build_browser_args_omits_optimization_hints():
+    config = BrowserConfig(headless=True)
+    args = BrowserManager(browser_config=config)._build_browser_args()["args"]
+    names = _disable_feature_names(args)
+    assert "OptimizationHints" not in names
+    assert "MediaRouter" in names
+    assert "DialMediaRouteProvider" in names
+
+
+def test_light_mode_still_omits_optimization_hints():
+    """light_mode appends BROWSER_DISABLE_OPTIONS; OptimizationHints must
+    not reappear there either. Aligns with #2219's merge expectations:
+    MediaRouter / DialMediaRouteProvider remain the crawl4ai defaults."""
+    config = BrowserConfig(headless=True, light_mode=True)
+    for flags in (
+        ManagedBrowser.build_browser_flags(config),
+        BrowserManager(browser_config=config)._build_browser_args()["args"],
+    ):
+        names = _disable_feature_names(flags)
+        assert "OptimizationHints" not in names
+        assert "MediaRouter" in names
+        assert "DialMediaRouteProvider" in names

From ce62ae674c34de5dd20f7ca0dcf3e37ed704810d Mon Sep 17 00:00:00 2001
From: ntohidi <nasrin@kidocode.com>
Date: Wed, 9 Sep 2026 13:33:33 +0200
Subject: [PATCH 11/26] fix(security): stop {"type":"dict","value":X}
 laundering typed objects past the untrusted gate

An untrusted request body could wrap a forbidden typed object in the plain-dict
envelope emitted by to_serializable_dict(). The unwrap recursed over
data["value"].items(), so the wrapped object was never seen as a whole typed
object and the UNTRUSTED_ALLOWED_TYPES gate never fired for it. from_kwargs()
then re-deserialized the surviving plain dict with the default TRUSTED
provenance, which constructs LLMConfig and resolves api_token="env:NAME"
through os.getenv - leaking OPENAI_API_KEY, and SECRET_KEY where it is set.

Two changes, defense in depth:

- from_serializable_dict(): after unwrapping {"type":"dict","value":X}, raise
  UntrustedConfigError under UNTRUSTED if the result is itself typed-object
  shaped. The shape test is factored into _is_typed_shape() and shared with the
  existing dispatch, so both places agree on what a typed object looks like.
- BrowserConfig.from_kwargs() and CrawlerRunConfig.from_kwargs() take the
  provenance and pass it down instead of silently defaulting to TRUSTED.
  load() threads it through.

TRUSTED (SDK / in-process) behavior is unchanged.

Reported by Adam Jordan (adamyordan). CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:N (8.1).
Test: tests/unit/test_config_provenance.py

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01W1ZJkE8DVxFirNUrbUQKrQ
---
 crawl4ai/async_configs.py            |  52 +++++++++---
 tests/unit/test_config_provenance.py | 118 +++++++++++++++++++++++++++
 2 files changed, 158 insertions(+), 12 deletions(-)
 create mode 100644 tests/unit/test_config_provenance.py

diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py
index a6691e657..340b3f9ba 100644
--- a/crawl4ai/async_configs.py
+++ b/crawl4ai/async_configs.py
@@ -449,6 +449,21 @@ def to_serializable_dict(obj: Any, ignore_default_value : bool = False):
     return str(obj)
 
 
+def _is_typed_shape(data: Any) -> bool:
+    """True for the dict shapes to_serializable_dict() emits:
+    {"type": "<ClassName>", "params": {...}} or {"type": "dict", "value": {...}}.
+
+    Plain business dicts that happen to carry a "type" key (JSON-Schema
+    fragments, JsonCss field specs like {"type": "text", "name": "..."}) have
+    neither "params" nor "value" and are not typed shapes.
+    """
+    return (
+        isinstance(data, dict)
+        and "type" in data
+        and ("params" in data or (data["type"] == "dict" and "value" in data))
+    )
+
+
 def from_serializable_dict(data: Any, provenance: "Provenance" = None) -> Any:
     """
     Recursively convert a serializable dictionary back to an object instance.
@@ -474,14 +489,23 @@ def from_serializable_dict(data: Any, provenance: "Provenance" = None) -> Any:
     # carry a "type" key (e.g. JSON-Schema fragments, JsonCss field specs like
     # {"type": "text", "name": "..."}) have neither "params" nor "value" and
     # must fall through to the raw-dict path below so they are passed as data.
-    if (
-        isinstance(data, dict)
-        and "type" in data
-        and ("params" in data or (data["type"] == "dict" and "value" in data))
-    ):
+    if _is_typed_shape(data):
         # Handle plain dictionaries
         if data["type"] == "dict" and "value" in data:
-            return {k: from_serializable_dict(v, provenance) for k, v in data["value"].items()}
+            unwrapped = {
+                k: from_serializable_dict(v, provenance) for k, v in data["value"].items()
+            }
+            # The unwrap above recurses over .items(), so data["value"] is never
+            # seen as a whole typed object and the type gate below never fires
+            # for it. Without this check an untrusted body launders a forbidden
+            # type past the gate by wrapping it:
+            #   {"type": "dict", "value": {"type": "LLMConfig", "params": {...}}}
+            if provenance == Provenance.UNTRUSTED and _is_typed_shape(unwrapped):
+                raise UntrustedConfigError(
+                    "an untrusted request may not wrap a typed object in "
+                    "{'type': 'dict', 'value': ...}"
+                )
+            return unwrapped
 
         # Security: only allow known-safe types to be deserialized.
         # Unknown types (e.g. logging.Logger serialized by older clients) are
@@ -988,11 +1012,13 @@ def __init__(
             )
 
     @staticmethod
-    def from_kwargs(kwargs: dict) -> "BrowserConfig":
+    def from_kwargs(kwargs: dict, provenance: "Provenance" = None) -> "BrowserConfig":
         # Auto-deserialize any dict values that use the {"type": ..., "params": ...}
         # serialization format (e.g. from JSON API requests or dump()/load() roundtrips).
         kwargs = {
-            k: from_serializable_dict(v) if isinstance(v, dict) and "type" in v else v
+            k: from_serializable_dict(v, provenance)
+            if isinstance(v, dict) and "type" in v
+            else v
             for k, v in kwargs.items()
         }
         # Only pass keys present in kwargs so that __init__ defaults (and
@@ -1078,7 +1104,7 @@ def load(data: dict, provenance: "Provenance" = None) -> "BrowserConfig":
         # so a body that sends bare kwargs (no {type,params}) cannot bypass it.
         if provenance == Provenance.UNTRUSTED and isinstance(config, dict):
             config = _enforce_untrusted("BrowserConfig", config)
-        return BrowserConfig.from_kwargs(config)
+        return BrowserConfig.from_kwargs(config, provenance)
 
     def set_nstproxy(
         self,
@@ -2119,12 +2145,14 @@ def __setattr__(self, name, value):
         super().__setattr__(name, value)
 
     @staticmethod
-    def from_kwargs(kwargs: dict) -> "CrawlerRunConfig":
+    def from_kwargs(kwargs: dict, provenance: "Provenance" = None) -> "CrawlerRunConfig":
         # Auto-deserialize any dict values that use the {"type": ..., "params": ...}
         # serialization format (e.g. from JSON API requests or dump()/load() roundtrips).
         # This covers markdown_generator, extraction_strategy, content_filter, etc.
         kwargs = {
-            k: from_serializable_dict(v) if isinstance(v, dict) and "type" in v else v
+            k: from_serializable_dict(v, provenance)
+            if isinstance(v, dict) and "type" in v
+            else v
             for k, v in kwargs.items()
         }
         # Only pass keys present in kwargs so that __init__ defaults (and
@@ -2147,7 +2175,7 @@ def load(data: dict, provenance: "Provenance" = None) -> "CrawlerRunConfig":
             return config
         if provenance == Provenance.UNTRUSTED and isinstance(config, dict):
             config = _enforce_untrusted("CrawlerRunConfig", config)
-        return CrawlerRunConfig.from_kwargs(config)
+        return CrawlerRunConfig.from_kwargs(config, provenance)
 
     def to_dict(self):
         return {
diff --git a/tests/unit/test_config_provenance.py b/tests/unit/test_config_provenance.py
new file mode 100644
index 000000000..49246a8e6
--- /dev/null
+++ b/tests/unit/test_config_provenance.py
@@ -0,0 +1,118 @@
+"""Untrusted config provenance gate.
+
+Regression test for the {"type": "dict", "value": <typed object>} laundering
+bypass: wrapping a forbidden typed object in a plain-dict envelope hid it from
+the type gate, and from_kwargs then re-deserialized it as TRUSTED, which
+resolves api_token="env:NAME" through os.getenv.
+"""
+import pytest
+
+from crawl4ai.async_configs import (
+    BrowserConfig,
+    CrawlerRunConfig,
+    Provenance,
+    UntrustedConfigError,
+)
+
+UNTRUSTED = Provenance.UNTRUSTED
+
+
+def _llm_strategy(env_var):
+    return {
+        "type": "LLMExtractionStrategy",
+        "params": {
+            "llm_config": {
+                "type": "dict",
+                "value": {"type": "LLMConfig", "params": {"api_token": f"env:{env_var}"}},
+            }
+        },
+    }
+
+
+def test_forbidden_type_refused_directly():
+    data = {"type": "CrawlerRunConfig", "params": {"extraction_strategy": _llm_strategy("SECRET_KEY")}}
+    with pytest.raises(UntrustedConfigError):
+        CrawlerRunConfig.load(data, provenance=UNTRUSTED)
+
+
+def test_forbidden_type_refused_when_wrapped(monkeypatch):
+    """The bypass: the whole strategy hidden behind {"type":"dict","value":...}."""
+    monkeypatch.setenv("SECRET_KEY", "canary-secret-key-value")
+    data = {
+        "type": "CrawlerRunConfig",
+        "extraction_strategy": {"type": "dict", "value": _llm_strategy("SECRET_KEY")},
+    }
+    with pytest.raises(UntrustedConfigError):
+        CrawlerRunConfig.load(data, provenance=UNTRUSTED)
+
+
+def test_forbidden_type_refused_when_wrapped_in_params(monkeypatch):
+    monkeypatch.setenv("OPENAI_API_KEY", "sk-canary-9f3a")
+    data = {
+        "type": "CrawlerRunConfig",
+        "params": {"extraction_strategy": {"type": "dict", "value": _llm_strategy("OPENAI_API_KEY")}},
+    }
+    with pytest.raises(UntrustedConfigError):
+        CrawlerRunConfig.load(data, provenance=UNTRUSTED)
+
+
+def test_browser_config_wrapped_proxy_refused():
+    """Same envelope, other config class, other forbidden type."""
+    data = {
+        "type": "BrowserConfig",
+        "params": {
+            "proxy_config": {
+                "type": "dict",
+                "value": {"type": "ProxyConfig", "params": {"server": "http://evil:8080"}},
+            }
+        },
+    }
+    with pytest.raises(UntrustedConfigError):
+        BrowserConfig.load(data, provenance=UNTRUSTED)
+
+
+def test_trusted_load_is_unchanged(monkeypatch):
+    """TRUSTED (SDK) callers keep the old behavior: no raise, and the wrapped
+    envelope still unwraps to the same plain dict it did before the fix."""
+    monkeypatch.setenv("CRAWL4AI_TEST_TOKEN", "tok-123")
+    cfg = CrawlerRunConfig.load(
+        {"type": "CrawlerRunConfig", "params": {"extraction_strategy": _llm_strategy("CRAWL4AI_TEST_TOKEN")}}
+    )
+    assert cfg.extraction_strategy.llm_config == {
+        "type": "LLMConfig",
+        "params": {"api_token": "env:CRAWL4AI_TEST_TOKEN"},
+    }
+
+
+def test_trusted_top_level_kwargs_still_deserialize(monkeypatch):
+    """from_kwargs still builds typed objects for TRUSTED callers."""
+    monkeypatch.setenv("CRAWL4AI_TEST_TOKEN", "tok-123")
+    cfg = CrawlerRunConfig.load(
+        {
+            "type": "CrawlerRunConfig",
+            "extraction_strategy": {"type": "dict", "value": _llm_strategy("CRAWL4AI_TEST_TOKEN")},
+        }
+    )
+    assert cfg.extraction_strategy.llm_config.api_token == "tok-123"
+
+
+def test_plain_business_dict_with_type_key_still_passes():
+    """A JsonCss schema carries "type" keys but is data, not a typed object."""
+    schema = {
+        "name": "Items",
+        "baseSelector": ".item",
+        "fields": [{"name": "title", "selector": "h1", "type": "text"}],
+    }
+    cfg = CrawlerRunConfig.load(
+        {
+            "type": "CrawlerRunConfig",
+            "params": {
+                "extraction_strategy": {
+                    "type": "JsonCssExtractionStrategy",
+                    "params": {"schema": {"type": "dict", "value": schema}},
+                }
+            },
+        },
+        provenance=UNTRUSTED,
+    )
+    assert cfg.extraction_strategy.schema["baseSelector"] == ".item"

From dd542e7550f7d35c712f4744e40e7abe4b45a067 Mon Sep 17 00:00:00 2001
From: ntohidi <nasrin@kidocode.com>
Date: Mon, 14 Sep 2026 10:55:18 +0200
Subject: [PATCH 12/26] fix(tables): expand rowspan/colspan into a grid and
 keep <th> row headers

DefaultTableExtraction.extract_table_data collected body cells with `.//td`,
so a `<th scope="row">` key column was dropped: every following cell shifted
left and the row was padded with a phantom trailing "". `rowspan` was ignored
entirely, so a spanning value never reached the rows it covers.

Body rows now go through _build_grid, which lays each cell into the rectangle
it spans -- including slots in rows not read yet -- and lets a later row fill
whatever slots are still free. No span state is carried between iterations, so
a short row cannot leave a stale span behind to overwrite a real value one row
further down.

The first row is only skipped as an implicit header when it holds no <td>. A
first row mixing `<th scope="row">` with `<td>` is a key/value data row (the
shape Wikipedia infoboxes use) and stays in `rows`, so the set of rows is
unchanged from before -- only the dropped key column comes back.

Scoring in is_data_table is untouched.

Fixes #2258.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016bfBtYmF5ExJctGcqpKvWv
---
 crawl4ai/table_extraction.py     |  56 +++++++++++---
 tests/test_table_rowspan_grid.py | 123 +++++++++++++++++++++++++++++++
 2 files changed, 168 insertions(+), 11 deletions(-)
 create mode 100644 tests/test_table_rowspan_grid.py

diff --git a/crawl4ai/table_extraction.py b/crawl4ai/table_extraction.py
index 7edb3b761..2379872f2 100644
--- a/crawl4ai/table_extraction.py
+++ b/crawl4ai/table_extraction.py
@@ -210,6 +210,39 @@ def is_data_table(self, table: etree.Element, **kwargs) -> bool:
         threshold = kwargs.get("table_score_threshold", self.table_score_threshold)
         return score >= threshold
     
+    @staticmethod
+    def _build_grid(rows: List[etree.Element]) -> List[List[str]]:
+        """Lay <tr> cells into a rectangular grid, honouring colspan/rowspan.
+
+        Each cell writes its text into every slot of the rectangle it spans,
+        including slots in later rows, so a row only has to fill the slots the
+        rows above it left free. Nothing is carried between iterations, so a
+        short row cannot leave a stale span behind (see issue #2258).
+        """
+        grid: List[List[str]] = [[] for _ in rows]
+
+        def put(r: int, c: int, text: str) -> None:
+            if r >= len(grid):  # rowspan reaching past the last row
+                return
+            row = grid[r]
+            row.extend([None] * (c + 1 - len(row)))
+            row[c] = text
+
+        for r, tr in enumerate(rows):
+            c = 0
+            for cell in tr.xpath("./th|./td"):
+                while c < len(grid[r]) and grid[r][c] is not None:
+                    c += 1  # slot already taken by a rowspan from above
+                text = cell.text_content().strip()
+                colspan = max(int(cell.get("colspan") or 1), 1)
+                rowspan = max(int(cell.get("rowspan") or 1), 1)
+                for dr in range(rowspan):
+                    for dc in range(colspan):
+                        put(r + dr, c + dc, text)
+                c += colspan
+
+        return [[text or "" for text in row] for row in grid if row]
+
     def extract_table_data(self, table: etree.Element) -> Dict[str, Any]:
         """
         Extract structured data from a table element.
@@ -232,6 +265,7 @@ def extract_table_data(self, table: etree.Element) -> Dict[str, Any]:
         
         # Extract headers with colspan handling
         headers = []
+        implicit_header_row = None
         thead_rows = table.xpath(".//thead/tr")
         if thead_rows:
             header_cells = thead_rows[0].xpath(".//th")
@@ -243,21 +277,21 @@ def extract_table_data(self, table: etree.Element) -> Dict[str, Any]:
             # Check first row for headers
             first_row = table.xpath(".//tr[1]")
             if first_row:
-                for cell in first_row[0].xpath(".//th|.//td"):
+                implicit_header_row = first_row[0]
+                for cell in implicit_header_row.xpath(".//th|.//td"):
                     text = cell.text_content().strip()
                     colspan = int(cell.get("colspan", 1))
                     headers.extend([text] * colspan)
         
-        # Extract rows with colspan handling
-        rows = []
-        for row in table.xpath(".//tr[not(ancestor::thead)]"):
-            row_data = []
-            for cell in row.xpath(".//td"):
-                text = cell.text_content().strip()
-                colspan = int(cell.get("colspan", 1))
-                row_data.extend([text] * colspan)
-            if row_data:
-                rows.append(row_data)
+        # Extract rows, laying each cell into a grid so colspan/rowspan land in
+        # the columns they actually cover and <th> row headers are kept (#2258).
+        body_rows = table.xpath(".//tr[not(ancestor::thead)]")
+        # A first row made only of <th> is the header, so don't repeat it as
+        # data. A first row mixing <th scope="row"> with <td> is a key/value
+        # data row and must stay.
+        if implicit_header_row is not None and not implicit_header_row.xpath("./td"):
+            body_rows = body_rows[1:]
+        rows = self._build_grid(body_rows)
         
         # Align rows with headers
         max_columns = len(headers) if headers else (
diff --git a/tests/test_table_rowspan_grid.py b/tests/test_table_rowspan_grid.py
new file mode 100644
index 000000000..a54724daf
--- /dev/null
+++ b/tests/test_table_rowspan_grid.py
@@ -0,0 +1,123 @@
+"""Table grid expansion: <th> row headers, colspan and rowspan.
+
+Issue #2258: body rows were collected with `.//td`, so a `<th scope="row">`
+key column was dropped and every following cell shifted left; `rowspan` was
+ignored entirely, so a spanning value never reached the rows it covers.
+
+https://github.com/unclecode/crawl4ai/issues/2258
+"""
+
+from lxml import html as lhtml
+
+from crawl4ai import DefaultTableExtraction
+
+
+def _extract(html: str):
+    # Wrap the snippet so fromstring always returns a container element.
+    return DefaultTableExtraction().extract_tables(lhtml.fromstring(f"<div>{html}</div>"))
+
+
+def test_th_row_headers_in_body_are_kept():
+    """A <th scope="row"> key column must stay the first column."""
+    tables = _extract("""
+    <table>
+    <tr><th></th><th>Feature A</th><th>Feature B</th></tr>
+    <tr><th scope="row">Item 1</th><td>yes</td><td>yes</td></tr>
+    <tr><th scope="row">Item 2</th><td>no</td><td>yes</td></tr>
+    </table>""")
+    assert len(tables) == 1
+    assert tables[0]["headers"] == ["", "Feature A", "Feature B"]
+    assert tables[0]["rows"] == [["Item 1", "yes", "yes"], ["Item 2", "no", "yes"]]
+
+
+def test_rowspan_value_repeats_into_covered_rows():
+    """A rowspan cell's value appears in every row it covers."""
+    tables = _extract("""
+    <table>
+    <thead><tr><th>Group</th><th>Option X</th><th>Option Y</th></tr></thead>
+    <tbody>
+    <tr><td rowspan="2">Group 1</td><td>value x</td><td>value y</td></tr>
+    <tr><td colspan="2">note that applies to X and Y</td></tr>
+    </tbody></table>""")
+    assert len(tables) == 1
+    assert tables[0]["headers"] == ["Group", "Option X", "Option Y"]
+    assert tables[0]["rows"] == [
+        ["Group 1", "value x", "value y"],
+        ["Group 1", "note that applies to X and Y", "note that applies to X and Y"],
+    ]
+
+
+def test_rowspan_does_not_leak_past_a_short_row():
+    """A span in the last column must land in the short row, not the next one.
+
+    Regression guard: carrying rowspan state between rows and advancing it
+    cell-by-cell stops early on a short row, leaving the span live so it
+    overwrites a real value one row further down.
+    """
+    tables = _extract("""
+    <table>
+    <thead><tr><th>H1</th><th>H2</th><th>H3</th></tr></thead>
+    <tbody>
+    <tr><td>a</td><td>b</td><td rowspan="2">SPAN</td></tr>
+    <tr><td>c</td></tr>
+    <tr><td>x</td><td>y</td><td>z</td></tr>
+    </tbody></table>""")
+    assert tables[0]["rows"] == [
+        ["a", "b", "SPAN"],
+        ["c", "", "SPAN"],
+        ["x", "y", "z"],
+    ]
+
+
+def test_rowspan_and_colspan_on_the_same_cell():
+    """A cell spanning both ways fills the whole rectangle it covers."""
+    tables = _extract("""
+    <table>
+    <thead><tr><th>A</th><th>B</th><th>C</th><th>D</th></tr></thead>
+    <tbody>
+    <tr><td rowspan="2" colspan="2">BIG</td><td>c1</td><td>d1</td></tr>
+    <tr><td>c2</td><td>d2</td></tr>
+    </tbody></table>""")
+    assert tables[0]["rows"] == [
+        ["BIG", "BIG", "c1", "d1"],
+        ["BIG", "BIG", "c2", "d2"],
+    ]
+
+
+def test_rowspan_reaching_past_the_last_row_is_clamped():
+    """An over-long rowspan must not invent rows or raise."""
+    tables = _extract("""
+    <table>
+    <thead><tr><th>A</th><th>B</th></tr></thead>
+    <tbody>
+    <tr><td rowspan="9">X</td><td>b1</td></tr>
+    <tr><td>b2</td></tr>
+    </tbody></table>""")
+    assert tables[0]["rows"] == [["X", "b1"], ["X", "b2"]]
+
+
+def test_plain_table_without_spans_is_unchanged():
+    """Regression guard for the common case."""
+    tables = _extract("""
+    <table>
+    <thead><tr><th>Name</th><th>Qty</th><th>Price</th></tr></thead>
+    <tbody>
+    <tr><td>Widget</td><td>2</td><td>10</td></tr>
+    <tr><td>Gadget</td><td>5</td><td>25</td></tr>
+    </tbody></table>""")
+    assert tables[0]["headers"] == ["Name", "Qty", "Price"]
+    assert tables[0]["rows"] == [["Widget", "2", "10"], ["Gadget", "5", "25"]]
+
+
+def test_key_value_first_row_is_not_mistaken_for_a_header():
+    """A first row of <th scope="row"> + <td> is data, not a header row.
+
+    Real pages (Wikipedia infoboxes) open a two-column key/value table with
+    such a row. Treating it as the header would drop it from `rows`.
+    """
+    tables = _extract("""
+    <table>
+    <tr><th scope="row">Food</th><td>Apple pie</td></tr>
+    <tr><th scope="row">Drink</th><td>Apple cider</td></tr>
+    </table>""")
+    assert tables[0]["rows"] == [["Food", "Apple pie"], ["Drink", "Apple cider"]]

From 9340d09b260bcf57b3d8e025a1e431bb268a4e43 Mon Sep 17 00:00:00 2001
From: ntohidi <nasrin@kidocode.com>
Date: Mon, 14 Sep 2026 11:34:25 +0200
Subject: [PATCH 13/26] fix(playground): replace the Python config editor with
 a JSON params editor
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The Advanced Config panel still posted {type, code} to /config/dump. `code` is
forbidden under the untrusted trust boundary, so the pre-flight always 400'd:

    UntrustedConfigError: field 'code' is not permitted on CrawlerRunConfig
    from an untrusted request

A regex fallback in runCrawl() then scraped the editor text for `stream=True`
and sent {params: {stream: true}} instead, so crawl runs succeeded while
silently discarding every other field the user typed. BrowserConfig has no
`stream`, so no fallback fired and the run aborted outright.

/config/dump stopped eval-ing snippets on purpose -- it was a
gadget-construction oracle -- so the request shape could not simply be patched.
The editor now holds the `params` object itself, which is what the endpoint
accepts today. Bad JSON and rejected fields both stop the run and report the
server's message, naming the offending field.

- CodeMirror switched to JSON mode; templates reseeded from
  UNTRUSTED_FIELD_ALLOWLIST. The old BrowserConfig template used `extra_args`,
  which is not allowlisted and would have 400'd under any request shape.
- pyConfigToJson -> validateConfig, parsing locally before the round trip.
- Regex fallback removed; a rejected config aborts instead of shrinking.
- #cfg-status is cleared when the endpoint changes, so a stale
  '✖ config error' no longer lingers over md/llm, which skip validation.

Fixes #2260.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016bfBtYmF5ExJctGcqpKvWv
---
 deploy/docker/static/playground/index.html    | 111 ++++++++-------
 .../tests/test_playground_config_protocol.py  | 127 ++++++++++++++++++
 2 files changed, 189 insertions(+), 49 deletions(-)
 create mode 100644 deploy/docker/tests/test_playground_config_protocol.py

diff --git a/deploy/docker/static/playground/index.html b/deploy/docker/static/playground/index.html
index e874da472..f905652f0 100644
--- a/deploy/docker/static/playground/index.html
+++ b/deploy/docker/static/playground/index.html
@@ -33,10 +33,10 @@
     <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/github-dark.min.css">
     <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.11/clipboard.min.js"></script>
-    <!-- CodeMirror (python mode) -->
+    <!-- CodeMirror (JSON mode) -->
     <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.16/codemirror.min.css">
     <script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.16/codemirror.min.js"></script>
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.16/mode/python/python.min.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.16/mode/javascript/javascript.min.js"></script>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.16/addon/edit/matchbrackets.min.js"></script>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.16/addon/selection/active-line.min.js"></script>
     <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.16/theme/darcula.min.css">
@@ -251,7 +251,7 @@ <h2 class="font-medium">Request Builder</h2>
                 <!-- Advanced config for /crawl endpoints -->
                 <details id="adv-config" class="mb-4">
                     <summary class="text-sm text-secondary cursor-pointer">Advanced Config <span
-                        class="text-xs text-primary">(Python → auto‑JSON)</span></summary>
+                        class="text-xs text-primary">(JSON params)</span></summary>
 
                     <!-- Toolbar -->
                     <div class="flex items-center justify-end space-x-3 mt-2">
@@ -498,12 +498,24 @@ <h2 class="font-medium text-accent">🔥 Stress Test</h2>
         });
 
         // ================ ADVANCED CONFIG EDITOR ================
+        // The server validates every field against UNTRUSTED_FIELD_ALLOWLIST, so the
+        // editor holds the `params` object literally rather than a Python snippet:
+        // /config/dump stopped eval-ing code in the 0.8.x security work (issue #2260).
+        const TEMPLATES = {
+            CrawlerRunConfig: `{
+    "stream": true,
+    "cache_mode": "bypass"
+}`,
+            BrowserConfig: `{
+    "headless": true,
+    "viewport_width": 1280,
+    "viewport_height": 720
+}`,
+        };
+
         const cm = CodeMirror(document.getElementById('adv-editor'), {
-            value: `CrawlerRunConfig(
-    stream=True,
-    cache_mode=CacheMode.BYPASS,
-)`,
-            mode: 'python',
+            value: TEMPLATES.CrawlerRunConfig,
+            mode: 'application/json',
             lineNumbers: true,
             theme: 'darcula',
             tabSize: 4,
@@ -513,23 +525,9 @@ <h2 class="font-medium text-accent">🔥 Stress Test</h2>
             lineWrapping: true,
         });
 
-        const TEMPLATES = {
-            CrawlerRunConfig: `CrawlerRunConfig(
-    stream=True,
-    cache_mode=CacheMode.BYPASS,
-)`,
-            BrowserConfig: `BrowserConfig(
-    headless=True,
-    extra_args=[
-        "--no-sandbox",
-        "--disable-gpu",
-    ],
-)`,
-        };
-
         document.getElementById('cfg-type').addEventListener('change', (e) => {
             cm.setValue(TEMPLATES[e.target.value]);
-            document.getElementById('cfg-status').textContent = '';
+            setCfgStatus('', 'text-secondary');
         });
         
         // Handle endpoint selection change to show appropriate options
@@ -539,6 +537,10 @@ <h2 class="font-medium text-accent">🔥 Stress Test</h2>
             const llmOptions = document.getElementById('llm-options');
             const advConfig = document.getElementById('adv-config');
             
+            // Advanced Config is /crawl-only; clear its status so a previous
+            // run's '✖ config error' does not linger on md/llm (#2260).
+            setCfgStatus('', 'text-secondary');
+
             // Hide all option sections first
             mdOptions.classList.add('hidden');
             llmOptions.classList.add('hidden');
@@ -559,30 +561,46 @@ <h2 class="font-medium text-accent">🔥 Stress Test</h2>
             }
         });
 
-        async function pyConfigToJson() {
-            const code = cm.getValue().trim();
-            if (!code) return {};
+        function setCfgStatus(text, cls) {
+            const el = document.getElementById('cfg-status');
+            el.textContent = text;
+            el.className = 'text-xs ml-2 ' + cls;
+        }
+
+        // Validate the editor's params against /config/dump and return the
+        // normalized { type, params } the crawl endpoints expect. Throws with the
+        // server's message so the caller can surface the offending field.
+        async function validateConfig() {
+            const text = cm.getValue().trim();
+            if (!text) return {};
+
+            let params;
+            try {
+                params = JSON.parse(text);
+            } catch (err) {
+                setCfgStatus('✖ not valid JSON', 'text-red-400');
+                throw new Error('Advanced Config is not valid JSON: ' + err.message);
+            }
+            if (params === null || typeof params !== 'object' || Array.isArray(params)) {
+                setCfgStatus('✖ expected a JSON object', 'text-red-400');
+                throw new Error('Advanced Config must be a JSON object of parameters');
+            }
+            if (!Object.keys(params).length) return {};
 
-            // Server requires `type` alongside `code` (CrawlerRunConfig | BrowserConfig).
-            // The UI already tracks this in #cfg-type; omitting it always 400s Advanced Config.
             const cfgType = document.getElementById('cfg-type').value;
             const res = await authFetch('/config/dump', {
                 method: 'POST',
                 headers: { 'Content-Type': 'application/json' },
-                body: JSON.stringify({ type: cfgType, code }),
+                body: JSON.stringify({ type: cfgType, params }),
             });
 
-            const statusEl = document.getElementById('cfg-status');
             if (!res.ok) {
                 const msg = await res.text();
-                statusEl.textContent = '✖ config error';
-                statusEl.className = 'text-xs text-red-400';
+                setCfgStatus('✖ config error', 'text-red-400');
                 throw new Error(msg || 'Invalid config');
             }
 
-            statusEl.textContent = '✓ parsed';
-            statusEl.className = 'text-xs text-green-400';
-
+            setCfgStatus('✓ validated', 'text-green-400');
             return await res.json();
         }
 
@@ -686,12 +704,12 @@ <h2 class="font-medium text-accent">🔥 Stress Test</h2>
         async function runCrawl() {
             const endpoint = document.getElementById('endpoint').value;
             const urls = document.getElementById('urls').value.trim().split(/\n/).filter(u => u);
-            // 1) grab python from CodeMirror, validate via /config/dump.
+            // 1) grab the params from CodeMirror, validate via /config/dump.
             // Advanced Config is /crawl-only, so md/llm skip this entirely.
             let advConfig = {};
             if (endpoint === 'crawl' || endpoint === 'crawl_stream') {
                 try {
-                    const cfgJson = await pyConfigToJson(); // may throw
+                    const cfgJson = await validateConfig(); // may throw
                     if (Object.keys(cfgJson).length) {
                         const cfgType = document.getElementById('cfg-type').value;
                         advConfig = cfgType === 'CrawlerRunConfig'
@@ -699,18 +717,13 @@ <h2 class="font-medium text-accent">🔥 Stress Test</h2>
                             : { browser_config: cfgJson };
                     }
                 } catch (err) {
-                    const codeText = cm.getValue();
-                    const streamFlag = /stream\s*=\s*True/i.test(codeText);
-                    if (streamFlag) {
-                        // Fallback: minimal dump-shaped config so shouldUseStream + server load agree
-                        advConfig = { crawler_config: { type: 'CrawlerRunConfig', params: { stream: true } } };
-                    } else {
-                        updateStatus('error');
-                        document.querySelector('#response-content code').textContent =
-                            JSON.stringify({ error: err.message }, null, 2);
-                        forceHighlightElement(document.querySelector('#response-content code'));
-                        return; // stop run
-                    }
+                    // No fallback: a rejected config used to be swapped for a
+                    // stream-only one, which silently discarded everything else (#2260).
+                    updateStatus('error');
+                    document.querySelector('#response-content code').textContent =
+                        JSON.stringify({ error: err.message }, null, 2);
+                    forceHighlightElement(document.querySelector('#response-content code'));
+                    return; // stop run
                 }
             }
 
diff --git a/deploy/docker/tests/test_playground_config_protocol.py b/deploy/docker/tests/test_playground_config_protocol.py
new file mode 100644
index 000000000..dcb330e2f
--- /dev/null
+++ b/deploy/docker/tests/test_playground_config_protocol.py
@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+"""The playground's Advanced Config panel must speak the protocol the server
+still accepts.
+
+Issue #2260: the panel posted {type, code} to /config/dump long after the 0.8.x
+security work dropped the eval-based `code` path. `code` is forbidden under the
+untrusted trust boundary, so the pre-flight always 400'd; a regex fallback in
+runCrawl() then swapped the user's config for a stream-only one, so everything
+else they typed was silently discarded.
+
+These tests read the shipped page as text -- no browser, no server boot -- and
+check the templates against the real untrusted gate.
+"""
+
+import json
+import os
+import re
+import unittest
+
+from crawl4ai.async_configs import (
+    UNTRUSTED_FIELD_ALLOWLIST,
+    BrowserConfig,
+    CrawlerRunConfig,
+    Provenance,
+)
+
+PLAYGROUND = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)), "..", "static", "playground", "index.html"
+)
+CONFIG_CLASSES = {"CrawlerRunConfig": CrawlerRunConfig, "BrowserConfig": BrowserConfig}
+
+
+def read_page() -> str:
+    with open(PLAYGROUND, encoding="utf-8") as fh:
+        return fh.read()
+
+
+def read_templates() -> dict:
+    """Pull the TEMPLATES literal out of the page."""
+    block = re.search(r"const TEMPLATES = \{(.*?)\n        \};", read_page(), re.S)
+    assert block, "TEMPLATES literal not found in the playground"
+    return dict(re.findall(r"(\w+): `(\{.*?\})`", block.group(1), re.S))
+
+
+class TestConfigProtocol(unittest.TestCase):
+    def test_config_dump_is_not_sent_a_code_field(self):
+        """`code` is forbidden under UNTRUSTED, so sending it always 400s."""
+        page = read_page()
+        call = re.search(r"authFetch\('/config/dump'.*?\}\);", page, re.S)
+        self.assertIsNotNone(call, "/config/dump call not found")
+        self.assertNotIn("code", call.group(0).split("body:")[1],
+                         "playground must not post a `code` field to /config/dump")
+        self.assertIn("params", call.group(0),
+                      "playground must post the {type, params} shape /config/dump accepts")
+
+    def test_rejected_config_aborts_instead_of_falling_back(self):
+        """The stream=True regex fallback is what made the config loss silent."""
+        page = read_page()
+        self.assertNotIn("streamFlag", page, "the regex fallback must be gone")
+        self.assertNotRegex(page, r"stream\\s\*=\\s\*True",
+                            "the editor text must not be regex-scraped for stream")
+
+
+class TestTemplatesSurviveTheUntrustedGate(unittest.TestCase):
+    def test_templates_are_valid_json_objects(self):
+        templates = read_templates()
+        self.assertEqual(set(templates), set(CONFIG_CLASSES))
+        for name, body in templates.items():
+            with self.subTest(config=name):
+                params = json.loads(body)
+                self.assertIsInstance(params, dict)
+                self.assertTrue(params, "template must not be empty")
+
+    def test_every_template_field_is_allowlisted(self):
+        """A template field outside the allowlist would 400 on every run."""
+        for name, body in read_templates().items():
+            with self.subTest(config=name):
+                unknown = set(json.loads(body)) - UNTRUSTED_FIELD_ALLOWLIST[name]
+                self.assertFalse(unknown, f"{name} template uses forbidden field(s): {unknown}")
+
+    def test_every_template_field_reaches_the_config_object(self):
+        """Load each template the way /config/dump does and read the result back.
+
+        dump() omits values equal to the class default, so assert on the object,
+        not on the echoed payload.
+        """
+        for name, body in read_templates().items():
+            with self.subTest(config=name):
+                params = json.loads(body)
+                obj = CONFIG_CLASSES[name].load(
+                    {"type": name, "params": params}, provenance=Provenance.UNTRUSTED
+                )
+                applied = {key: getattr(obj, key) for key in params}
+                self.assertEqual(applied, params)
+
+    def test_stream_reaches_the_shape_should_use_stream_reads(self):
+        """runCrawl picks the streaming endpoint off crawler_config.params.stream."""
+        params = json.loads(read_templates()["CrawlerRunConfig"])
+        dumped = CrawlerRunConfig.load(
+            {"type": "CrawlerRunConfig", "params": params}, provenance=Provenance.UNTRUSTED
+        ).dump()
+        self.assertIs(dumped["params"].get("stream"), True)
+
+    def test_the_old_code_protocol_is_still_refused(self):
+        """Guards the reason for this change: {type, code} cannot come back."""
+        for name, cls in CONFIG_CLASSES.items():
+            with self.subTest(config=name):
+                with self.assertRaises(Exception) as ctx:
+                    cls.load({"type": name, "code": f"{name}()"}, provenance=Provenance.UNTRUSTED)
+                self.assertIn("code", str(ctx.exception))
+
+
+class TestConfigStatusIsReset(unittest.TestCase):
+    def test_status_is_cleared_when_the_endpoint_changes(self):
+        """md/llm skip validation, so a stale '✖ config error' used to linger."""
+        page = read_page()
+        handler = re.search(
+            r"getElementById\('endpoint'\)\.addEventListener\('change'.*?\n        \}\);",
+            page, re.S,
+        )
+        self.assertIsNotNone(handler, "endpoint change handler not found")
+        self.assertIn("setCfgStatus('',", handler.group(0),
+                      "switching endpoint must clear #cfg-status")
+
+
+if __name__ == "__main__":
+    unittest.main()

From e138aa0f68e5cbb4186e0110abc4a03123a5b28c Mon Sep 17 00:00:00 2001
From: ntohidi <nasrin@kidocode.com>
Date: Mon, 14 Sep 2026 12:13:03 +0200
Subject: [PATCH 14/26] fix(playground): forward the typed params, and clear
 status when config is empty
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Two defects found while driving the panel in a real browser against the Docker
server:

- /config/dump's echo is not re-submittable. dump() fills in server-derived
  fields, and BrowserConfig gains a generated `headers` dict that is not on
  UNTRUSTED_FIELD_ALLOWLIST. Posting the echo to /crawl therefore 400'd with
  "field 'headers' is not permitted on BrowserConfig". CrawlerRunConfig only
  escaped by luck: every field dump() adds there happens to be allowlisted.
  /config/dump is a validator, so the page now forwards what the user typed
  and keeps the echo as a pass/fail signal.

- An empty editor returned early without touching #cfg-status, so a previous
  run's '✖ config error' stayed on screen while the run went ahead.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016bfBtYmF5ExJctGcqpKvWv
---
 deploy/docker/static/playground/index.html    | 20 +++++++++++----
 .../tests/test_playground_config_protocol.py  | 25 ++++++++++++++++---
 2 files changed, 37 insertions(+), 8 deletions(-)

diff --git a/deploy/docker/static/playground/index.html b/deploy/docker/static/playground/index.html
index f905652f0..1b0353b5b 100644
--- a/deploy/docker/static/playground/index.html
+++ b/deploy/docker/static/playground/index.html
@@ -568,11 +568,14 @@ <h2 class="font-medium text-accent">🔥 Stress Test</h2>
         }
 
         // Validate the editor's params against /config/dump and return the
-        // normalized { type, params } the crawl endpoints expect. Throws with the
-        // server's message so the caller can surface the offending field.
+        // { type, params } the crawl endpoints expect. Throws with the server's
+        // message so the caller can surface the offending field.
         async function validateConfig() {
             const text = cm.getValue().trim();
-            if (!text) return {};
+            if (!text) {
+                setCfgStatus('', 'text-secondary');  // no config is not an error
+                return {};
+            }
 
             let params;
             try {
@@ -585,7 +588,10 @@ <h2 class="font-medium text-accent">🔥 Stress Test</h2>
                 setCfgStatus('✖ expected a JSON object', 'text-red-400');
                 throw new Error('Advanced Config must be a JSON object of parameters');
             }
-            if (!Object.keys(params).length) return {};
+            if (!Object.keys(params).length) {
+                setCfgStatus('', 'text-secondary');  // no config is not an error
+                return {};
+            }
 
             const cfgType = document.getElementById('cfg-type').value;
             const res = await authFetch('/config/dump', {
@@ -601,7 +607,11 @@ <h2 class="font-medium text-accent">🔥 Stress Test</h2>
             }
 
             setCfgStatus('✓ validated', 'text-green-400');
-            return await res.json();
+            // Send back what the user typed, not /config/dump's echo. dump()
+            // fills in server-derived fields -- BrowserConfig gains generated
+            // `headers` -- that the untrusted gate then refuses on /crawl, so
+            // the echo is a validation result, not a re-submittable payload.
+            return { type: cfgType, params };
         }
 
         // ================ SERVER COMMUNICATION ================
diff --git a/deploy/docker/tests/test_playground_config_protocol.py b/deploy/docker/tests/test_playground_config_protocol.py
index dcb330e2f..926b113d4 100644
--- a/deploy/docker/tests/test_playground_config_protocol.py
+++ b/deploy/docker/tests/test_playground_config_protocol.py
@@ -96,10 +96,29 @@ def test_every_template_field_reaches_the_config_object(self):
     def test_stream_reaches_the_shape_should_use_stream_reads(self):
         """runCrawl picks the streaming endpoint off crawler_config.params.stream."""
         params = json.loads(read_templates()["CrawlerRunConfig"])
-        dumped = CrawlerRunConfig.load(
-            {"type": "CrawlerRunConfig", "params": params}, provenance=Provenance.UNTRUSTED
+        self.assertIs(params.get("stream"), True)
+
+    def test_the_validated_params_are_forwarded_not_the_dump_echo(self):
+        """dump() adds server-derived fields the untrusted gate then refuses.
+
+        BrowserConfig picks up generated `headers`, which is not allowlisted, so
+        posting /config/dump's echo to /crawl 400s. The page must forward the
+        params the user typed.
+        """
+        params = json.loads(read_templates()["BrowserConfig"])
+        echo = BrowserConfig.load(
+            {"type": "BrowserConfig", "params": params}, provenance=Provenance.UNTRUSTED
         ).dump()
-        self.assertIs(dumped["params"].get("stream"), True)
+        forbidden = set(echo["params"]) - UNTRUSTED_FIELD_ALLOWLIST["BrowserConfig"]
+        self.assertTrue(
+            forbidden,
+            "if dump() ever became re-submittable this guard can go, but check the page first",
+        )
+        page = read_page()
+        self.assertIn("return { type: cfgType, params };", page,
+                      "validateConfig must forward the typed params, not the echo")
+        self.assertNotIn("return await res.json();", page,
+                         "the /config/dump echo must not be forwarded to /crawl")
 
     def test_the_old_code_protocol_is_still_refused(self):
         """Guards the reason for this change: {type, code} cannot come back."""

From ee1a295394a09269d722c9f96289d89eae10d022 Mon Sep 17 00:00:00 2001
From: ntohidi <nasrin@kidocode.com>
Date: Mon, 14 Sep 2026 12:58:07 +0200
Subject: [PATCH 15/26] fix(tables): clamp colspan and rowspan so one cell
 cannot hang the parse

_build_grid walked rowspan * colspan slots per cell. put() ignored writes past
the last row, but the loop still spun, so the cost was the product of the two
attributes rather than of the table. develop never read rowspan at all, so this
arrived with the grid.

Measured, one cell, two-row table:

    <td rowspan="1000000" colspan="1000000">   develop 0.02s   grid  >10s, no end
    <td colspan="10000000">                    develop 0.10s   grid  2.79s

Scraped HTML is untrusted input, so a crafted cell hangs the crawl -- and on the
Docker server that holds a pooled browser.

Both spans are now clamped: colspan to the HTML Standard's cap of 1000, which is
what browsers apply, and rowspan to the rows that actually exist, a tighter bound
than the standard's 65534. All three cases above drop to 0.00s and the cost is
linear in the grid again.

Junk values are read the way a browser reads them, as 1, instead of raising:
colspan="100%" used to escape extract_table_data as a ValueError.

Reported in review of #2261.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016bfBtYmF5ExJctGcqpKvWv
---
 crawl4ai/table_extraction.py     | 25 +++++++++++++++-----
 tests/test_table_rowspan_grid.py | 39 ++++++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+), 6 deletions(-)

diff --git a/crawl4ai/table_extraction.py b/crawl4ai/table_extraction.py
index 2379872f2..8440cadfd 100644
--- a/crawl4ai/table_extraction.py
+++ b/crawl4ai/table_extraction.py
@@ -210,8 +210,11 @@ def is_data_table(self, table: etree.Element, **kwargs) -> bool:
         threshold = kwargs.get("table_score_threshold", self.table_score_threshold)
         return score >= threshold
     
-    @staticmethod
-    def _build_grid(rows: List[etree.Element]) -> List[List[str]]:
+    # HTML Standard caps colspan at 1000; browsers clamp to the same value.
+    COLSPAN_LIMIT = 1000
+
+    @classmethod
+    def _build_grid(cls, rows: List[etree.Element]) -> List[List[str]]:
         """Lay <tr> cells into a rectangular grid, honouring colspan/rowspan.
 
         Each cell writes its text into every slot of the rectangle it spans,
@@ -221,9 +224,14 @@ def _build_grid(rows: List[etree.Element]) -> List[List[str]]:
         """
         grid: List[List[str]] = [[] for _ in rows]
 
+        def span(value, limit: int) -> int:
+            """Read a span attribute the way a browser does: junk counts as 1."""
+            try:
+                return min(max(int(value), 1), limit)
+            except (TypeError, ValueError):
+                return 1
+
         def put(r: int, c: int, text: str) -> None:
-            if r >= len(grid):  # rowspan reaching past the last row
-                return
             row = grid[r]
             row.extend([None] * (c + 1 - len(row)))
             row[c] = text
@@ -234,8 +242,13 @@ def put(r: int, c: int, text: str) -> None:
                 while c < len(grid[r]) and grid[r][c] is not None:
                     c += 1  # slot already taken by a rowspan from above
                 text = cell.text_content().strip()
-                colspan = max(int(cell.get("colspan") or 1), 1)
-                rowspan = max(int(cell.get("rowspan") or 1), 1)
+                # Both spans are clamped, so one crafted cell cannot make the
+                # work explode: scraped HTML is untrusted, and rowspan*colspan
+                # is a product. COLSPAN_LIMIT is the HTML Standard's cap, which
+                # is what browsers apply; a rowspan cannot reach past the last
+                # row, which is a tighter bound than the standard's 65534.
+                colspan = span(cell.get("colspan"), cls.COLSPAN_LIMIT)
+                rowspan = span(cell.get("rowspan"), len(grid) - r)
                 for dr in range(rowspan):
                     for dc in range(colspan):
                         put(r + dr, c + dc, text)
diff --git a/tests/test_table_rowspan_grid.py b/tests/test_table_rowspan_grid.py
index a54724daf..35125a4b9 100644
--- a/tests/test_table_rowspan_grid.py
+++ b/tests/test_table_rowspan_grid.py
@@ -7,6 +7,8 @@
 https://github.com/unclecode/crawl4ai/issues/2258
 """
 
+import time
+
 from lxml import html as lhtml
 
 from crawl4ai import DefaultTableExtraction
@@ -121,3 +123,40 @@ def test_key_value_first_row_is_not_mistaken_for_a_header():
     <tr><th scope="row">Drink</th><td>Apple cider</td></tr>
     </table>""")
     assert tables[0]["rows"] == [["Food", "Apple pie"], ["Drink", "Apple cider"]]
+
+
+def test_absurd_spans_are_clamped_instead_of_exploding():
+    """rowspan * colspan is a product, and scraped HTML is untrusted.
+
+    One crafted cell must not be able to hang the crawl. rowspan is bounded by
+    the rows that exist, colspan by the HTML Standard's cap.
+    """
+    html = """
+    <table>
+    <thead><tr><th>A</th><th>B</th></tr></thead>
+    <tbody>
+    <tr><td rowspan="1000000" colspan="1000000">X</td><td>b1</td></tr>
+    <tr><td>c1</td><td>c2</td></tr>
+    </tbody></table>"""
+    started = time.perf_counter()
+    tables = _extract(html)
+    elapsed = time.perf_counter() - started
+
+    assert elapsed < 2, f"clamping regressed: took {elapsed:.1f}s"
+    rows = tables[0]["rows"]
+    assert len(rows) == 2, "rowspan must not invent rows beyond the table"
+    limit = DefaultTableExtraction.COLSPAN_LIMIT
+    assert all(len(row) <= limit for row in rows), "colspan must be capped"
+
+
+def test_invalid_span_attribute_counts_as_one():
+    """Browsers treat junk spans as 1; parsing must not raise on them."""
+    tables = _extract("""
+    <table>
+    <thead><tr><th>A</th><th>B</th></tr></thead>
+    <tbody>
+    <tr><td colspan="100%">a</td><td>b</td></tr>
+    <tr><td rowspan="lots">c</td><td>d</td></tr>
+    <tr><td colspan="-3">e</td><td>f</td></tr>
+    </tbody></table>""")
+    assert tables[0]["rows"] == [["a", "b"], ["c", "d"], ["e", "f"]]

From 519d85fc65ee1522f972e4d9028519597bfa92e1 Mon Sep 17 00:00:00 2001
From: ntohidi <nasrin@kidocode.com>
Date: Mon, 14 Sep 2026 14:12:14 +0200
Subject: [PATCH 16/26] fix(ci): don't let the stargazer tracking step block
 Discord notifications

The "Send to Google Apps Script (Stars only)" step uses `curl -fSs`, so an
HTTP error from GOOGLE_SCRIPT_ENDPOINT exits 22 and kills the job before the
Discord notification runs. The endpoint is currently returning 403, so every
`watch` event fails.

Spreadsheet tracking is best-effort, so mark the step continue-on-error.

Fixes #2255

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Utx8BTDqXTscjquLV7vgLq
---
 .github/workflows/main.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index b53eb7be4..cb324a145 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -18,6 +18,7 @@ jobs:
     steps:
       - name: Send to Google Apps Script (Stars only)
         if: github.event_name == 'watch'
+        continue-on-error: true
         run: |
           curl -fSs -X POST "${{ secrets.GOOGLE_SCRIPT_ENDPOINT }}" \
             -H 'Content-Type: application/json' \

From 19f3e5e11f3337dde0091a67bea02fa79c8721aa Mon Sep 17 00:00:00 2001
From: ntohidi <nasrin@kidocode.com>
Date: Mon, 14 Sep 2026 16:45:47 +0200
Subject: [PATCH 17/26] fix(deep-crawl): drop the O(n^2) parent scan and the
 duplicate best-first enqueue

BFS matched each fetched result back to its parent with
`next((parent for (u, parent) in current_level if u == url), None)`, re-scanning
the whole level once per result. On a page with high fan-out that pure-Python
bookkeeping dominates the level. current_level is already a list of
(url, parent) pairs, so build the lookup once per level instead. reversed()
keeps the old first-parent-wins behaviour if a resumed level repeats a URL.

BestFirst only added a URL to `visited` when it was dequeued, so two pages
linking to the same third page pushed it onto the priority queue twice. The
duplicate was dropped at dequeue time, so output stayed correct, but the URL was
scored and queued for nothing. BFS and DFS both already mark URLs at discovery
time; make BestFirst match and the dequeue guard becomes unreachable. Restoring
a checkpoint now de-dupes the saved queue and treats everything queued as seen,
so states written by older builds still resume without a repeat crawl.

Fixes #2242

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Utx8BTDqXTscjquLV7vgLq
---
 crawl4ai/deep_crawling/bff_strategy.py        |  11 +-
 crawl4ai/deep_crawling/bfs_strategy.py        |   8 +-
 .../test_deep_crawl_perf_2242.py              | 147 ++++++++++++++++++
 3 files changed, 159 insertions(+), 7 deletions(-)
 create mode 100644 tests/deep_crawling/test_deep_crawl_perf_2242.py

diff --git a/crawl4ai/deep_crawling/bff_strategy.py b/crawl4ai/deep_crawling/bff_strategy.py
index 511fde692..4bc247c0d 100644
--- a/crawl4ai/deep_crawling/bff_strategy.py
+++ b/crawl4ai/deep_crawling/bff_strategy.py
@@ -183,6 +183,7 @@ async def link_discovery(
                 self.stats.urls_skipped += 1
                 continue
                 
+            visited.add(base_url)
             valid_links.append(base_url)
             
         # Record the new depths and add to next_links
@@ -214,6 +215,10 @@ async def _arun_best_first(
             self._pages_crawled = self._resume_state.get("pages_crawled", 0)
             # Restore queue from saved items
             queue_items = self._resume_state.get("queue_items", [])
+            # Older checkpoints could hold a URL twice and recorded only crawled
+            # URLs in "visited"; drop the repeats and treat everything queued as seen.
+            queue_items = list({item["url"]: item for item in queue_items}.values())
+            visited.update(item["url"] for item in queue_items)
             for item in queue_items:
                 await queue.put((item["score"], item["depth"], item["url"], item["parent_url"]))
             # Initialize shadow list if callback is set
@@ -226,7 +231,7 @@ async def _arun_best_first(
             # Original initialization
             initial_score = self.url_scorer.score(start_url) if self.url_scorer else 0
             await queue.put((-initial_score, 0, start_url, None))
-            visited: Set[str] = set()
+            visited: Set[str] = {start_url}
             depths: Dict[str, int] = {start_url: 0}
             # Initialize shadow list if callback is set
             if self._on_state_change:
@@ -263,10 +268,6 @@ async def _arun_best_first(
                         self._queue_shadow.remove(item)
                     except ValueError:
                         pass  # Item may have been removed already
-                score, depth, url, parent_url = item
-                if url in visited:
-                    continue
-                visited.add(url)
                 batch.append(item)
 
             if not batch:
diff --git a/crawl4ai/deep_crawling/bfs_strategy.py b/crawl4ai/deep_crawling/bfs_strategy.py
index dfb759272..2b00a6cda 100644
--- a/crawl4ai/deep_crawling/bfs_strategy.py
+++ b/crawl4ai/deep_crawling/bfs_strategy.py
@@ -248,6 +248,8 @@ async def _arun_batch(
 
             next_level: List[Tuple[str, Optional[str]]] = []
             urls = [url for url, _ in current_level]
+            # reversed() keeps first-parent-wins if a resumed level repeats a URL
+            parents = dict(reversed(current_level))
 
             # Clone the config to disable deep crawling recursion and enforce batch mode.
             batch_config = config.clone(deep_crawl_strategy=None, stream=False)
@@ -258,7 +260,7 @@ async def _arun_batch(
                 depth = depths.get(url, 0)
                 result.metadata = result.metadata or {}
                 result.metadata["depth"] = depth
-                parent_url = next((parent for (u, parent) in current_level if u == url), None)
+                parent_url = parents.get(url)
                 result.metadata["parent_url"] = parent_url
                 results.append(result)
 
@@ -336,6 +338,8 @@ async def _arun_stream(
 
             next_level: List[Tuple[str, Optional[str]]] = []
             urls = [url for url, _ in current_level]
+            # reversed() keeps first-parent-wins if a resumed level repeats a URL
+            parents = dict(reversed(current_level))
             visited.update(urls)
 
             stream_config = config.clone(deep_crawl_strategy=None, stream=True)
@@ -348,7 +352,7 @@ async def _arun_stream(
                 depth = depths.get(url, 0)
                 result.metadata = result.metadata or {}
                 result.metadata["depth"] = depth
-                parent_url = next((parent for (u, parent) in current_level if u == url), None)
+                parent_url = parents.get(url)
                 result.metadata["parent_url"] = parent_url
                 
                 # Count only successful crawls
diff --git a/tests/deep_crawling/test_deep_crawl_perf_2242.py b/tests/deep_crawling/test_deep_crawl_perf_2242.py
new file mode 100644
index 000000000..f27826767
--- /dev/null
+++ b/tests/deep_crawling/test_deep_crawl_perf_2242.py
@@ -0,0 +1,147 @@
+"""Regression tests for issue #2242.
+
+1. BFS matched a fetched result back to its parent by re-scanning the whole
+   level, which is O(n^2) per level.
+2. Best-First only marked a URL visited when it was dequeued, so two pages
+   linking to the same third page pushed it onto the priority queue twice.
+"""
+
+import asyncio
+from collections import Counter
+from typing import Any, Dict, List
+from unittest.mock import MagicMock
+
+import pytest
+
+from crawl4ai.deep_crawling import BFSDeepCrawlStrategy, BestFirstCrawlingStrategy
+
+
+def make_config(stream: bool = False):
+    config = MagicMock()
+    config.clone = MagicMock(return_value=config)
+    config.stream = stream
+    return config
+
+
+class FakeResult:
+    """Cheap stand-in for CrawlResult (MagicMock is too slow to time against)."""
+
+    def __init__(self, url: str, children: List[str]):
+        self.url = url
+        self.success = True
+        self.metadata: Dict[str, Any] = {}
+        self.links = {
+            "internal": [{"href": child} for child in children],
+            "external": [],
+        }
+
+
+def make_crawler(link_map: Dict[str, List[str]]):
+    """Mock crawler serving a fixed url -> child urls map."""
+
+    async def arun_many(urls, config):
+        results = [FakeResult(url, link_map.get(url, [])) for url in urls]
+
+        if config.stream:
+            async def gen():
+                for result in results:
+                    yield result
+            return gen()
+        return results
+
+    crawler = MagicMock()
+    crawler.arun_many = arun_many
+    return crawler
+
+
+ROOT = "https://example.com"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("stream", [False, True])
+async def test_bfs_parent_url_is_correct_for_every_child(stream):
+    """Parent bookkeeping must stay correct now that it uses a lookup table."""
+    children = [f"{ROOT}/p{i}" for i in range(50)]
+    link_map = {ROOT: children, **{c: [f"{c}/leaf"] for c in children}}
+
+    strategy = BFSDeepCrawlStrategy(max_depth=1, max_pages=100)
+    crawler, config = make_crawler(link_map), make_config(stream=stream)
+
+    if stream:
+        results = [r async for r in strategy._arun_stream(ROOT, crawler, config)]
+    else:
+        results = await strategy._arun_batch(ROOT, crawler, config)
+
+    parents = {r.url: r.metadata["parent_url"] for r in results}
+    assert parents[ROOT] is None
+    assert len(results) == len(children) + 1
+    for child in children:
+        assert parents[child] == ROOT
+
+
+@pytest.mark.asyncio
+async def test_bfs_level_bookkeeping_scales_linearly():
+    """Matching results back to parents must not re-scan the level per result.
+
+    max_depth=0 keeps link discovery out of the loop and resume_state seeds a
+    level of arbitrary width, so the only per-result work left is the parent
+    lookup. Quadratic bookkeeping shows up as a ~4x cost for 2x the URLs.
+    """
+
+    async def run(n: int) -> float:
+        urls = [f"{ROOT}/p{i}" for i in range(n)]
+        strategy = BFSDeepCrawlStrategy(
+            max_depth=0,
+            max_pages=n + 1,
+            resume_state={
+                "visited": urls,
+                "pending": [{"url": u, "parent_url": ROOT} for u in urls],
+                "depths": {u: 1 for u in urls},
+                "pages_crawled": 0,
+            },
+        )
+        crawler, config = make_crawler({}), make_config()
+        loop = asyncio.get_running_loop()
+        start = loop.time()
+        results = await strategy._arun_batch(ROOT, crawler, config)
+        elapsed = loop.time() - start
+        assert len(results) == n
+        return elapsed
+
+    async def best_of(n: int, rounds: int = 3) -> float:
+        return min([await run(n) for _ in range(rounds)])
+
+    await run(200)  # warm up
+    small = await best_of(2000)
+    large = await best_of(4000)
+
+    # 2x the URLs: linear is ~2x, the old full-level scan was ~4x.
+    assert large / small < 3, f"level bookkeeping looks superlinear: {large / small:.1f}x"
+
+
+@pytest.mark.asyncio
+async def test_best_first_queues_a_shared_url_once():
+    """Two parents linking to the same page must enqueue it a single time."""
+    link_map = {
+        ROOT: [f"{ROOT}/a", f"{ROOT}/b"],
+        f"{ROOT}/a": [f"{ROOT}/shared"],
+        f"{ROOT}/b": [f"{ROOT}/shared"],
+        f"{ROOT}/shared": [],
+    }
+
+    snapshots: List[List[str]] = []
+
+    async def on_state_change(state: Dict[str, Any]):
+        snapshots.append([item["url"] for item in state["queue_items"]])
+
+    strategy = BestFirstCrawlingStrategy(
+        max_depth=2, max_pages=10, on_state_change=on_state_change
+    )
+    crawled = [
+        r.url
+        async for r in strategy._arun_stream(ROOT, make_crawler(link_map), make_config(stream=True))
+    ]
+
+    for queue in snapshots:
+        assert not [u for u, n in Counter(queue).items() if n > 1], f"duplicate in queue: {queue}"
+    assert sorted(crawled) == sorted(link_map)

From 61eb573416fbae2c7562214d142ac27f351cfdea Mon Sep 17 00:00:00 2001
From: ntohidi <nasrin@kidocode.com>
Date: Mon, 14 Sep 2026 17:12:04 +0200
Subject: [PATCH 18/26] fix(config): keep malformed timeouts at the default
 ceiling

Follow-up to #2212. Three loose ends from review:

- _cap_timeout returned the configured ceiling for any non-numeric or
  non-positive value, so raising CRAWL4AI_MAX_TIMEOUT_MS also raised what
  junk input became. An untrusted caller sending page_timeout 0 or "abc"
  got the full raised ceiling of held browser page. It now falls back to
  the 60s default, still bounded by a tightened ceiling.

- MIGRATION.md's 300000ms example collides with limits.wall_clock_s (300)
  and crawler.timeouts.batch_process (300.0), so an operator following it
  exactly still got a 504 at 300s. Say to raise those too.

- "60_000" sat in the "not a positive integer" parametrize list, but
  int("60_000") is 60000, so it was accepted, not refused. The test passed
  only because that value equals the default. Moved to its own accepted
  case and added coverage for the clamp fallback.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PVaRrydnAhwnnRYpe5an1U
---
 crawl4ai/async_configs.py     |  4 +++-
 deploy/docker/MIGRATION.md    | 10 ++++++++++
 tests/test_config_defaults.py | 34 +++++++++++++++++++++++++++++++++-
 3 files changed, 46 insertions(+), 2 deletions(-)

diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py
index 4e2037f4d..a5c2b6577 100644
--- a/crawl4ai/async_configs.py
+++ b/crawl4ai/async_configs.py
@@ -357,8 +357,10 @@ def _clamp_untrusted(type_name: str, params: dict) -> dict:
 
     def _cap_timeout(v):
         # 0 historically meant "no timeout"; treat as the cap, never unbounded.
+        # Malformed input is the one value an attacker gets for free, so it
+        # falls back to the 60s default rather than to a raised ceiling.
         if not isinstance(v, (int, float)) or v <= 0:
-            return ceiling
+            return min(_DEFAULT_MAX_TIMEOUT_MS, ceiling)
         return min(int(v), ceiling)
 
     if type_name == "CrawlerRunConfig":
diff --git a/deploy/docker/MIGRATION.md b/deploy/docker/MIGRATION.md
index c508e220f..5123c950f 100644
--- a/deploy/docker/MIGRATION.md
+++ b/deploy/docker/MIGRATION.md
@@ -189,6 +189,16 @@ a smaller value tightens it. A value that is not a positive integer is refused
 with a warning and the 60000ms default kept, so a typo cannot silently widen
 the bound.
 
+Raising this ceiling alone is not enough. Two other deadlines cut a crawl
+short first, and both are in `config.yml`:
+
+- `limits.wall_clock_s` (default `300`) — the per-crawl deadline; the request
+  gets a 504 at that point no matter what `page_timeout` says.
+- `crawler.timeouts.batch_process` (default `300.0`) — the batch crawl budget.
+
+So a 300000ms ceiling needs `wall_clock_s` and `batch_process` raised past 300
+too, or the extra timeout can never be reached.
+
 ### Error responses are generic
 
 5xx responses return `{"error": "Internal server error", "correlation_id": "…"}`.
diff --git a/tests/test_config_defaults.py b/tests/test_config_defaults.py
index ad86f92f0..72f98ed39 100644
--- a/tests/test_config_defaults.py
+++ b/tests/test_config_defaults.py
@@ -361,7 +361,7 @@ def test_env_can_tighten_the_ceiling(self, monkeypatch):
 
     # A typo must not silently widen a DoS bound, so the default is kept and
     # the operator is told rather than left to find out under load.
-    @pytest.mark.parametrize("value", ["", "abc", "0", "-1", "60_000", "1e5"])
+    @pytest.mark.parametrize("value", ["", "abc", "0", "-1", "1e5"])
     def test_a_non_positive_integer_keeps_the_default(self, monkeypatch, value):
         monkeypatch.setenv("CRAWL4AI_MAX_TIMEOUT_MS", value)
 
@@ -373,6 +373,38 @@ def test_a_non_positive_integer_keeps_the_default(self, monkeypatch, value):
 
         assert config.page_timeout == 60_000
 
+    def test_an_underscored_integer_is_accepted(self, monkeypatch):
+        # int("60_000") == 60000 in Python, so this is a valid ceiling, not a typo.
+        monkeypatch.setenv("CRAWL4AI_MAX_TIMEOUT_MS", "120_000")
+
+        config = CrawlerRunConfig.load(
+            {"page_timeout": 500_000}, provenance=Provenance.UNTRUSTED
+        )
+
+        assert config.page_timeout == 120_000
+
+    def test_malformed_timeout_falls_back_to_the_default_not_the_ceiling(
+        self, monkeypatch
+    ):
+        # The one field an untrusted caller gets for free must not inherit a
+        # raised ceiling just by being junk.
+        monkeypatch.setenv("CRAWL4AI_MAX_TIMEOUT_MS", "300000")
+
+        config = CrawlerRunConfig.load(
+            {"page_timeout": 0}, provenance=Provenance.UNTRUSTED
+        )
+
+        assert config.page_timeout == 60_000
+
+    def test_malformed_timeout_respects_a_tightened_ceiling(self, monkeypatch):
+        monkeypatch.setenv("CRAWL4AI_MAX_TIMEOUT_MS", "5000")
+
+        config = CrawlerRunConfig.load(
+            {"page_timeout": "abc"}, provenance=Provenance.UNTRUSTED
+        )
+
+        assert config.page_timeout == 5_000
+
     @pytest.mark.parametrize("value", ["abc", "0", "-1"])
     def test_a_bad_value_warns(self, monkeypatch, value):
         monkeypatch.setenv("CRAWL4AI_MAX_TIMEOUT_MS", value)

From e5e101183ac7b62784f8025df77743e2d4263127 Mon Sep 17 00:00:00 2001
From: ntohidi <nasrin@kidocode.com>
Date: Mon, 14 Sep 2026 18:05:08 +0200
Subject: [PATCH 19/26] fix(deep-crawl): keep the shallowest depth when
 de-duplicating best-first links

The first cut of the best-first de-duplication marked a URL seen the moment it
was discovered, which froze it at the depth it was *first* found rather than the
shallowest depth reachable. Best-first does not visit levels in order, so a URL
can be discovered at depth 3 through a high-scoring branch and only later at
depth 2 through a slower one. Freezing depth 3 means link_discovery bails at
4 > max_depth and that whole subtree is silently lost.

The duplicate enqueue was doing double duty: it was also the depth relaxation.
So de-duplicate on depth instead of on identity. A URL already queued at an
equal or shallower depth is skipped, which is the waste the issue reported; a
strictly shallower re-discovery still re-queues, and the existing dequeue guard
drops the stale deeper copy.

This reverts the `visited`-at-discovery change, the removed dequeue guard and
the resume-state compatibility shim, none of which are needed now: `visited`
keeps its original crawled-only meaning and the whole fix is one guard in
link_discovery.

Found in review of #2265.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Utx8BTDqXTscjquLV7vgLq
---
 crawl4ai/deep_crawling/bff_strategy.py        | 18 ++++++----
 .../test_deep_crawl_perf_2242.py              | 36 ++++++++++++++++++-
 2 files changed, 47 insertions(+), 7 deletions(-)

diff --git a/crawl4ai/deep_crawling/bff_strategy.py b/crawl4ai/deep_crawling/bff_strategy.py
index 4bc247c0d..abf53f054 100644
--- a/crawl4ai/deep_crawling/bff_strategy.py
+++ b/crawl4ai/deep_crawling/bff_strategy.py
@@ -179,11 +179,17 @@ async def link_discovery(
             base_url = normalize_url_for_deep_crawl(url, source_url)
             if base_url in visited:
                 continue
+            # Already queued at an equal or shallower depth: a second entry would
+            # be scored and dropped for nothing. A strictly shallower find still
+            # has to re-queue, so the URL is crawled at its true distance from the
+            # start and its own children stay inside max_depth.
+            queued_depth = depths.get(base_url)
+            if queued_depth is not None and queued_depth <= new_depth:
+                continue
             if not await self.can_process_url(base_url, new_depth):
                 self.stats.urls_skipped += 1
                 continue
                 
-            visited.add(base_url)
             valid_links.append(base_url)
             
         # Record the new depths and add to next_links
@@ -215,10 +221,6 @@ async def _arun_best_first(
             self._pages_crawled = self._resume_state.get("pages_crawled", 0)
             # Restore queue from saved items
             queue_items = self._resume_state.get("queue_items", [])
-            # Older checkpoints could hold a URL twice and recorded only crawled
-            # URLs in "visited"; drop the repeats and treat everything queued as seen.
-            queue_items = list({item["url"]: item for item in queue_items}.values())
-            visited.update(item["url"] for item in queue_items)
             for item in queue_items:
                 await queue.put((item["score"], item["depth"], item["url"], item["parent_url"]))
             # Initialize shadow list if callback is set
@@ -231,7 +233,7 @@ async def _arun_best_first(
             # Original initialization
             initial_score = self.url_scorer.score(start_url) if self.url_scorer else 0
             await queue.put((-initial_score, 0, start_url, None))
-            visited: Set[str] = {start_url}
+            visited: Set[str] = set()
             depths: Dict[str, int] = {start_url: 0}
             # Initialize shadow list if callback is set
             if self._on_state_change:
@@ -268,6 +270,10 @@ async def _arun_best_first(
                         self._queue_shadow.remove(item)
                     except ValueError:
                         pass  # Item may have been removed already
+                score, depth, url, parent_url = item
+                if url in visited:
+                    continue
+                visited.add(url)
                 batch.append(item)
 
             if not batch:
diff --git a/tests/deep_crawling/test_deep_crawl_perf_2242.py b/tests/deep_crawling/test_deep_crawl_perf_2242.py
index f27826767..f53712931 100644
--- a/tests/deep_crawling/test_deep_crawl_perf_2242.py
+++ b/tests/deep_crawling/test_deep_crawl_perf_2242.py
@@ -121,7 +121,7 @@ async def best_of(n: int, rounds: int = 3) -> float:
 
 @pytest.mark.asyncio
 async def test_best_first_queues_a_shared_url_once():
-    """Two parents linking to the same page must enqueue it a single time."""
+    """Two parents at the same depth must enqueue a shared child only once."""
     link_map = {
         ROOT: [f"{ROOT}/a", f"{ROOT}/b"],
         f"{ROOT}/a": [f"{ROOT}/shared"],
@@ -145,3 +145,37 @@ async def on_state_change(state: Dict[str, Any]):
     for queue in snapshots:
         assert not [u for u, n in Counter(queue).items() if n > 1], f"duplicate in queue: {queue}"
     assert sorted(crawled) == sorted(link_map)
+
+
+@pytest.mark.asyncio
+async def test_best_first_keeps_the_shallowest_depth_for_a_shared_url():
+    """De-duplicating must not freeze a URL at the depth it was first seen.
+
+    ROOT -> A, B, F0..F11;  A -> C;  C -> X;  B -> X;  X -> LEAF
+
+    The scores crawl C before B, so X is discovered first at depth 3 and only
+    later at depth 2. The twelve fillers push B out of the first BATCH_SIZE
+    pull, which is what lets C run ahead of B - without them both land in one
+    batch and the ordering that exposes this never happens.
+
+    If the shallower re-discovery is dropped as a duplicate, X stays at depth 3
+    and link_discovery bails at 4 > max_depth, losing LEAF.
+    """
+    a, b, c, x, leaf = (f"{ROOT}/{p}" for p in ("a", "b", "c", "x", "leaf"))
+    fillers = [f"{ROOT}/f{i}" for i in range(12)]
+    link_map = {ROOT: [a, b] + fillers, a: [c], c: [x], b: [x], x: [leaf]}
+    scores = {a: 0.9, b: 0.1, c: 0.95, x: 0.5, leaf: 0.5, **{f: 0.8 for f in fillers}}
+
+    class DictScorer:
+        def score(self, url):
+            return scores.get(url, 0.0)
+
+    strategy = BestFirstCrawlingStrategy(max_depth=3, max_pages=100, url_scorer=DictScorer())
+    crawled = {
+        r.url: r.metadata["depth"]
+        async for r in strategy._arun_stream(ROOT, make_crawler(link_map), make_config(stream=True))
+    }
+
+    assert crawled[x] == 2, "X must be crawled at its shallowest depth, not the first one seen"
+    assert leaf in crawled, "LEAF is within max_depth once X sits at depth 2"
+    assert crawled == {ROOT: 0, a: 1, b: 1, c: 2, x: 2, leaf: 3, **{f: 1 for f in fillers}}

From 142c1f34f9242b638b78978161b9e5bc547d2f98 Mon Sep 17 00:00:00 2001
From: nightcityblade <nightcityblade@gmail.com>
Date: Tue, 15 Sep 2026 23:03:50 +0800
Subject: [PATCH 20/26] docs: list 0.9.x as supported

---
 SECURITY.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/SECURITY.md b/SECURITY.md
index 3245b9f57..3f4336321 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -4,7 +4,8 @@
 
 | Version | Supported          |
 | ------- | ------------------ |
-| 0.8.x   | :white_check_mark: |
+| 0.9.x   | :white_check_mark: |
+| 0.8.x   | :x: (upgrade recommended) |
 | 0.7.x   | :x: (upgrade recommended) |
 | < 0.7   | :x:                |
 

From be00ff2db649c72ad776661cdec91cdfde7ea872 Mon Sep 17 00:00:00 2001
From: Krzysztof Olipra <krzysztof.olipra@elevenlabs.io>
Date: Fri, 18 Sep 2026 16:25:13 -0500
Subject: [PATCH 21/26] feat: robots parser test dynamic port

---
 tests/general/test_robot_parser.py    | 9 +++++----
 tests/unit/test_robots_query_rules.py | 8 ++------
 2 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/tests/general/test_robot_parser.py b/tests/general/test_robot_parser.py
index 6ed20fbc9..6690339b3 100644
--- a/tests/general/test_robot_parser.py
+++ b/tests/general/test_robot_parser.py
@@ -134,14 +134,15 @@ async def query_robots(request):
             query_app.router.add_get('/robots.txt', query_robots)
             query_runner = web.AppRunner(query_app)
             await query_runner.setup()
-            query_site = web.TCPSite(query_runner, 'localhost', 8081)
+            query_site = web.TCPSite(query_runner, '127.0.0.1', 0)
             await query_site.start()
-            return query_runner
+            query_port = query_runner.addresses[0][1]
+            return query_runner, query_port
 
-        query_runner = await start_query_server()
+        query_runner, query_port = await start_query_server()
         try:
             print("\n4b. Testing query-string robots.txt rules...")
-            query_base = "http://localhost:8081"
+            query_base = f"http://127.0.0.1:{query_port}"
 
             result = await parser.can_fetch(f"{query_base}/", "bot")
             print(f"Plain root (/): {'allowed' if result else 'denied'}")
diff --git a/tests/unit/test_robots_query_rules.py b/tests/unit/test_robots_query_rules.py
index 90b1c61bf..e88e57bb2 100644
--- a/tests/unit/test_robots_query_rules.py
+++ b/tests/unit/test_robots_query_rules.py
@@ -43,14 +43,10 @@ def test_preserve_bare_query_line_rewriting(line, expected):
 def test_preserve_bare_query_keeps_document_structure():
     """Untouched lines, blank lines and ordering survive verbatim.
 
-    The rewrite is splitlines()-based, so a trailing newline is not preserved.
-    That is harmless: the only caller re-splits the result immediately.
+    Compared line by line, since the only caller re-splits the result
+    immediately; whether a trailing newline survives is deliberately unpinned.
     """
     source = "User-agent: *\nDisallow: /private/\n\nDisallow: /*?\nAllow: /public/\n"
-    assert _preserve_bare_query(source) == (
-        "User-agent: *\nDisallow: /private/\n\nDisallow: /*?*\nAllow: /public/"
-    )
-    # What the caller actually consumes is unaffected by the missing newline.
     assert _preserve_bare_query(source).splitlines() == [
         "User-agent: *", "Disallow: /private/", "", "Disallow: /*?*", "Allow: /public/",
     ]

From 15f4b29b8ba2980d5e0cd6ba3d23cd93dd608043 Mon Sep 17 00:00:00 2001
From: ntohidi <nasrin@kidocode.com>
Date: Tue, 22 Sep 2026 10:58:39 +0200
Subject: [PATCH 22/26] fix: default max_pages_before_recycle in DEFAULT_CONFIG

load_config() deep-merges config.yml over DEFAULT_CONFIG, so a mounted
custom config.yml without the new key left RECYCLE_PAGES at 0 and kept
the #2231 bug after upgrade. An explicit 0 in config.yml still disables
recycling.
---
 deploy/docker/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deploy/docker/utils.py b/deploy/docker/utils.py
index 8f8ecb180..0fc2e69c0 100644
--- a/deploy/docker/utils.py
+++ b/deploy/docker/utils.py
@@ -64,7 +64,7 @@ class FilterType(str, Enum):
         "memory_threshold_percent": 95.0,
         "rate_limiter": {"enabled": True, "base_delay": [1.0, 2.0]},
         "timeouts": {"stream_init": 30.0, "batch_process": 300.0},
-        "pool": {"max_pages": 40, "idle_ttl_sec": 300},
+        "pool": {"max_pages": 40, "idle_ttl_sec": 300, "max_pages_before_recycle": 200},
         "browser": {
             "kwargs": {"headless": True, "text_mode": True},
             "extra_args": [

From 6218bb815c520bf6b0489ffb0717bdc76070f7f6 Mon Sep 17 00:00:00 2001
From: Nasrin <nasrin@kidocode.com>
Date: Tue, 22 Sep 2026 11:20:27 +0200
Subject: [PATCH 23/26] fix(robots): don't patch robotparser on Python 3.14+
 (#2278)

* fix(robots): don't patch robotparser on Python 3.14+

The wildcard monkey patch in utils.py overrides RuleLine.applies_to
unconditionally. Python 3.14 rewrote urllib.robotparser with native
wildcard, '$' and RFC 9309 longest-match support, where applies_to
returns the match *length* used to rank competing rules. The patch
returns a bool, so every wildcard rule collapses to the lowest
priority and Allow: overrides stop working:

    User-agent: *
    Disallow: /
    Allow: /public/*.html

denied /public/a.html on 3.14. Gate the patch to Python < 3.14, where
robotparser has no wildcard support and still needs it.

Follow-up to #2229, which fixed the 'Disallow: /*?' half of #2225.

Refs #2225

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* fix(robots): don't rewrite bare-'?' rules on Python 3.14+ either

Review follow-up on the previous commit, which gated the RuleLine
monkey patch but left _preserve_bare_query running on every Python.

On 3.14 that rewrite is not just unnecessary, it is wrong. The stdlib
ranks rules by match length, and '/*?*' matches to end of string, so
it outranks a narrower competing Allow:

    User-agent: *
    Allow: /*?q=
    Disallow: /*?

/search?q=1 is allowed by the stdlib and denied after the rewrite, so
Crawl4AI skipped pages robots.txt permits. Gate the call with the same
sys.version_info < (3, 14) as the patch, and say so in the docstring,
which claimed the two forms were always equivalent.

Also move the RuleLine import inside the branch that uses it and drop
a duplicate 'import re'.

Refs #2225

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 crawl4ai/utils.py                     | 68 ++++++++++++++----------
 tests/unit/test_robots_query_rules.py | 75 +++++++++++++++++++++++++++
 2 files changed, 117 insertions(+), 26 deletions(-)

diff --git a/crawl4ai/utils.py b/crawl4ai/utils.py
index f4cb7a62c..d6f9118c8 100644
--- a/crawl4ai/utils.py
+++ b/crawl4ai/utils.py
@@ -50,27 +50,34 @@
 import inspect
 
 
-# Monkey patch to fix wildcard handling in urllib.robotparser
-from urllib.robotparser import RuleLine
-import re
-
-original_applies_to = RuleLine.applies_to
-
-def patched_applies_to(self, filename):
-   # Handle wildcards in paths
-   if '*' in self.path or '%2A' in self.path or self.path in ("*", "%2A"):
-       pattern = self.path.replace('%2A', '*')
-       pattern = re.escape(pattern).replace('\\*', '.*')
-       pattern = '^' + pattern
-       if pattern.endswith('\\$'):
-           pattern = pattern[:-2] + '$'
-       try:
-           return bool(re.match(pattern, filename))
-       except re.error:
-           return original_applies_to(self, filename)
-   return original_applies_to(self, filename)
-
-RuleLine.applies_to = patched_applies_to
+# Monkey patch to fix wildcard handling in urllib.robotparser.
+# Python 3.14 rewrote robotparser with native wildcard, '$' and RFC 9309
+# longest-match support, and its applies_to returns a match *length* used to
+# rank rules. Patching it there returns a bool, which collapses every wildcard
+# rule to the lowest priority and breaks Allow: overrides -- so only patch the
+# older implementation, which has no wildcard support at all.
+import sys
+
+if sys.version_info < (3, 14):
+    from urllib.robotparser import RuleLine
+
+    original_applies_to = RuleLine.applies_to
+
+    def patched_applies_to(self, filename):
+       # Handle wildcards in paths
+       if '*' in self.path or '%2A' in self.path or self.path in ("*", "%2A"):
+           pattern = self.path.replace('%2A', '*')
+           pattern = re.escape(pattern).replace('\\*', '.*')
+           pattern = '^' + pattern
+           if pattern.endswith('\\$'):
+               pattern = pattern[:-2] + '$'
+           try:
+               return bool(re.match(pattern, filename))
+           except re.error:
+               return original_applies_to(self, filename)
+       return original_applies_to(self, filename)
+
+    RuleLine.applies_to = patched_applies_to
 # Monkey patch ends
 
 def chunk_documents(
@@ -252,8 +259,14 @@ def needs_update(self):
 def _preserve_bare_query(rules_text: str) -> str:
     """Append '*' to Allow/Disallow values ending in a bare '?' (e.g. '/*?').
 
-    '/*?' and '/*?*' allow/deny exactly the same URLs; the rewrite only
-    survives parsers that would otherwise drop the trailing '?'.
+    Only correct below Python 3.14, and only called there. Those parsers drop a
+    trailing '?' when normalizing the rule path, collapsing '/*?' to '/*' and
+    blocking the whole site; they also take the first matching rule, so making a
+    rule longer cannot change which one wins.
+
+    From 3.14 the stdlib keeps the '?' and ranks rules by match length, and the
+    rewrite becomes actively wrong: '/*?*' matches to end of string, so it
+    outranks a competing 'Allow: /*?q=' that the raw '/*?' would have lost to.
     """
     fixed = []
     for raw_line in rules_text.splitlines():
@@ -372,9 +385,12 @@ async def can_fetch(self, url: str, user_agent: str = "*") -> bool:
 
         # Create parser for this check
         parser = RobotFileParser()
-        # Old Pythons drop a trailing '?' from rules, so '/*?' becomes
-        # '/*' and blocks the whole site. '/*?*' matches the same URLs.
-        parser.parse(_preserve_bare_query(rules).splitlines())
+        # Below 3.14, rewrite rules ending in a bare '?' so they survive path
+        # normalization. From 3.14 the stdlib handles them, and the rewrite
+        # would skew its longest-match ranking -- see _preserve_bare_query.
+        if sys.version_info < (3, 14):
+            rules = _preserve_bare_query(rules)
+        parser.parse(rules.splitlines())
         
         # If parser can't read rules, allow access
         if not parser.mtime():
diff --git a/tests/unit/test_robots_query_rules.py b/tests/unit/test_robots_query_rules.py
index e88e57bb2..67ce1bc4f 100644
--- a/tests/unit/test_robots_query_rules.py
+++ b/tests/unit/test_robots_query_rules.py
@@ -6,6 +6,7 @@
 """
 
 import asyncio
+import sys
 
 import pytest
 
@@ -86,3 +87,77 @@ def test_ordinary_rules_still_apply(tmp_path):
     rules = "User-agent: *\nDisallow: /private/\nAllow: /public/\n"
     assert _can_fetch(rules, "/public/page", tmp_path) is True
     assert _can_fetch(rules, "/private/secret", tmp_path) is False
+
+
+def test_wildcard_patch_is_scoped_to_old_pythons():
+    """The patch must be installed exactly where it is needed, and nowhere else.
+
+    Python 3.14 supports wildcards natively and ranks rules by match length; the
+    patch returns a bool, which would flatten that ranking (see crawl4ai.utils).
+    """
+    from urllib.robotparser import RuleLine
+
+    is_patched = RuleLine.applies_to.__name__ == "patched_applies_to"
+    assert is_patched == (sys.version_info < (3, 14))
+
+
+@pytest.mark.parametrize(
+    "path, expected",
+    [("/a.php", False), ("/deep/a.php", False), ("/a.html", True)],
+)
+def test_wildcard_rules_work_on_every_python(path, expected, tmp_path):
+    """Wildcard support is the whole point of the patch - it must survive the gate."""
+    rules = "User-agent: *\nDisallow: /*.php\n"
+    assert _can_fetch(rules, path, tmp_path) is expected
+
+
+@pytest.mark.skipif(
+    sys.version_info < (3, 14),
+    reason="longest-match Allow precedence only exists in the 3.14+ stdlib parser",
+)
+@pytest.mark.parametrize(
+    "path, expected",
+    [("/public/a.html", True), ("/private/a.html", False), ("/public/a.txt", False)],
+)
+def test_allow_overrides_broad_disallow(path, expected, tmp_path):
+    """Regression: the old unconditional patch denied /public/a.html on 3.14."""
+    rules = "User-agent: *\nDisallow: /\nAllow: /public/*.html\n"
+    assert _can_fetch(rules, path, tmp_path) is expected
+
+
+# A narrower Allow: competing with the bare-'?' Disallow:. On 3.14 the stdlib
+# ranks rules by match length, and rewriting '/*?' to '/*?*' makes the Disallow
+# match to end of string -- so the rewrite would beat the Allow: and deny a URL
+# robots.txt permits. Below 3.14 the first matching rule wins regardless of
+# length, so the rewrite is safe there and the Allow: still loses to nothing.
+COMPETING_RULES = "User-agent: *\nAllow: /*?q=\nDisallow: /*?\n"
+
+
+@pytest.mark.skipif(
+    sys.version_info < (3, 14),
+    reason="longest-match Allow precedence only exists in the 3.14+ stdlib parser",
+)
+@pytest.mark.parametrize(
+    "path, expected",
+    [("/search?q=1", True), ("/search?x=1", False), ("/search", True)],
+)
+def test_query_allow_outranks_bare_query_disallow(path, expected, tmp_path):
+    """Regression: _preserve_bare_query must not run on 3.14.
+
+    '/*?*' matches longer than '/*?q=', so the rewrite would flip /search?q=1
+    from allowed to denied.
+    """
+    assert _can_fetch(COMPETING_RULES, path, tmp_path) is expected
+
+
+@pytest.mark.skipif(
+    sys.version_info >= (3, 14),
+    reason="pre-3.14 parsers take the first matching rule, not the longest",
+)
+@pytest.mark.parametrize(
+    "path, expected",
+    [("/search?q=1", True), ("/search?x=1", False), ("/search", True)],
+)
+def test_query_allow_still_wins_below_py314(path, expected, tmp_path):
+    """The same rules must give the same answers below 3.14, via the rewrite."""
+    assert _can_fetch(COMPETING_RULES, path, tmp_path) is expected

From aa941f9ceb4be172cfe0524b1cd36f1619f43c57 Mon Sep 17 00:00:00 2001
From: ntohidi <nasrin@kidocode.com>
Date: Tue, 22 Sep 2026 11:50:28 +0200
Subject: [PATCH 24/26] chore(ci): drop the dead Google Apps Script stargazer
 step

The Apps Script endpoint that logged stargazers to a Google Sheet has
been returning 403 on every star since at least #2255, and the sheet is
no longer used. #2263 made the step non-blocking, but that just leaves a
curl that fails on every single star forever.

Remove the step entirely. The Discord stargazer notification is
unaffected. The GOOGLE_SCRIPT_ENDPOINT repo secret is now unused and can
be deleted.

Refs #2255
---
 .github/workflows/main.yml | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index cb324a145..98e63ba3d 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -16,13 +16,6 @@ jobs:
   notify-discord:
     runs-on: ubuntu-latest
     steps:
-      - name: Send to Google Apps Script (Stars only)
-        if: github.event_name == 'watch'
-        continue-on-error: true
-        run: |
-          curl -fSs -X POST "${{ secrets.GOOGLE_SCRIPT_ENDPOINT }}" \
-            -H 'Content-Type: application/json' \
-            -d '{"url":"${{ github.event.sender.html_url }}"}'
       - name: Set webhook based on event type
         id: set-webhook
         run: |

From 06c8acfbefc4e24270a28df6e34e4b66f6238e84 Mon Sep 17 00:00:00 2001
From: Soham Kukreti <kukretisoham@gmail.com>
Date: Wed, 23 Sep 2026 17:04:44 +0530
Subject: [PATCH 25/26] docs(security): list fixed issues and features per
 release through v0.9.3

SECURITY.md stopped at v0.8.1; add per-release tables for v0.8.5 to v0.9.3, tag features by version, and note that fixes are not backported.
---
 SECURITY.md | 155 +++++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 134 insertions(+), 21 deletions(-)

diff --git a/SECURITY.md b/SECURITY.md
index 3f4336321..6cbcaae00 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -9,6 +9,8 @@
 | 0.7.x   | :x: (upgrade recommended) |
 | < 0.7   | :x:                |
 
+Fixes are not backported to earlier lines. The 0.9.3 advisories listed below are not patched in any 0.8.x release; if you run 0.8.x, upgrade.
+
 ## Reporting a Vulnerability
 
 We take security vulnerabilities seriously. If you discover a security issue, please report it responsibly.
@@ -68,9 +70,9 @@ If you're running the Crawl4AI Docker API in production:
    export SECRET_KEY="your-secure-random-key-here"
    ```
 
-2. **Hooks are Disabled by Default** (v0.8.0+)
-   - Only enable if you trust all API users
-   - Set `CRAWL4AI_HOOKS_ENABLED=true` only when necessary
+2. **Hooks are Declarative** (v0.9.0+)
+   - Request-supplied hook code is no longer accepted; only the fixed action set in `GET /hooks/info`
+   - On 0.8.x, hooks are disabled by default; set `CRAWL4AI_HOOKS_ENABLED=true` only if you trust all API users
 
 3. **Network Security**
    - Run behind a reverse proxy (nginx, traefik)
@@ -92,44 +94,155 @@ When using Crawl4AI as a Python library:
 
 ## Known Security Issues
 
-### Fixed in v0.8.0
+All issues below are fixed in the version named. Advisories with a GHSA id are published under [Security Advisories](https://github.com/unclecode/crawl4ai/security/advisories). Full detail for every release is in [CHANGELOG.md](CHANGELOG.md); reporter credits are in [SECURITY-CREDITS.md](SECURITY-CREDITS.md).
+
+### Fixed in v0.9.3 (2026-08-31)
+
+| ID | Severity | Component | Description | Fix |
+|----|----------|-----------|-------------|-----|
+| GHSA-xpp7-j28w-2gvx | HIGH | Library + Docker API | Arbitrary file write via `PDFContentScrapingStrategy` image-write fields in untrusted config bodies (CWE-22) | `save_images_locally` / `image_save_dir` filtered at the trust boundary; `extract_images` forced off for untrusted bodies |
+| GHSA-q5rj-45vw-vp2g | HIGH | Library | SSRF via PDF download redirects; DNS rebinding (CWE-918) | Redirects resolved manually with a per-hop destination check (max 5 hops); peer IP of the read response validated |
+| GHSA-v2rm-hvrj-2x9q | MEDIUM | Library | Denial of service via unbounded PDF size and page count (CWE-400) | `max_pdf_bytes` (100 MiB) and `max_pdf_pages` (2000) caps; untrusted bodies cannot raise them |
+| GHSA-7g3g-vhm6-79f3 | MEDIUM | Library | XSS via unescaped PDF paragraph text in `cleaned_html` (CWE-79) | Paragraph text escaped like every other sink |
+| GHSA-m446-hp3q-qfxp | HIGH | Docker Playground | DOM-based XSS via `innerHTML` round-trip in the result viewer, leading to API token theft (CWE-79) | Round-trip removed; highlight.js renders from `textContent` |
+
+### Fixed in v0.9.2 (2026-07-15)
+
+| ID | Severity | Component | Description | Fix |
+|----|----------|-----------|-------------|-----|
+| - | LOW | Docker API | `/monitor/ws` WebSocket returned 500 under JWT auth because the router-level token dependency cannot run on WebSocket scopes | Auth enforced by `AuthGateMiddleware`; admin routes keep `require_admin` |
+
+### Fixed in v0.9.1 (2026-07-08)
+
+| ID | Severity | Component | Description | Fix |
+|----|----------|-----------|-------------|-----|
+| - | MEDIUM | Docker API | Rate-limit Redis storage connected without the configured password | Rate limiter authenticates to Redis |
+
+### Fixed in v0.9.0 (2026-06-18)
+
+Secure-by-default rework of the Docker API server. The pip library is unchanged. See `deploy/docker/MIGRATION.md`.
+
+| ID | Severity | Component | Description | Fix |
+|----|----------|-----------|-------------|-----|
+| - | CRITICAL | Docker API | Unauthenticated API served on `0.0.0.0` by default (CWE-306) | Auth on by default; loopback bind unless `CRAWL4AI_API_TOKEN` is set |
+| - | CRITICAL | Docker API | Request-supplied hook code executed on the server (CWE-94) | `hooks.code` removed; fixed set of declarative hook actions |
+| - | HIGH | Docker API | Chromium launch-arg injection via request-supplied `browser_config.extra_args` (CWE-94) | `extra_args` rejected at the network boundary |
+| - | HIGH | Docker API | Path traversal to file write via download sinks (CWE-22) | basename + realpath + `O_NOFOLLOW` confinement |
+| - | HIGH | Docker API | SSRF on `/crawl/stream` and `/crawl` with `stream=true` (CWE-918) | Destination validation added; HTTP 400 on disallowed targets |
+| - | HIGH | Docker API | Request body could drive browser internals (`js_code`, `proxy_config`, `cdp_url`, `user_data_dir`, `cookies`, `headers`, `init_scripts`, `base_url`, ...) | Request trust boundary: scalar declarative options only, HTTP 400 otherwise |
+| - | MEDIUM | Docker API | Weak JWT, unscoped monitor actions, permissive CORS, TLS verification off, unauthenticated Redis, unbounded job queue, verbose 5xx, unvalidated webhook headers | Hardened defaults for each; see CHANGELOG 0.9.0 |
+
+### Fixed in v0.8.9 (2026-06-04)
 
-| ID | Severity | Description | Fix |
-|----|----------|-------------|-----|
-| CVE-pending-1 | CRITICAL | RCE via hooks `__import__` | Removed from allowed builtins |
-| CVE-pending-2 | HIGH | LFI via `file://` URLs | URL scheme validation added |
+| ID | Severity | Component | Description | Fix |
+|----|----------|-----------|-------------|-----|
+| - | HIGH | Docker API | SSRF via `proxy_config.server`, deprecated `proxy`, or `--proxy-server` / `--host-resolver-rules` in `extra_args` (CWE-918) | All proxy destinations validated with the global-routability check; proxy/DNS flags stripped from `extra_args` |
 
-### Fixed in v0.8.1
+### Fixed in v0.8.8 (2026-06-04)
 
-| ID | Severity | Description | Fix |
-|----|----------|-------------|-----|
-| CVE-pending-3 | CRITICAL | RCE via deserialization + `eval()` in `/crawl` endpoint | Allowlisted deserializable types; AST-validated computed field expressions |
+| ID | Severity | Component | Description | Fix |
+|----|----------|-----------|-------------|-----|
+| - | HIGH | Docker API | SSRF filter bypass via IPv6 transition forms (NAT64, 6to4, IPv4-mapped, `::`) (CWE-918) | Reject any resolved address that is not globally routable; errors no longer echo the address |
+| - | HIGH | Docker API | Symlink / TOCTOU bypass of the `output_path` directory restriction on `/screenshot` and `/pdf` (CWE-59/22) | Resolve symlinks, re-check containment, write with `O_NOFOLLOW` |
+| - | HIGH | Docker API | LLM credential exfiltration via request-supplied `base_url` on `/md`, `/llm`, `/llm/job`; `env:` token resolved protected variables (CWE-522/200) | `base_url` ignored; `LLMConfig` refuses protected env vars |
+| - | LOW | Docker API | CRLF log injection (CWE-117); unvalidated webhook request headers (CWE-93) | Control characters stripped from log records; webhook header name/value validation |
 
-See [Security Advisory](https://github.com/unclecode/crawl4ai/security/advisories) for details.
+### Fixed in v0.8.7 (2026-06-01)
+
+| ID | Severity | Component | Description | Fix |
+|----|----------|-----------|-------------|-----|
+| - | CRITICAL (9.8) | Docker API | AST sandbox escape via `gi_frame.f_back` in computed-field `eval()` leading to pre-auth RCE (CWE-94/913) | `eval()` removed from computed fields; `_safe_eval_expression` deleted |
+| - | CRITICAL (9.8) | Docker API | Hook sandbox escape: injected modules carried full `__builtins__` (CWE-94) | Injected builtins stripped; dangerous allowlist entries removed |
+| - | CRITICAL (9.8) | Docker API | Hardcoded default JWT secret `"mysecret"` allowed token forgery (CWE-798) | Default removed; weak secrets rejected; ephemeral key generated if none set |
+| - | HIGH (9.1) | Docker API | Arbitrary file write via `output_path` on `/screenshot` and `/pdf` (CWE-22) | Writes restricted to `CRAWL4AI_OUTPUT_DIR`; `..` rejected |
+| - | HIGH (8.6) | Docker API | SSRF via webhook URL on `/crawl/job` and `/llm/job` (CWE-918) | Blocklist; `follow_redirects=False` |
+| - | HIGH (8.6) | Docker API | SSRF via `/crawl`, `/md`, `/llm`; IPv6-mapped IPv4 bypass (CWE-918) | Destination validation on all entry points; IPv6-mapped IPv4 normalized |
+| - | HIGH (8.1) | Docker API | Arbitrary JavaScript execution via `/execute_js` (CWE-94) | Disabled by default (`CRAWL4AI_EXECUTE_JS_ENABLED`); `--disable-web-security` removed from default args |
+| - | MEDIUM (6.5) | Docker API | `/monitor/*` routes, including destructive actions, unauthenticated (CWE-306) | `token_dep` on the router; explicit check on the WebSocket |
+| - | MEDIUM (6.1) | Docker API | Stored XSS in the monitor dashboard via `innerHTML` (CWE-79) | Server-side `html.escape()`; client-side `escapeHtml()` |
+| - | MEDIUM | Docker API | `eval()` in `/config/dump` | Replaced by Pydantic-validated JSON |
+
+### Fixed in v0.8.6 (2026-03-24)
+
+| ID | Severity | Component | Description | Fix |
+|----|----------|-----------|-------------|-----|
+| - | CRITICAL | Library + Docker | PyPI supply-chain compromise of the `litellm` dependency | Dependency replaced with `unclecode-litellm` |
+
+### Fixed in v0.8.5 (2026-03-18)
+
+| ID | Severity | Component | Description | Fix |
+|----|----------|-----------|-------------|-----|
+| CVE-2025-49844 | CRITICAL (10.0) | Docker (Redis) | Lua use-after-free in bundled Redis | Redis upgraded to 7.2.7 |
+| - | MEDIUM | Library | XSS via `innerHTML` in iframe processing | `DOMParser` used instead (#1796) |
+| - | MEDIUM | Docker API | `/token` endpoint issued tokens without checking `api_token` | `api_token` required when configured (#1795) |
+
+### Fixed in v0.8.1 (2026-01-30)
+
+| ID | Severity | Component | Description | Fix |
+|----|----------|-----------|-------------|-----|
+| CVE-pending-3 | CRITICAL | Docker API | RCE via deserialization + `eval()` in `/crawl` endpoint | Allowlisted deserializable types; AST-validated computed field expressions (later removed entirely in 0.8.7) |
+
+### Fixed in v0.8.0 (2026-01-16)
+
+| ID | Severity | Component | Description | Fix |
+|----|----------|-----------|-------------|-----|
+| CVE-pending-1 | CRITICAL | Docker API | RCE via hooks `__import__` | Removed from allowed builtins; hooks disabled by default |
+| CVE-pending-2 | HIGH | Docker API | LFI via `file://` URLs | URL scheme validation added |
 
 ## Security Features
 
+### v0.9.3+
+
+- **PDF egress policy**: PDF downloads validate every redirect hop and the peer IP actually read, so the Docker SSRF policy also covers the out-of-browser `requests` path
+- **PDF resource caps**: `max_pdf_bytes` (100 MiB) and `max_pdf_pages` (2000), not raisable from an untrusted request body
+- **PDF image-write fields filtered** from untrusted bodies; `extract_images` forced off
+- **Egress proxy upstream chaining**: the pinning egress proxy honours `HTTP_PROXY` / `HTTPS_PROXY` and refuses non-http proxy schemes
+- **Per-crawl wall clock**: Docker config ships `limits.wall_clock_s` of 300 seconds
+
+### v0.9.0+ (Docker API server, secure by default)
+
+- **Authentication on by default**: loopback bind unless `CRAWL4AI_API_TOKEN` is set; `Authorization: Bearer <token>` required on every request except `GET /health`
+- **Request trust boundary**: crawl request bodies carry scalar, declarative options only; browser internals and code fields are rejected with HTTP 400; unknown fields dropped; timeouts, viewport and scroll counts clamped
+- **Declarative hooks**: `hooks.code` removed; fixed action set (`block_resources`, `add_cookies`, `set_headers`, `scroll_to_bottom`, `wait_for_timeout`)
+- **`extra_args` rejected** over the network, closing the Chromium launch-arg injection class
+- **Artifact ids instead of `output_path`**: `/screenshot` and `/pdf` return an `artifact_id`; fetch via authenticated `GET /artifacts/{id}` with TTL and quota
+- **Download path confinement**: basename + realpath + `O_NOFOLLOW`
+- **SSRF validation on every crawl path**, including `/crawl/stream`; internal targets need `CRAWL4AI_ALLOW_INTERNAL_URLS=true`
+- **Hardened JWT** (tokens from older versions are invalid; re-mint via `POST /token`); **admin scope** for `/monitor/actions/*` and `/monitor/stats/reset`
+- **CORS deny by default** (`security.cors_allow_origins`); **strict security headers**
+- **TLS verification on**; escape hatch `CRAWL4AI_ALLOW_INSECURE_TLS=true`
+- **Redis password-protected and loopback-only**; port no longer published
+- **Bounded job queue**: body size, per-crawl wall clock, queue size and per-principal concurrency capped
+- **Generic 5xx responses** with a correlation id; **webhook headers validated** (HTTP 422 on malformed or hop-by-hop headers)
+- **LLM provider by name only**: `base_url` removed; providers constrained by `config.llm.allowed_providers`
+
+### v0.8.7+ to v0.8.9+ (Docker API server)
+
+- **Global-routability SSRF check** on crawl targets, webhook URLs and proxy destinations, evaluated on IPv6 transition forms
+- **`/execute_js` disabled by default** (`CRAWL4AI_EXECUTE_JS_ENABLED`)
+- **No default JWT secret**; weak secrets rejected
+- **Monitor routes authenticated**; monitor dashboard output escaped
+- **`eval()` removed** from computed fields and `/config/dump`
+- **Protected env vars** cannot be resolved through the `LLMConfig` `env:` token form
+- **CRLF-safe logging**
+
 ### v0.8.1+
 
 - **Deserialization Allowlist**: Only known-safe types can be instantiated via API config
-- **Safe Expression Evaluation**: Computed fields use AST validation (no `__import__`, no dunder access)
 
 ### v0.8.0+
 
 - **URL Scheme Validation**: Blocks `file://`, `javascript:`, `data:` URLs on API
-- **Hooks Disabled by Default**: Opt-in via `CRAWL4AI_HOOKS_ENABLED=true`
+- **Hooks Disabled by Default**: Opt-in via `CRAWL4AI_HOOKS_ENABLED=true` (replaced by declarative hooks in 0.9.0)
 - **Restricted Hook Builtins**: No `__import__`, `eval`, `exec`, `open`
-- **JWT Authentication**: Optional but recommended for production
+- **JWT Authentication**: Optional (on by default since 0.9.0)
 - **Rate Limiting**: Configurable request limits
 - **Security Headers**: X-Frame-Options, CSP, HSTS when enabled
 
 ## Acknowledgments
 
-We thank the following security researchers for responsibly disclosing vulnerabilities:
-
-- **Alec M** — RCE via deserialization in `/crawl` endpoint (January 2026)
-- **[Neo by ProjectDiscovery](https://projectdiscovery.io/blog/introducing-neo)** — RCE and LFI vulnerabilities (December 2025)
+We thank the security researchers who responsibly disclosed the issues above. The full list, with the vulnerability each person reported and the release that fixed it, is maintained in [SECURITY-CREDITS.md](SECURITY-CREDITS.md).
 
 ---
 
-*Last updated: February 2026*
+*Last updated: September 2026*

From 133e1d92e37885dfccc03ea2e3687d06c98b7ceb Mon Sep 17 00:00:00 2001
From: ntohidi <nasrin@kidocode.com>
Date: Wed, 23 Sep 2026 14:09:09 +0200
Subject: [PATCH 26/26] docs: release notes, changelog, README, and security
 credits for v0.9.4

Bump the version to 0.9.4 and add the v0.9.4 fixes to SECURITY.md.
---
 CHANGELOG.md                |  57 +++++++++++++++
 Dockerfile                  |   2 +-
 README.md                   |  23 +++++-
 SECURITY-CREDITS.md         |   4 ++
 SECURITY.md                 |  14 ++++
 crawl4ai/__version__.py     |   2 +-
 docs/blog/release-v0.9.4.md | 140 ++++++++++++++++++++++++++++++++++++
 7 files changed, 238 insertions(+), 4 deletions(-)
 create mode 100644 docs/blog/release-v0.9.4.md

diff --git a/CHANGELOG.md b/CHANGELOG.md
index bee04104b..d79bef5c1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,63 @@ All notable changes to Crawl4AI will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.9.4] - 2026-09-23
+
+0.9.4 is a security release. It closes three coordinated-disclosure advisories: two SSRF paths that bypassed the Docker server's egress controls, and a trust-boundary bypass that let a non-admin API client read server environment variables. It also makes content pruning about 10x faster with the new lxml-native `PruningContentFilterLXML`, now the default, and ships the bug fixes that accumulated on `develop` since 0.9.3. There are no breaking changes. Users who self-host the Docker server should upgrade.
+
+### Security
+
+- **Blind SSRF via the robots.txt fetch (CWE-918, medium)**: `RobotsParser.can_fetch()` fetched `/robots.txt` on a bare `aiohttp` client that followed redirects and re-resolved the host, so `check_robots_txt` in an untrusted request body could make the Docker server reach internal, loopback, and cloud-metadata addresses. The fetch now goes through the server's pinning egress proxy, which checks every hop and dials the pinned IP. Credit: [arpe1618](https://github.com/arpe1618). (GHSA-f77g-77vp-r96v)
+- **SSRF with response disclosure via `link_preview_config` (CWE-918, high)**: the URL seeder fetched every link on a crawled page with its own `httpx` client, outside the egress controls, and returned each page's parsed `<head>` to the caller. The seeder's fetches now go through the same pinning egress proxy, and `LinkPreviewConfig` gets caps on `max_links`, `concurrency`, and `timeout` for untrusted bodies. Credit: Ibrahim AlJaafreh ([LinkedIn](https://www.linkedin.com/in/ibrahim-aljaafreh-glitch/)), Cystack RedTeam ([cystack.ps](https://cystack.ps)). (GHSA-wh5w-hmj3-vgg7)
+- **Untrusted-config gate bypass via dict-wrapper laundering (CWE-501, high)**: wrapping a forbidden typed object such as `LLMConfig` in `{"type": "dict", "value": {...}}` slipped it past `UNTRUSTED_ALLOWED_TYPES`, and `from_kwargs` then rebuilt it as trusted. A non-admin client could read any server environment variable, including LLM keys and `SECRET_KEY`. The unwrapped value is now re-checked under the untrusted gate, and `from_kwargs` carries the caller's provenance instead of defaulting to trusted. Credit: Adam Jordan ([adamyordan](https://github.com/adamyordan)). (GHSA-5w5p-vcv6-mm3f)
+
+The two SSRF fixes share one mechanism: the new `crawl4ai/egress_policy.py` holds a process-wide egress proxy URL for the library's own HTTP clients. The Docker server registers its existing `PinningProxy` there at boot. A plain library caller sets nothing and sees no change.
+
+All reporters are credited in `SECURITY-CREDITS.md`. GitHub Security Advisories accompany this release.
+
+### Added
+
+- `PruningContentFilterLXML`: an lxml-native pruning filter. It computes every per-node metric in one bottom-up pass instead of re-walking each subtree, so pruning is O(N) instead of super-linear. Output is byte-identical to `PruningContentFilter`. Measured pruning time: medium page 134 to 13 ms, 6000-card page 2200 to 260 ms. It is now the default for the Docker server's fit filter and the CLI pruning filter.
+- `CRAWL4AI_MAX_TIMEOUT_MS` sets the ceiling for `page_timeout`, `wait_for_timeout`, and `body_visibility_timeout` on untrusted configs. The default stays 60000 ms. (#2212, thanks @damusix; #2266)
+- Docker server: `crawler.pool.max_pages_before_recycle` (default 200) recycles a pooled browser context after it serves that many pages. A context gets slower with sustained use, and the idle janitor never fires on a busy server. Set it to 0 to disable. (#2232, issue #2231)
+
+### Deprecated
+
+- `PruningContentFilter` emits a `DeprecationWarning` on direct use. Switch to `PruningContentFilterLXML`, which takes the same arguments and gives the same output. Existing import paths keep working.
+
+### Fixed
+
+**Crawler and core**
+
+- Deep crawl: BFS no longer re-scans the whole level to match each result to its parent, and BestFirst no longer enqueues the same URL twice. De-duplication keeps the shallowest depth, so no subtree is lost. (#2265, issue #2242)
+- Tables: `rowspan` and `colspan` are expanded into a grid, and `<th>` row headers are kept instead of shifting the row left. Spans are clamped, so one cell cannot hang the parse. (#2261, issue #2258)
+- robots.txt: `Disallow: /*?` no longer blocks the whole site. (#2229, thanks @Nalhin)
+- robots.txt: the wildcard patch is skipped on Python 3.14+, where the standard library already supports wildcards and the patch broke `Allow:` precedence. (#2278)
+- Timeouts: malformed or non-positive timeout values fall back to the 60 s default instead of the configured ceiling. (#2266)
+- Chrome for Testing no longer crashes under `--headless=new` on macOS arm64. `OptimizationHints` is no longer disabled. (#2241, issue #2239, thanks @Zsanz3)
+
+**Docker server**
+
+- Playground: the Advanced Config panel is a JSON params editor. The old Python editor sent a `code` field that the untrusted boundary rejects. (#2262, issue #2260)
+- Playground: `md` and `llm` runs skip the `/config/dump` pre-flight, which failed on the legacy `code` field. (#2224, issue #2222)
+- The permanent browser is built with the egress-hardened default config, so its pool signature matches incoming requests. (#2237)
+
+**Documentation and CI**
+
+- `SECURITY.md` lists 0.9.x as supported. (#2269, thanks @nightcityblade)
+- The Discord stargazer notification no longer depends on a dead Google Apps Script step. (#2263, #2279)
+
+### Tests
+
+- `tests/unit/test_egress_policy.py` and `deploy/docker/tests/test_security_ssrf_seeder.py`: seeder and robots.txt fetches through the egress proxy.
+- `tests/unit/test_config_provenance.py`: direct, wrapped, and nested forbidden types refused under the untrusted gate.
+- `tests/async/test_content_filter_prune_lxml.py`: `PruningContentFilterLXML` output matches `PruningContentFilter`.
+- Coverage for the deep-crawl, table, robots.txt, timeout, and pool-recycle fixes.
+
+### Breaking Changes
+
+None.
+
 ## [0.9.3] - 2026-08-31
 
 0.9.3 is a security release. It closes five coordinated-disclosure advisories in the PDF processing path and the Docker Playground UI, and ships the 33 bug fixes that accumulated on `develop` since 0.9.2, most of them in the Docker server. There are no new features and no breaking changes. Users who accept untrusted URLs on the Docker server, or who open PDFs from sources they do not control, should upgrade.
diff --git a/Dockerfile b/Dockerfile
index 7ef9c2fb7..d8dee3626 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,7 +1,7 @@
 FROM python:3.12-slim-bookworm AS build
 
 # C4ai version
-ARG C4AI_VER=0.9.3
+ARG C4AI_VER=0.9.4
 ENV C4AI_VERSION=$C4AI_VER
 LABEL c4ai.version=$C4AI_VER
 
diff --git a/README.md b/README.md
index 310ac0240..cd3e70c50 100644
--- a/README.md
+++ b/README.md
@@ -37,9 +37,11 @@ Limited slots._
 
 Crawl4AI turns the web into clean, LLM ready Markdown for RAG, agents, and data pipelines. Fast, controllable, battle tested by a 50k+ star community.
 
-[✨ Check out latest update v0.9.3](#-recent-updates)
+[✨ Check out latest update v0.9.4](#-recent-updates)
 
-✨ **New in v0.9.3**: Security release. Closes five coordinated-disclosure advisories: arbitrary file write, SSRF, and denial of service in the PDF processing path, plus two XSS issues in the Docker Playground. Also ships 33 bug fixes across the Docker server, crawler, and PDF handling. No new features, no breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.3.md)
+✨ **New in v0.9.4**: Security release. Closes three coordinated-disclosure advisories: two SSRF paths (robots.txt and link preview) that bypassed the Docker server's egress controls, and a config trust-boundary bypass that leaked server environment variables. Also adds `PruningContentFilterLXML`, about 10x faster pruning, now the default. No breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.4.md)
+
+✨ Recent v0.9.3: Security release. Closes five coordinated-disclosure advisories: arbitrary file write, SSRF, and denial of service in the PDF processing path, plus two XSS issues in the Docker Playground. Also ships 33 bug fixes across the Docker server, crawler, and PDF handling. No new features, no breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.3.md)
 
 ✨ Recent v0.9.2: Maintenance patch release. Fixes a `MemoryAdaptiveDispatcher` task/page leak when a streaming crawl is closed, Docker Playground "Advanced Config" and Monitor WebSocket auth, Playwright headless-shell packaging, and GPU (`ENABLE_GPU=true`) Docker builds. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.2.md)
 
@@ -567,6 +569,23 @@ async def test_news_crawl():
 ## ✨ Recent Updates
 
 <details open>
+<summary><strong>Version 0.9.4 Release Highlights - Security Release and Faster Pruning</strong></summary>
+
+A security release closing three coordinated-disclosure advisories. Two are SSRF paths that did not go through the Docker server's egress rule: the robots.txt fetch behind `check_robots_txt`, and the URL seeder behind `link_preview_config`, which also returned the fetched `<head>` to the caller. Both now go through the server's pinning egress proxy. The third is a bypass of the untrusted-config gate: a `{"type": "dict"}` wrapper let a forbidden `LLMConfig` through, so a non-admin client could read server environment variables.
+
+It also adds `PruningContentFilterLXML`, an lxml-native pruning filter that is about 10x faster and gives byte-identical output. It is now the default, and `PruningContentFilter` is deprecated. Bug fixes cover deep-crawl speed, tables with `rowspan`/`colspan`, robots.txt rules, pooled browser recycling, and the Docker Playground.
+
+No breaking changes.
+
+```bash
+pip install -U crawl4ai
+```
+
+[Full v0.9.4 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.4.md)
+
+</details>
+
+<details>
 <summary><strong>Version 0.9.3 Release Highlights - Security Release</strong></summary>
 
 A security release closing five coordinated-disclosure advisories. Four are in the PDF processing path: an arbitrary file write through `PDFContentScrapingStrategy` image-write fields, an SSRF where the PDF download followed redirects into internal addresses, a denial of service from unbounded PDF size and page count, and an XSS from unescaped PDF text in `cleaned_html`. The fifth is a DOM-based XSS in the Docker Playground that could expose the operator's API token.
diff --git a/SECURITY-CREDITS.md b/SECURITY-CREDITS.md
index 2d49ba0d0..781e13ce8 100644
--- a/SECURITY-CREDITS.md
+++ b/SECURITY-CREDITS.md
@@ -21,3 +21,7 @@ We thank the following security researchers for their responsible disclosure:
 | Zhixi "Jace" Sun | GitHub: [manus-use](https://github.com/manus-use) | Arbitrary file write via unconfined PDFContentScrapingStrategy image-write fields in untrusted config bodies (0.9.3) | 2026-08-24 |
 | Nguyen Tran Thanh Lam | GitHub: [c240030](https://github.com/c240030) | SSRF via PDF download redirects, DoS via unbounded PDF size and page count, XSS via unescaped PDF text in cleaned_html (0.9.3) | 2026-07-27 |
 | e1codes | GitHub: [e1codes](https://github.com/e1codes) | DOM-based XSS in the Docker Playground leading to operator API-token theft (0.9.3) | 2026-07-24 |
+| x0root | GitHub: [x0root](https://github.com/x0root) | SSRF in the hosted service at stage.crawl4ai.com | 2026-09-02 |
+| arpe1618 | GitHub: [arpe1618](https://github.com/arpe1618) | Blind SSRF via the robots.txt fetch in RobotsParser.can_fetch bypassing the Docker egress controls (0.9.4) | 2026-09-04 |
+| Ibrahim AlJaafreh - Cystack RedTeam | [LinkedIn](https://www.linkedin.com/in/ibrahim-aljaafreh-glitch/), [cystack.ps](https://cystack.ps) | SSRF with response disclosure via link_preview_config through the URL seeder (0.9.4) | 2026-09-04 |
+| Adam Jordan | GitHub: [adamyordan](https://github.com/adamyordan) | Untrusted-config gate bypass via dict-wrapper laundering, leaking server env vars (0.9.4) | 2026-09-08 |
diff --git a/SECURITY.md b/SECURITY.md
index 6cbcaae00..e5d4ecd5f 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -96,6 +96,14 @@ When using Crawl4AI as a Python library:
 
 All issues below are fixed in the version named. Advisories with a GHSA id are published under [Security Advisories](https://github.com/unclecode/crawl4ai/security/advisories). Full detail for every release is in [CHANGELOG.md](CHANGELOG.md); reporter credits are in [SECURITY-CREDITS.md](SECURITY-CREDITS.md).
 
+### Fixed in v0.9.4 (2026-09-23)
+
+| ID | Severity | Component | Description | Fix |
+|----|----------|-----------|-------------|-----|
+| GHSA-f77g-77vp-r96v | MEDIUM | Library + Docker API | Blind SSRF via the robots.txt fetch in `RobotsParser.can_fetch`, outside the egress controls; redirects followed, DNS re-resolved (CWE-918) | Fetch routed through the pinning egress proxy; TLS verification restored |
+| GHSA-wh5w-hmj3-vgg7 | HIGH | Library + Docker API | SSRF with response disclosure via `link_preview_config`: the URL seeder fetched every link outside the egress controls and returned the parsed `<head>` (CWE-918) | Seeder fetches routed through the pinning egress proxy; `LinkPreviewConfig` caps `max_links`, `concurrency`, `timeout` for untrusted bodies |
+| GHSA-5w5p-vcv6-mm3f | HIGH | Docker API | Untrusted-config gate bypass via `{"type": "dict"}` wrapper laundering, leaking server env vars such as LLM keys and `SECRET_KEY` (CWE-501) | Unwrapped value re-checked under the untrusted gate; `from_kwargs` carries the caller's provenance |
+
 ### Fixed in v0.9.3 (2026-08-31)
 
 | ID | Severity | Component | Description | Fix |
@@ -191,6 +199,12 @@ Secure-by-default rework of the Docker API server. The pip library is unchanged.
 
 ## Security Features
 
+### v0.9.4+
+
+- **Library egress proxy**: `crawl4ai/egress_policy.py` routes the library's own HTTP clients (URL seeder, robots.txt) through the Docker server's pinning proxy, so they get the same resolve-and-pin rule as the browser
+- **Link preview caps**: `LinkPreviewConfig` `max_links` (100), `concurrency` (10), and `timeout` (10 s) clamped for untrusted bodies
+- **Nested typed objects gated**: `{"type": "dict"}` wrappers are re-checked against `UNTRUSTED_ALLOWED_TYPES`
+
 ### v0.9.3+
 
 - **PDF egress policy**: PDF downloads validate every redirect hop and the peer IP actually read, so the Docker SSRF policy also covers the out-of-browser `requests` path
diff --git a/crawl4ai/__version__.py b/crawl4ai/__version__.py
index f8f1eb5d0..d332e3a47 100644
--- a/crawl4ai/__version__.py
+++ b/crawl4ai/__version__.py
@@ -1,7 +1,7 @@
 # crawl4ai/__version__.py
 
 # This is the version that will be used for stable releases
-__version__ = "0.9.3"
+__version__ = "0.9.4"
 
 # For nightly builds, this gets set during build process
 __nightly_version__ = None
diff --git a/docs/blog/release-v0.9.4.md b/docs/blog/release-v0.9.4.md
new file mode 100644
index 000000000..451d3b114
--- /dev/null
+++ b/docs/blog/release-v0.9.4.md
@@ -0,0 +1,140 @@
+# Crawl4AI v0.9.4: Security Release and 10x Faster Pruning
+
+*September 2026 - 4 min read*
+
+---
+
+I'm releasing Crawl4AI v0.9.4. It closes three coordinated-disclosure advisories, makes content pruning about 10x faster, and ships the bug fixes that landed on `develop` since 0.9.3. No breaking changes.
+
+Two of the three advisories share one root cause. The Docker server sends every attacker-influenced fetch through one egress rule: reject any destination that resolves to a non-global IP, re-check each redirect hop, and dial the pinned IP so DNS cannot rebind. The browser, the PDF downloader, and the job webhook all went through it. The robots.txt check and the URL seeder did not. They used their own HTTP clients. The third advisory is a trust-boundary bypass that let an ordinary API client read server environment variables.
+
+If you self-host the Docker server, upgrade.
+
+## What's new at a glance
+
+- **SSRF via robots.txt**: `check_robots_txt` could make the server fetch internal addresses
+- **SSRF via link preview**: the seeder fetched internal links and returned their `<head>` to the caller
+- **Config gate bypass**: a dict wrapper let a forbidden `LLMConfig` through and leaked env vars
+- **10x faster pruning**: new `PruningContentFilterLXML`, now the default
+- **Bug fixes**: deep crawl, tables, robots.txt, timeouts, pooled browsers, and the Playground
+
+## Security fixes
+
+### Blind SSRF via the robots.txt fetch
+
+**GHSA-f77g-77vp-r96v, CWE-918, medium.** Credit: [arpe1618](https://github.com/arpe1618).
+
+`RobotsParser.can_fetch()` built `{scheme}://{host}/robots.txt` from the caller's URL and fetched it on a bare `aiohttp` session. No pinning, redirects followed by default, and TLS verification off. `check_robots_txt` is allowed in an untrusted request body, so any API client could choose the host. A public host whose robots.txt answers `302` to `169.254.169.254` was enough to reach cloud metadata.
+
+The response body was not returned to the caller, so this was blind. It still gave internal request delivery, port discovery through timing, and a small boolean side channel through the robots decision.
+
+### SSRF with response disclosure via link_preview_config
+
+**GHSA-wh5w-hmj3-vgg7, CWE-918, high.** Credit: Ibrahim AlJaafreh ([LinkedIn](https://www.linkedin.com/in/ibrahim-aljaafreh-glitch/)), Cystack RedTeam ([cystack.ps](https://cystack.ps)).
+
+`link_preview_config` is allowed in an untrusted body, and `LinkPreviewConfig` had no field allowlist. A client could set `include_external=True` and `include_patterns=["*"]`, point the crawl at a page it controls, and the server would fetch every link on it. That included internal and metadata addresses. The seeder then parsed each response's `<head>` and returned it in `result.links[*].head_data`. Unlike the robots issue, this one disclosed content.
+
+### The fix for both SSRF issues
+
+A new module, `crawl4ai/egress_policy.py`, holds one process-wide egress proxy URL for the library's own HTTP clients. The seeder and the robots.txt fetch now pass `proxy=proxy_url()`. The Docker server already runs a `PinningProxy` for Chromium, and it registers that proxy here at boot.
+
+A proxy, not a validator hook, because a validator resolves the name, checks it, and throws the address away. The client then resolves again when it connects, and that gap is the DNS rebinding window. The proxy resolves, pins, and dials the same address, and it re-checks every redirect hop.
+
+A plain library caller sets nothing. `proxy_url()` returns `None`, which is the default on both `httpx` and `aiohttp`, so nothing changes.
+
+Also in this fix:
+
+- The robots.txt fetch verifies TLS now.
+- `LinkPreviewConfig` is capped for untrusted bodies: `max_links` 100, `concurrency` 10, `timeout` 10 seconds.
+
+### Untrusted-config gate bypass via dict-wrapper laundering
+
+**GHSA-5w5p-vcv6-mm3f, CWE-501, high.** Credit: Adam Jordan ([adamyordan](https://github.com/adamyordan)).
+
+The Docker server limits which config types an untrusted body may build. `LLMConfig` is not on that list, because it resolves `api_token="env:NAME"` from the server environment. Two defects combined:
+
+1. `from_serializable_dict()` unwrapped `{"type": "dict", "value": X}` by walking `X.items()`. It never checked `X` itself as a typed object, so a forbidden type inside `value` was never seen by the gate.
+2. `CrawlerRunConfig.from_kwargs` and `BrowserConfig.from_kwargs` called `from_serializable_dict()` with no provenance, and the default was trusted. The laundered object was rebuilt as trusted.
+
+The result: an ordinary, non-admin API client could read any server environment variable, including `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, and `SECRET_KEY`. With `SECRET_KEY` it could forge an admin JWT.
+
+The unwrapped value is now re-checked under the untrusted gate, and `from_kwargs` carries the caller's provenance. If you run the Docker server with secrets in its environment, rotate `SECRET_KEY` and any LLM keys after you upgrade.
+
+## Faster pruning: PruningContentFilterLXML
+
+`PruningContentFilter` called `get_text()` and `encode_contents()` on every node while pruning. Each call walks the whole subtree, so the cost grew faster than the page. A page with 6000 cards spent about 2.2 seconds just pruning.
+
+`PruningContentFilterLXML` computes every per-node metric (text length, inner-HTML length, link text, word count) once, in one bottom-up pass. Then it scores and prunes top-down. That is O(N).
+
+The output is byte-identical to the old filter. The scoring, thresholds, tag weights, and bs4 quirks are reproduced 1:1, and it was checked node by node on real pages and adversarial shapes.
+
+| Page | Before | After |
+|---|---|---|
+| medium | 134 ms | 13 ms |
+| 200 cards | 67 ms | 7 ms |
+| 6000 cards | 2200 ms | 260 ms |
+
+It is now the default for the Docker server's fit filter and the CLI. To use it in your own code:
+
+```python
+from crawl4ai import PruningContentFilterLXML
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+md = DefaultMarkdownGenerator(content_filter=PruningContentFilterLXML(threshold=0.48))
+```
+
+`PruningContentFilter` still works, but now warns with a `DeprecationWarning`. It takes the same arguments, so the switch is a rename.
+
+## Bug fixes
+
+### Crawler and core
+
+- **Deep crawl speed**: BFS re-scanned the whole level to find each result's parent, which was O(n^2) on wide pages. BestFirst enqueued the same URL twice. Both are fixed, and de-duplication keeps the shallowest depth, so no subtree is lost. (#2265, issue #2242)
+- **Tables with spans**: `rowspan` and `colspan` are expanded into a real grid, and `<th>` row headers are kept instead of shifting every cell left. Spans are clamped, so one huge cell cannot hang the parse. (#2261, issue #2258)
+- **robots.txt `Disallow: /*?`**: no longer blocks the whole site. (#2229, thanks @Nalhin)
+- **robots.txt on Python 3.14**: the wildcard patch is skipped. 3.14 supports wildcards natively, and the patch broke `Allow:` precedence. (#2278)
+- **Timeout ceiling**: `CRAWL4AI_MAX_TIMEOUT_MS` sets the ceiling for timeouts on untrusted configs. The default stays 60 seconds. A malformed or non-positive value falls back to the 60-second default, not to the ceiling. (#2212, thanks @damusix; #2266)
+- **macOS arm64 crash**: Chrome for Testing no longer crashes under `--headless=new`. (#2241, issue #2239, thanks @Zsanz3)
+
+### Docker server
+
+- **Pooled browsers slow down**: a pooled browser context gets slower with use, and the idle janitor never fires on a busy server. `crawler.pool.max_pages_before_recycle` (default 200) now recycles a context after that many pages. (#2232, issue #2231)
+- **Playground Advanced Config**: now a JSON params editor. The old Python editor sent a `code` field that the untrusted boundary rejects. (#2262, issue #2260)
+- **Playground `md` and `llm` runs**: they no longer die on the `/config/dump` pre-flight. (#2224, issue #2222)
+- **Permanent browser**: built with the egress-hardened default config, so its pool signature matches incoming requests. (#2237)
+
+### Documentation and CI
+
+- `SECURITY.md` lists 0.9.x as supported. (#2269, thanks @nightcityblade)
+- The Discord stargazer notification no longer depends on a dead Google Apps Script step. (#2263, #2279)
+
+## Breaking changes
+
+None.
+
+## Upgrade
+
+```bash
+pip install -U crawl4ai
+crawl4ai-doctor  # verify installation
+```
+
+Docker users: pull the new image once the Docker release workflow finishes.
+
+```bash
+docker pull unclecode/crawl4ai:0.9.4
+```
+
+## Acknowledgments
+
+Thank you to [arpe1618](https://github.com/arpe1618), Ibrahim AlJaafreh of Cystack RedTeam, and Adam Jordan ([adamyordan](https://github.com/adamyordan)) for reporting these issues privately and giving us time to fix them before disclosure. All reporters are listed in [SECURITY-CREDITS.md](https://github.com/unclecode/crawl4ai/blob/main/SECURITY-CREDITS.md).
+
+Thanks to the community contributors behind the bug fixes in this release: @Nalhin (#2229), @damusix (#2212), @Zsanz3 (#2241), and @nightcityblade (#2269).
+
+If you find a security issue in Crawl4AI, please report it privately. See [SECURITY.md](https://github.com/unclecode/crawl4ai/blob/main/SECURITY.md).
+
+## Support & Resources
+
+- [Documentation](https://docs.crawl4ai.com)
+- [GitHub Issues](https://github.com/unclecode/crawl4ai/issues)
+- [Discord Community](https://discord.gg/crawl4ai)