From 8bd0776bb7ee2befb0e0e74ea86fa1056591d309 Mon Sep 17 00:00:00 2001 From: Talel Boussetta Date: Fri, 25 Sep 2026 10:29:46 +0100 Subject: [PATCH 1/3] fix(models): default CrawlResult links and media to the scraped shape CrawlResult.links and .media defaulted to {}. Successful crawls fill them from the scraper's result, but the results AsyncWebCrawler builds itself -- the robots.txt refusal, "All proxies failed", the exception path in arun, the anti-bot raw-HTML fallback -- never set them, so those results had no "internal"/"external" or "images" keys: result.links["internal"] raised KeyError, and the Docker API returned "links": {} for exactly the entries of a batch a client has to handle. Default both to the keys a scraped page gets -- {"internal": [], "external": []} and {"images": [], "videos": [], "audios": []} ("tables" is moved out of media into CrawlResult.tables on the success path) -- so every construction site is covered, including future ones. --- crawl4ai/models.py | 11 +++- tests/unit/test_crawl_result_default_shape.py | 58 +++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_crawl_result_default_shape.py diff --git a/crawl4ai/models.py b/crawl4ai/models.py index 7ab3794cc..14098980b 100644 --- a/crawl4ai/models.py +++ b/crawl4ai/models.py @@ -133,8 +133,15 @@ class CrawlResult(BaseModel): fit_html: Optional[str] = None success: bool cleaned_html: Optional[str] = None - media: Dict[str, List[Dict]] = {} - links: Dict[str, List[Dict]] = {} + # The keys a scraped page gets (arun moves media's "tables" to .tables), so + # a result built without them -- robots.txt refusal, a failed fetch -- + # keeps the shape. + media: Dict[str, List[Dict]] = Field( + default_factory=lambda: {"images": [], "videos": [], "audios": []} + ) + links: Dict[str, List[Dict]] = Field( + default_factory=lambda: {"internal": [], "external": []} + ) downloaded_files: Optional[List[str]] = None js_execution_result: Optional[Dict[str, Any]] = None screenshot: Optional[str] = None diff --git a/tests/unit/test_crawl_result_default_shape.py b/tests/unit/test_crawl_result_default_shape.py new file mode 100644 index 000000000..9e965f075 --- /dev/null +++ b/tests/unit/test_crawl_result_default_shape.py @@ -0,0 +1,58 @@ +"""A CrawlResult built without links/media must still have their keys. + +Successful crawls fill `links` and `media` from the scraper's result (arun moves +media's "tables" to CrawlResult.tables). Results built without them -- the +robots.txt refusal, the "all proxies failed" and exception paths in +AsyncWebCrawler.arun -- used to default to `{}`, so `result.links["internal"]` +raised KeyError on exactly the results a batch caller has to handle, and the +Docker API returned `"links": {}` / `"media": {}` for them. +""" + +from crawl4ai.models import CrawlResult + +LINKS = {"internal": [], "external": []} +MEDIA = {"images": [], "videos": [], "audios": []} + + +def _refused(url="https://example.com/private"): + # The shape AsyncWebCrawler.arun returns when robots.txt disallows a URL. + return CrawlResult( + url=url, + html="", + success=False, + status_code=403, + error_message="Access denied by robots.txt", + ) + + +def test_result_without_links_or_media_has_the_scraped_shape(): + result = _refused() + + assert result.links == LINKS + assert result.media == MEDIA + + +def test_default_is_not_shared_between_results(): + first, second = _refused(), _refused("https://example.com/other") + + first.links["internal"].append({"href": "https://example.com/a"}) + first.media["images"].append({"src": "https://example.com/a.png"}) + + assert second.links == LINKS + assert second.media == MEDIA + + +def test_serialized_result_keeps_the_keys(): + dumped = _refused().model_dump() + + assert dumped["links"] == LINKS + assert dumped["media"] == MEDIA + + +def test_explicit_values_are_kept(): + links = {"internal": [{"href": "https://example.com/a"}], "external": []} + result = CrawlResult( + url="https://example.com/", html="

", success=True, links=links + ) + + assert result.links == links From 1148176b3a5b2ecb3f49d12480619fc6bfc7131a Mon Sep 17 00:00:00 2001 From: Talel Boussetta Date: Fri, 25 Sep 2026 11:21:57 +0100 Subject: [PATCH 2/3] docs(models): show the new CrawlResult defaults; tighten the isolation test The CrawlResult sketches in docs/md_v2/api/crawl-result.md, docs/md_v2/core/crawler-result.md and the /execute_js docstring still showed `= {}` for links and media. The isolation test now also asserts the nested lists are distinct objects, and compares against fresh literals. --- deploy/docker/server.py | 4 ++-- docs/md_v2/api/crawl-result.md | 4 ++-- docs/md_v2/core/crawler-result.md | 4 ++-- tests/unit/test_crawl_result_default_shape.py | 24 ++++++++++++------- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/deploy/docker/server.py b/deploy/docker/server.py index 248155278..3d71a307f 100644 --- a/deploy/docker/server.py +++ b/deploy/docker/server.py @@ -842,8 +842,8 @@ class CrawlResult(BaseModel): html: str success: bool cleaned_html: Optional[str] = None - media: Dict[str, List[Dict]] = {} - links: Dict[str, List[Dict]] = {} + media: Dict[str, List[Dict]] = {"images": [], "videos": [], "audios": []} + links: Dict[str, List[Dict]] = {"internal": [], "external": []} downloaded_files: Optional[List[str]] = None js_execution_result: Optional[Dict[str, Any]] = None screenshot: Optional[str] = None diff --git a/docs/md_v2/api/crawl-result.md b/docs/md_v2/api/crawl-result.md index f061836c9..a7345e217 100644 --- a/docs/md_v2/api/crawl-result.md +++ b/docs/md_v2/api/crawl-result.md @@ -11,8 +11,8 @@ class CrawlResult(BaseModel): success: bool cleaned_html: Optional[str] = None fit_html: Optional[str] = None # Preprocessed HTML optimized for extraction - media: Dict[str, List[Dict]] = {} - links: Dict[str, List[Dict]] = {} + media: Dict[str, List[Dict]] = {"images": [], "videos": [], "audios": []} + links: Dict[str, List[Dict]] = {"internal": [], "external": []} downloaded_files: Optional[List[str]] = None screenshot: Optional[str] = None pdf : Optional[bytes] = None diff --git a/docs/md_v2/core/crawler-result.md b/docs/md_v2/core/crawler-result.md index 79f723185..2a3f5e922 100644 --- a/docs/md_v2/core/crawler-result.md +++ b/docs/md_v2/core/crawler-result.md @@ -22,8 +22,8 @@ class CrawlResult(BaseModel): fit_html: Optional[str] = None success: bool cleaned_html: Optional[str] = None - media: Dict[str, List[Dict]] = {} - links: Dict[str, List[Dict]] = {} + media: Dict[str, List[Dict]] = {"images": [], "videos": [], "audios": []} + links: Dict[str, List[Dict]] = {"internal": [], "external": []} downloaded_files: Optional[List[str]] = None js_execution_result: Optional[Dict[str, Any]] = None screenshot: Optional[str] = None diff --git a/tests/unit/test_crawl_result_default_shape.py b/tests/unit/test_crawl_result_default_shape.py index 9e965f075..4311ec176 100644 --- a/tests/unit/test_crawl_result_default_shape.py +++ b/tests/unit/test_crawl_result_default_shape.py @@ -10,8 +10,13 @@ from crawl4ai.models import CrawlResult -LINKS = {"internal": [], "external": []} -MEDIA = {"images": [], "videos": [], "audios": []} + +def links_shape(): + return {"internal": [], "external": []} + + +def media_shape(): + return {"images": [], "videos": [], "audios": []} def _refused(url="https://example.com/private"): @@ -28,25 +33,28 @@ def _refused(url="https://example.com/private"): def test_result_without_links_or_media_has_the_scraped_shape(): result = _refused() - assert result.links == LINKS - assert result.media == MEDIA + assert result.links == links_shape() + assert result.media == media_shape() def test_default_is_not_shared_between_results(): first, second = _refused(), _refused("https://example.com/other") + assert first.links["internal"] is not second.links["internal"] + assert first.media["images"] is not second.media["images"] + first.links["internal"].append({"href": "https://example.com/a"}) first.media["images"].append({"src": "https://example.com/a.png"}) - assert second.links == LINKS - assert second.media == MEDIA + assert second.links == links_shape() + assert second.media == media_shape() def test_serialized_result_keeps_the_keys(): dumped = _refused().model_dump() - assert dumped["links"] == LINKS - assert dumped["media"] == MEDIA + assert dumped["links"] == links_shape() + assert dumped["media"] == media_shape() def test_explicit_values_are_kept(): From e88c4417d45081699a8f79a7767ff9646fe60640 Mon Sep 17 00:00:00 2001 From: Talel Boussetta Date: Sat, 26 Sep 2026 16:59:11 +0100 Subject: [PATCH 3/3] docs(models): show the new CrawlResult defaults in the SDK reference complete-sdk-reference.md repeats the CrawlResult sketch from docs/md_v2/api/crawl-result.md and still showed `= {}` for links and media. --- docs/md_v2/complete-sdk-reference.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/md_v2/complete-sdk-reference.md b/docs/md_v2/complete-sdk-reference.md index 2e86db1e7..afa6a8650 100644 --- a/docs/md_v2/complete-sdk-reference.md +++ b/docs/md_v2/complete-sdk-reference.md @@ -1077,8 +1077,8 @@ class CrawlResult(BaseModel): success: bool cleaned_html: Optional[str] = None fit_html: Optional[str] = None # Preprocessed HTML optimized for extraction - media: Dict[str, List[Dict]] = {} - links: Dict[str, List[Dict]] = {} + media: Dict[str, List[Dict]] = {"images": [], "videos": [], "audios": []} + links: Dict[str, List[Dict]] = {"internal": [], "external": []} downloaded_files: Optional[List[str]] = None screenshot: Optional[str] = None pdf : Optional[bytes] = None