From f585aed3d575198d84b5cec881b770939a618bda Mon Sep 17 00:00:00 2001 From: IslamHisham Date: Wed, 5 Aug 2026 14:07:13 -0400 Subject: [PATCH 1/3] Enhance source creation with optional content extraction Added functionality to optionally fetch and extract content from URLs using trafilatura. --- .../implementations/serper_web_search_service.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/app/services/implementations/serper_web_search_service.py b/app/services/implementations/serper_web_search_service.py index 7b83baf..1179909 100644 --- a/app/services/implementations/serper_web_search_service.py +++ b/app/services/implementations/serper_web_search_service.py @@ -2,7 +2,7 @@ from datetime import UTC, datetime from typing import List, Optional from uuid import UUID, uuid4 - +import trafilatura import aiohttp from sqlalchemy.exc import IntegrityError @@ -134,8 +134,15 @@ async def _update_source_analysis( return await self.source_repository.update(source) async def _create_new_source( - self, item: dict, search_id: UUID, domain_id: UUID, credibility_score: float + self, item: dict, search_id: UUID, domain_id: UUID, credibility_score: float, get_content: bool, ) -> Optional[SourceModel]: + full_content = None + try: + if get_content: + downloaded = trafilatura.fetch_url(item["link"]) + full_content = trafilatura.extract(downloaded) + except: + pass try: source = SourceModel( id=uuid4(), @@ -144,7 +151,7 @@ async def _create_new_source( title=item.get("title", "Untitled"), snippet=item.get("snippet", ""), domain_id=domain_id, - content=None, + content=full_content, credibility_score=credibility_score, created_at=datetime.now(UTC), updated_at=datetime.now(UTC), From 2c733f44900f383b4b84a067440788e700242813 Mon Sep 17 00:00:00 2001 From: IslamHisham Date: Wed, 5 Aug 2026 14:12:09 -0400 Subject: [PATCH 2/3] Add default value for get_content in _create_new_source Set default value for get_content parameter to True. --- app/services/implementations/serper_web_search_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/implementations/serper_web_search_service.py b/app/services/implementations/serper_web_search_service.py index 1179909..944865e 100644 --- a/app/services/implementations/serper_web_search_service.py +++ b/app/services/implementations/serper_web_search_service.py @@ -134,7 +134,7 @@ async def _update_source_analysis( return await self.source_repository.update(source) async def _create_new_source( - self, item: dict, search_id: UUID, domain_id: UUID, credibility_score: float, get_content: bool, + self, item: dict, search_id: UUID, domain_id: UUID, credibility_score: float, get_content: bool = True, ) -> Optional[SourceModel]: full_content = None try: From 7e9247baeea04bcaeb5207b33b9cd8a5f96d745f Mon Sep 17 00:00:00 2001 From: IslamHisham Date: Wed, 5 Aug 2026 14:18:05 -0400 Subject: [PATCH 3/3] Improve error handling in content fetching Log exceptions when fetching content from URLs. --- app/services/implementations/serper_web_search_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/services/implementations/serper_web_search_service.py b/app/services/implementations/serper_web_search_service.py index 944865e..a56bea2 100644 --- a/app/services/implementations/serper_web_search_service.py +++ b/app/services/implementations/serper_web_search_service.py @@ -141,8 +141,8 @@ async def _create_new_source( if get_content: downloaded = trafilatura.fetch_url(item["link"]) full_content = trafilatura.extract(downloaded) - except: - pass + except Exception as e: + logging.exception("An error occurred while fetching content from {}.".format(item["link"])) try: source = SourceModel( id=uuid4(),