From 06d5de0171eabaab4a050dc0d9cf64b69bed2f19 Mon Sep 17 00:00:00 2001 From: IslamHisham Date: Wed, 5 Aug 2026 13:22:04 -0400 Subject: [PATCH 1/6] Add trafilatura dependency Added trafilatura version 2.2.0 to requirements. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d7d0002..7353315 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # 1. Intel/AMD Linux Servers (GKE Standard) -> Force the small CPU version torch==2.4.0+cpu; sys_platform == 'linux' and platform_machine == 'x86_64' - +trafilatura==2.2.0 aiohappyeyeballs==2.4.3 aiohttp==3.10.10 aiosignal==1.3.1 From f585aed3d575198d84b5cec881b770939a618bda Mon Sep 17 00:00:00 2001 From: IslamHisham Date: Wed, 5 Aug 2026 14:07:13 -0400 Subject: [PATCH 2/6] Enhance source creation with optional content extraction Added functionality to optionally fetch and extract content from URLs using trafilatura. --- .../implementations/serper_web_search_service.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/app/services/implementations/serper_web_search_service.py b/app/services/implementations/serper_web_search_service.py index 7b83baf..1179909 100644 --- a/app/services/implementations/serper_web_search_service.py +++ b/app/services/implementations/serper_web_search_service.py @@ -2,7 +2,7 @@ from datetime import UTC, datetime from typing import List, Optional from uuid import UUID, uuid4 - +import trafilatura import aiohttp from sqlalchemy.exc import IntegrityError @@ -134,8 +134,15 @@ async def _update_source_analysis( return await self.source_repository.update(source) async def _create_new_source( - self, item: dict, search_id: UUID, domain_id: UUID, credibility_score: float + self, item: dict, search_id: UUID, domain_id: UUID, credibility_score: float, get_content: bool, ) -> Optional[SourceModel]: + full_content = None + try: + if get_content: + downloaded = trafilatura.fetch_url(item["link"]) + full_content = trafilatura.extract(downloaded) + except: + pass try: source = SourceModel( id=uuid4(), @@ -144,7 +151,7 @@ async def _create_new_source( title=item.get("title", "Untitled"), snippet=item.get("snippet", ""), domain_id=domain_id, - content=None, + content=full_content, credibility_score=credibility_score, created_at=datetime.now(UTC), updated_at=datetime.now(UTC), From 2c733f44900f383b4b84a067440788e700242813 Mon Sep 17 00:00:00 2001 From: IslamHisham Date: Wed, 5 Aug 2026 14:12:09 -0400 Subject: [PATCH 3/6] Add default value for get_content in _create_new_source Set default value for get_content parameter to True. --- app/services/implementations/serper_web_search_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/implementations/serper_web_search_service.py b/app/services/implementations/serper_web_search_service.py index 1179909..944865e 100644 --- a/app/services/implementations/serper_web_search_service.py +++ b/app/services/implementations/serper_web_search_service.py @@ -134,7 +134,7 @@ async def _update_source_analysis( return await self.source_repository.update(source) async def _create_new_source( - self, item: dict, search_id: UUID, domain_id: UUID, credibility_score: float, get_content: bool, + self, item: dict, search_id: UUID, domain_id: UUID, credibility_score: float, get_content: bool = True, ) -> Optional[SourceModel]: full_content = None try: From 7e9247baeea04bcaeb5207b33b9cd8a5f96d745f Mon Sep 17 00:00:00 2001 From: IslamHisham Date: Wed, 5 Aug 2026 14:18:05 -0400 Subject: [PATCH 4/6] Improve error handling in content fetching Log exceptions when fetching content from URLs. --- app/services/implementations/serper_web_search_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/services/implementations/serper_web_search_service.py b/app/services/implementations/serper_web_search_service.py index 944865e..a56bea2 100644 --- a/app/services/implementations/serper_web_search_service.py +++ b/app/services/implementations/serper_web_search_service.py @@ -141,8 +141,8 @@ async def _create_new_source( if get_content: downloaded = trafilatura.fetch_url(item["link"]) full_content = trafilatura.extract(downloaded) - except: - pass + except Exception as e: + logging.exception("An error occurred while fetching content from {}.".format(item["link"])) try: source = SourceModel( id=uuid4(), From 5c6a3f2b7ea9a956dbeba282f2a7ea7a94d260f5 Mon Sep 17 00:00:00 2001 From: IslamHisham Date: Wed, 5 Aug 2026 14:32:55 -0400 Subject: [PATCH 5/6] Enhance error logging in content fetching Improve error logging to include exception details. --- app/services/implementations/serper_web_search_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/implementations/serper_web_search_service.py b/app/services/implementations/serper_web_search_service.py index a56bea2..6b8b72d 100644 --- a/app/services/implementations/serper_web_search_service.py +++ b/app/services/implementations/serper_web_search_service.py @@ -142,7 +142,7 @@ async def _create_new_source( downloaded = trafilatura.fetch_url(item["link"]) full_content = trafilatura.extract(downloaded) except Exception as e: - logging.exception("An error occurred while fetching content from {}.".format(item["link"])) + logging.error("An error occurred while fetching content from {}, full error syntax is: {}.".format(item["link"], e)) try: source = SourceModel( id=uuid4(), From cbf88222d71bd6ad152ddf8d06652a3794893730 Mon Sep 17 00:00:00 2001 From: IslamHisham Date: Thu, 6 Aug 2026 14:22:20 -0400 Subject: [PATCH 6/6] Modify payload query of serper for Canadian source prioritization Updated payload to prioritize primary Canadian authoritative sources for claims. --- app/services/implementations/serper_web_search_service.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/services/implementations/serper_web_search_service.py b/app/services/implementations/serper_web_search_service.py index 6b8b72d..6ccb9d3 100644 --- a/app/services/implementations/serper_web_search_service.py +++ b/app/services/implementations/serper_web_search_service.py @@ -67,7 +67,8 @@ async def search_and_create_sources( """Search for sources and create or update records.""" try: logger.warning("SERPER SOURCE FILTER CODE IS RUNNING") - payload = {"q": claim_text, "location": "Canada", "gl": "ca"} + payload = {"q": " I will provide you with a claim. Prioritize primary Canadian authoritative sources to address this claim. Claim: " + claim_text, + "location": "Canada", "gl": "ca"} if language == "french": payload["hl"] = "fr"