diff --git a/app/services/implementations/serper_web_search_service.py b/app/services/implementations/serper_web_search_service.py index 7b83baf..6ccb9d3 100644 --- a/app/services/implementations/serper_web_search_service.py +++ b/app/services/implementations/serper_web_search_service.py @@ -2,7 +2,7 @@ from datetime import UTC, datetime from typing import List, Optional from uuid import UUID, uuid4 - +import trafilatura import aiohttp from sqlalchemy.exc import IntegrityError @@ -67,7 +67,8 @@ async def search_and_create_sources( """Search for sources and create or update records.""" try: logger.warning("SERPER SOURCE FILTER CODE IS RUNNING") - payload = {"q": claim_text, "location": "Canada", "gl": "ca"} + payload = {"q": " I will provide you with a claim. Prioritize primary Canadian authoritative sources to address this claim. Claim: " + claim_text, + "location": "Canada", "gl": "ca"} if language == "french": payload["hl"] = "fr" @@ -134,8 +135,15 @@ async def _update_source_analysis( return await self.source_repository.update(source) async def _create_new_source( - self, item: dict, search_id: UUID, domain_id: UUID, credibility_score: float + self, item: dict, search_id: UUID, domain_id: UUID, credibility_score: float, get_content: bool = True, ) -> Optional[SourceModel]: + full_content = None + try: + if get_content: + downloaded = trafilatura.fetch_url(item["link"]) + full_content = trafilatura.extract(downloaded) + except Exception as e: + logging.error("An error occurred while fetching content from {}, full error syntax is: {}.".format(item["link"], e)) try: source = SourceModel( id=uuid4(), @@ -144,7 +152,7 @@ async def _create_new_source( title=item.get("title", "Untitled"), snippet=item.get("snippet", ""), domain_id=domain_id, - content=None, + content=full_content, credibility_score=credibility_score, created_at=datetime.now(UTC), updated_at=datetime.now(UTC), diff --git a/requirements.txt b/requirements.txt index d7d0002..7353315 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # 1. Intel/AMD Linux Servers (GKE Standard) -> Force the small CPU version torch==2.4.0+cpu; sys_platform == 'linux' and platform_machine == 'x86_64' - +trafilatura==2.2.0 aiohappyeyeballs==2.4.3 aiohttp==3.10.10 aiosignal==1.3.1