From 7e5a78eccb484b2f6d9d8bacdb85f2d1335c82be Mon Sep 17 00:00:00 2001 From: riandradiva Date: Mon, 31 Aug 2026 11:14:49 +0700 Subject: [PATCH 1/3] feat: add mrscraper integrations --- lib/crewai-tools/src/crewai_tools/__init__.py | 34 + .../src/crewai_tools/tools/__init__.py | 34 + .../crewai_tools/tools/mrscraper/README.md | 122 + .../crewai_tools/tools/mrscraper/__init__.py | 48 + .../crewai_tools/tools/mrscraper/account.py | 23 + .../src/crewai_tools/tools/mrscraper/base.py | 48 + .../crewai_tools/tools/mrscraper/client.py | 100 + .../crewai_tools/tools/mrscraper/discovery.py | 81 + .../tools/mrscraper/extraction.py | 186 ++ .../crewai_tools/tools/mrscraper/payloads.py | 212 ++ .../crewai_tools/tools/mrscraper/results.py | 92 + .../crewai_tools/tools/mrscraper/schemas.py | 427 ++++ .../tools/mrscraper/scraper_creation.py | 123 + .../tools/mrscraper/scraper_runs.py | 72 + .../mrscraper/structured_data_prompts.json | 1 + .../crewai_tools/tools/mrscraper/toolkit.py | 113 + .../tools/mrscraper/test_mrscraper_tools.py | 625 +++++ lib/crewai-tools/tool.specs.json | 2071 +++++++++++++++++ scripts/test_mrscraper_real.py | 280 +++ 19 files changed, 4692 insertions(+) create mode 100644 lib/crewai-tools/src/crewai_tools/tools/mrscraper/README.md create mode 100644 lib/crewai-tools/src/crewai_tools/tools/mrscraper/__init__.py create mode 100644 lib/crewai-tools/src/crewai_tools/tools/mrscraper/account.py create mode 100644 lib/crewai-tools/src/crewai_tools/tools/mrscraper/base.py create mode 100644 lib/crewai-tools/src/crewai_tools/tools/mrscraper/client.py create mode 100644 lib/crewai-tools/src/crewai_tools/tools/mrscraper/discovery.py create mode 100644 lib/crewai-tools/src/crewai_tools/tools/mrscraper/extraction.py create mode 100644 lib/crewai-tools/src/crewai_tools/tools/mrscraper/payloads.py create mode 100644 lib/crewai-tools/src/crewai_tools/tools/mrscraper/results.py create mode 100644 lib/crewai-tools/src/crewai_tools/tools/mrscraper/schemas.py create mode 100644 lib/crewai-tools/src/crewai_tools/tools/mrscraper/scraper_creation.py create mode 100644 lib/crewai-tools/src/crewai_tools/tools/mrscraper/scraper_runs.py create mode 100644 lib/crewai-tools/src/crewai_tools/tools/mrscraper/structured_data_prompts.json create mode 100644 lib/crewai-tools/src/crewai_tools/tools/mrscraper/toolkit.py create mode 100644 lib/crewai-tools/tests/tools/mrscraper/test_mrscraper_tools.py create mode 100644 scripts/test_mrscraper_real.py diff --git a/lib/crewai-tools/src/crewai_tools/__init__.py b/lib/crewai-tools/src/crewai_tools/__init__.py index 2db6be89bc..dd25668971 100644 --- a/lib/crewai-tools/src/crewai_tools/__init__.py +++ b/lib/crewai-tools/src/crewai_tools/__init__.py @@ -119,6 +119,24 @@ MongoDBVectorSearchConfig, MongoDBVectorSearchTool, ) +from crewai_tools.tools.mrscraper import ( + MrScraperCrawlWebsiteUrlsTool, + MrScraperCreateListingScraperTool, + MrScraperCreatePromptScraperTool, + MrScraperCreateWebsiteCrawlScraperTool, + MrScraperExtractListingsTool, + MrScraperExtractPageByPromptTool, + MrScraperExtractStructuredDataTool, + MrScraperFetchRenderedHtmlTool, + MrScraperGetAccountInfoTool, + MrScraperGetLatestResultsTool, + MrScraperGetResultDetailTool, + MrScraperGetResultsTool, + MrScraperRunExistingScraperBatchTool, + MrScraperRunExistingScraperTool, + MrScraperSearchGoogleSerpTool, + create_mrscraper_toolkit, +) from crewai_tools.tools.multion_tool.multion_tool import MultiOnTool from crewai_tools.tools.mysql_search_tool.mysql_search_tool import MySQLSearchTool from crewai_tools.tools.nl2sql.nl2sql_tool import NL2SQLTool @@ -286,6 +304,21 @@ "MergeAgentHandlerTool", "MongoDBVectorSearchConfig", "MongoDBVectorSearchTool", + "MrScraperCrawlWebsiteUrlsTool", + "MrScraperCreateListingScraperTool", + "MrScraperCreatePromptScraperTool", + "MrScraperCreateWebsiteCrawlScraperTool", + "MrScraperExtractListingsTool", + "MrScraperExtractPageByPromptTool", + "MrScraperExtractStructuredDataTool", + "MrScraperFetchRenderedHtmlTool", + "MrScraperGetAccountInfoTool", + "MrScraperGetLatestResultsTool", + "MrScraperGetResultDetailTool", + "MrScraperGetResultsTool", + "MrScraperRunExistingScraperBatchTool", + "MrScraperRunExistingScraperTool", + "MrScraperSearchGoogleSerpTool", "MultiOnTool", "MySQLSearchTool", "NL2SQLTool", @@ -338,6 +371,7 @@ "YoutubeVideoSearchTool", "ZapierActionTool", "ZapierActionTools", + "create_mrscraper_toolkit", ] __version__ = "1.15.18" diff --git a/lib/crewai-tools/src/crewai_tools/tools/__init__.py b/lib/crewai-tools/src/crewai_tools/tools/__init__.py index 2653490f76..51e9ae5db6 100644 --- a/lib/crewai-tools/src/crewai_tools/tools/__init__.py +++ b/lib/crewai-tools/src/crewai_tools/tools/__init__.py @@ -109,6 +109,24 @@ MongoDBVectorSearchConfig, MongoDBVectorSearchTool, ) +from crewai_tools.tools.mrscraper import ( + MrScraperCrawlWebsiteUrlsTool, + MrScraperCreateListingScraperTool, + MrScraperCreatePromptScraperTool, + MrScraperCreateWebsiteCrawlScraperTool, + MrScraperExtractListingsTool, + MrScraperExtractPageByPromptTool, + MrScraperExtractStructuredDataTool, + MrScraperFetchRenderedHtmlTool, + MrScraperGetAccountInfoTool, + MrScraperGetLatestResultsTool, + MrScraperGetResultDetailTool, + MrScraperGetResultsTool, + MrScraperRunExistingScraperBatchTool, + MrScraperRunExistingScraperTool, + MrScraperSearchGoogleSerpTool, + create_mrscraper_toolkit, +) from crewai_tools.tools.multion_tool.multion_tool import MultiOnTool from crewai_tools.tools.mysql_search_tool.mysql_search_tool import MySQLSearchTool from crewai_tools.tools.nl2sql.nl2sql_tool import NL2SQLTool @@ -270,6 +288,21 @@ "MongoDBToolSchema", "MongoDBVectorSearchConfig", "MongoDBVectorSearchTool", + "MrScraperCrawlWebsiteUrlsTool", + "MrScraperCreateListingScraperTool", + "MrScraperCreatePromptScraperTool", + "MrScraperCreateWebsiteCrawlScraperTool", + "MrScraperExtractListingsTool", + "MrScraperExtractPageByPromptTool", + "MrScraperExtractStructuredDataTool", + "MrScraperFetchRenderedHtmlTool", + "MrScraperGetAccountInfoTool", + "MrScraperGetLatestResultsTool", + "MrScraperGetResultDetailTool", + "MrScraperGetResultsTool", + "MrScraperRunExistingScraperBatchTool", + "MrScraperRunExistingScraperTool", + "MrScraperSearchGoogleSerpTool", "MultiOnTool", "MySQLSearchTool", "NL2SQLTool", @@ -320,4 +353,5 @@ "YoutubeChannelSearchTool", "YoutubeVideoSearchTool", "ZapierActionTools", + "create_mrscraper_toolkit", ] diff --git a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/README.md b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/README.md new file mode 100644 index 0000000000..38bd0b3ea3 --- /dev/null +++ b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/README.md @@ -0,0 +1,122 @@ +# MrScraper tools + +The MrScraper integration exposes 15 independent CrewAI tools. It uses the +`requests` dependency already included with `crewai-tools`; no vendor SDK or +optional extra is required. + +## Installation and authentication + +```bash +uv add crewai-tools +export MRSCRAPER_API_TOKEN="your-mrscraper-token" +``` + +Keep the token in the environment or a secret manager. Do not put it in a tool +argument, prompt, source file, trace, or task description. + +## Available tools + +Account: + +- `MrScraperGetAccountInfoTool` (`mrscraper_get_account_info`) + +Discovery: + +- `MrScraperCrawlWebsiteUrlsTool` (`mrscraper_crawl_website_urls`) +- `MrScraperSearchGoogleSerpTool` (`mrscraper_search_google_serp`) + +Extraction: + +- `MrScraperExtractPageByPromptTool` (`mrscraper_extract_page_by_prompt`) +- `MrScraperExtractListingsTool` (`mrscraper_extract_listings`) +- `MrScraperExtractStructuredDataTool` (`mrscraper_extract_structured_data`) +- `MrScraperFetchRenderedHtmlTool` (`mrscraper_fetch_rendered_html`) + +Results: + +- `MrScraperGetResultsTool` (`mrscraper_get_results`) +- `MrScraperGetLatestResultsTool` (`mrscraper_get_latest_results`) +- `MrScraperGetResultDetailTool` (`mrscraper_get_result_detail`) + +Scraper Creation: + +- `MrScraperCreatePromptScraperTool` (`mrscraper_create_prompt_scraper`) +- `MrScraperCreateListingScraperTool` (`mrscraper_create_listing_scraper`) +- `MrScraperCreateWebsiteCrawlScraperTool` (`mrscraper_create_website_crawl_scraper`) + +Scraper Runs: + +- `MrScraperRunExistingScraperTool` (`mrscraper_run_existing_scraper`) +- `MrScraperRunExistingScraperBatchTool` (`mrscraper_run_existing_scraper_batch`) + +## Direct and toolkit usage + +Use a single narrow tool when that is all an agent needs: + +```python +from crewai_tools import MrScraperExtractPageByPromptTool + +extract_product = MrScraperExtractPageByPromptTool() +result = extract_product.run( + url="https://example.com/products/123", + prompt="Extract the product name and current price", + output_schema={"name": "string", "price": "number"}, +) +``` + +The factory returns new independent tool instances. By default it returns all +15, configured with one shared HTTP client. Select case-insensitive groups or +exact public tool names when an agent should have a smaller capability set: + +```python +from crewai_tools import create_mrscraper_toolkit + +all_tools = create_mrscraper_toolkit() +read_tools = create_mrscraper_toolkit(groups=["Account", "Results"]) +selected = create_mrscraper_toolkit( + tool_names=[ + "mrscraper_search_google_serp", + "mrscraper_fetch_rendered_html", + ] +) +``` + +## Agent and Crew example + +```python +from crewai import Agent, Crew, Task +from crewai_tools import create_mrscraper_toolkit + +researcher = Agent( + role="Web researcher", + goal="Collect authorized public product information", + backstory="You make narrow, cost-aware scraping calls.", + tools=create_mrscraper_toolkit(groups=["Discovery", "Extraction"]), +) + +task = Task( + description="Find the relevant page and extract its product name and price.", + expected_output="A concise JSON-backed summary with the source URL.", + agent=researcher, +) + +result = Crew(agents=[researcher], tasks=[task]).kickoff() +``` + +## Return values and operational notes + +JSON objects, arrays, and scalar values are returned as deterministic compact +UTF-8 JSON text so they remain stable through Agents, Tasks, and Flows. HTML and +other plain-text responses are returned as the exact upstream string. The tools +provide synchronous `_run` implementations, matching comparable `requests`-based +integrations in this package; no duplicate async transport is maintained. + +Crawls, rendered browser calls, listing extraction, and batch runs can take time +and consume significant API allowance. Keep page counts and URL batches as small +as the task permits. POST requests are not retried automatically because they can +create duplicate jobs; API operation retry fields are passed only where the +MrScraper contract defines them. + +Only scrape content you are authorized to access. Review the target site's terms, +privacy requirements, robots policy, and applicable law before enabling automated +access, especially for login-protected or personal data. diff --git a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/__init__.py b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/__init__.py new file mode 100644 index 0000000000..679e42bd75 --- /dev/null +++ b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/__init__.py @@ -0,0 +1,48 @@ +"""Native MrScraper tools for CrewAI.""" + +from crewai_tools.tools.mrscraper.account import MrScraperGetAccountInfoTool +from crewai_tools.tools.mrscraper.discovery import ( + MrScraperCrawlWebsiteUrlsTool, + MrScraperSearchGoogleSerpTool, +) +from crewai_tools.tools.mrscraper.extraction import ( + MrScraperExtractListingsTool, + MrScraperExtractPageByPromptTool, + MrScraperExtractStructuredDataTool, + MrScraperFetchRenderedHtmlTool, +) +from crewai_tools.tools.mrscraper.results import ( + MrScraperGetLatestResultsTool, + MrScraperGetResultDetailTool, + MrScraperGetResultsTool, +) +from crewai_tools.tools.mrscraper.scraper_creation import ( + MrScraperCreateListingScraperTool, + MrScraperCreatePromptScraperTool, + MrScraperCreateWebsiteCrawlScraperTool, +) +from crewai_tools.tools.mrscraper.scraper_runs import ( + MrScraperRunExistingScraperBatchTool, + MrScraperRunExistingScraperTool, +) +from crewai_tools.tools.mrscraper.toolkit import create_mrscraper_toolkit + + +__all__ = [ + "MrScraperCrawlWebsiteUrlsTool", + "MrScraperCreateListingScraperTool", + "MrScraperCreatePromptScraperTool", + "MrScraperCreateWebsiteCrawlScraperTool", + "MrScraperExtractListingsTool", + "MrScraperExtractPageByPromptTool", + "MrScraperExtractStructuredDataTool", + "MrScraperFetchRenderedHtmlTool", + "MrScraperGetAccountInfoTool", + "MrScraperGetLatestResultsTool", + "MrScraperGetResultDetailTool", + "MrScraperGetResultsTool", + "MrScraperRunExistingScraperBatchTool", + "MrScraperRunExistingScraperTool", + "MrScraperSearchGoogleSerpTool", + "create_mrscraper_toolkit", +] diff --git a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/account.py b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/account.py new file mode 100644 index 0000000000..bec553d9ca --- /dev/null +++ b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/account.py @@ -0,0 +1,23 @@ +"""MrScraper account tool.""" + +from pydantic import BaseModel + +from crewai_tools.tools.mrscraper.base import MrScraperBaseTool +from crewai_tools.tools.mrscraper.schemas import GetAccountInfoInput + + +class MrScraperGetAccountInfoTool(MrScraperBaseTool): + """Retrieve subscription and token usage information.""" + + name: str = "mrscraper_get_account_info" + description: str = ( + "Use this narrow read-only tool to inspect MrScraper account details, " + "token usage, and token limits. It does not scrape a page or create a job." + ) + args_schema: type[BaseModel] = GetAccountInfoInput + + def _run(self) -> str: + return self._client.request("GET", "primary", "/api/v1/subscription-accounts") + + +__all__ = ["MrScraperGetAccountInfoTool"] diff --git a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/base.py b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/base.py new file mode 100644 index 0000000000..8e6146c7f2 --- /dev/null +++ b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/base.py @@ -0,0 +1,48 @@ +"""Base class and credential resolution for MrScraper tools.""" + +import os +from typing import Any + +from crewai.tools import BaseTool, EnvVar +from pydantic import Field, PrivateAttr + +from crewai_tools.tools.mrscraper.client import MrScraperClient + + +def resolve_api_token(api_token: str | None = None) -> str: + """Resolve a nonblank token without exposing its value in an error.""" + token = api_token if api_token is not None else os.getenv("MRSCRAPER_API_TOKEN") + if token is None or not token.strip(): + raise ValueError( + "MRSCRAPER_API_TOKEN is required; set it in the environment before " + "creating a MrScraper tool" + ) + return token + + +class MrScraperBaseTool(BaseTool): + """Base for public MrScraper tools with a private shared client.""" + + env_vars: list[EnvVar] = Field( + default_factory=lambda: [ + EnvVar( + name="MRSCRAPER_API_TOKEN", + description="MrScraper API token", + required=True, + ) + ] + ) + _client: MrScraperClient = PrivateAttr() + + def __init__( + self, + *, + api_token: str | None = None, + client: MrScraperClient | None = None, + **kwargs: Any, + ) -> None: + super().__init__(**kwargs) + self._client = client or MrScraperClient(resolve_api_token(api_token)) + + +__all__ = ["MrScraperBaseTool", "resolve_api_token"] diff --git a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/client.py b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/client.py new file mode 100644 index 0000000000..6018d53683 --- /dev/null +++ b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/client.py @@ -0,0 +1,100 @@ +"""Shared, secret-safe HTTP transport for MrScraper tools.""" + +import json +import re +from typing import Any, Literal + +import requests + + +Origin = Literal["primary", "serp", "rendered"] + +_ORIGINS: dict[Origin, str] = { + "primary": "https://api.app.mrscraper.com", + "serp": "https://sync.scraper.mrscraper.com", + "rendered": "https://api.mrscraper.com", +} +_TIMEOUT = (10, 660) +_ERROR_BODY_LIMIT = 1000 +_TOKEN_QUERY_RE = re.compile(r"([?&]token=)[^&\s]+", re.IGNORECASE) + + +class MrScraperClient: + """Make requests only to MrScraper's fixed API origins.""" + + def __init__(self, token: str, *, session: requests.Session | None = None) -> None: + if not token.strip(): + raise ValueError("MRSCRAPER_API_TOKEN must be a nonblank value") + self._token = token + self._session = session or requests.Session() + + def request( + self, + method: Literal["GET", "POST"], + origin: Origin, + path: str, + *, + params: dict[str, Any] | None = None, + json_body: dict[str, Any] | None = None, + force_text: bool = False, + ) -> str: + """Send one request and return deterministic JSON text or exact response text.""" + url = f"{_ORIGINS[origin]}{path}" + headers = self._headers(origin) + if origin == "rendered": + params = { + "token": self._token, + "browserRendering": "true", + **(params or {}), + } + try: + response = self._session.request( + method, + url, + headers=headers, + params=params, + json=json_body, + timeout=_TIMEOUT, + ) + except requests.RequestException as exc: + detail = self._sanitize(str(exc)) + raise RuntimeError(f"MrScraper request failed: {detail}") from None + + if not 200 <= response.status_code < 300: + sanitized = self._sanitize(response.text) + body = sanitized[:_ERROR_BODY_LIMIT] + suffix = "…" if len(sanitized) > _ERROR_BODY_LIMIT else "" + raise RuntimeError( + f"MrScraper API error (HTTP {response.status_code}): {body}{suffix}" + ) + + if force_text: + return self._sanitize(response.text) + + content_type = response.headers.get("Content-Type", "").lower() + if "json" not in content_type: + return self._sanitize(response.text) + try: + value = response.json() + except requests.JSONDecodeError: + return self._sanitize(response.text) + serialized = json.dumps(value, ensure_ascii=False, separators=(",", ":")) + return self._sanitize(serialized) + + def _headers(self, origin: Origin) -> dict[str, str]: + headers = { + "Accept": "application/json", + "Content-Type": "application/json", + } + if origin == "primary": + headers["x-api-token"] = self._token + elif origin == "serp": + headers["Authorization"] = f"Bearer {self._token}" + return headers + + def _sanitize(self, value: str) -> str: + redacted = value.replace(self._token, "[REDACTED]") + return _TOKEN_QUERY_RE.sub(r"\1[REDACTED]", redacted) + + +__all__ = ["MrScraperClient"] diff --git a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/discovery.py b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/discovery.py new file mode 100644 index 0000000000..3f29067631 --- /dev/null +++ b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/discovery.py @@ -0,0 +1,81 @@ +"""MrScraper discovery tools.""" + +from typing import Literal + +from pydantic import BaseModel + +from crewai_tools.tools.mrscraper.base import MrScraperBaseTool +from crewai_tools.tools.mrscraper.payloads import map_payload +from crewai_tools.tools.mrscraper.schemas import MapScraperInput, SearchGoogleSerpInput + + +class MrScraperCrawlWebsiteUrlsTool(MrScraperBaseTool): + """Discover URLs from a starting page.""" + + name: str = "mrscraper_crawl_website_urls" + description: str = ( + "Use this potentially expensive immediate Map crawl to discover website URLs. " + "Use the website-crawl creation tool when the intent is to create a reusable scraper." + ) + args_schema: type[BaseModel] = MapScraperInput + + def _run( + self, + url: str, + max_depth: int = 2, + max_pages: int = 50, + limit: int = 50, + include_patterns: str | None = None, + exclude_patterns: str | None = None, + ) -> str: + return self._client.request( + "POST", + "primary", + "/api/v1/scrapers-ai", + json_body=map_payload( + url=url, + max_depth=max_depth, + max_pages=max_pages, + limit=limit, + include_patterns=include_patterns, + exclude_patterns=exclude_patterns, + ), + ) + + +class MrScraperSearchGoogleSerpTool(MrScraperBaseTool): + """Search Google synchronously through MrScraper.""" + + name: str = "mrscraper_search_google_serp" + description: str = ( + "Use this for one narrow synchronous Google search. It returns compact JSON text " + "for JSON format or the exact upstream HTML string for HTML format." + ) + args_schema: type[BaseModel] = SearchGoogleSerpInput + + def _run( + self, + query: str, + region: str = "us", + language: str = "en", + page: int = 1, + format: Literal["json", "html"] = "json", + render_js: bool = False, + ) -> str: + return self._client.request( + "POST", + "serp", + "/api/google/serp/v2/sync", + json_body={ + "query": query, + "region": region, + "language": language, + "page": page, + "format": format, + "renderJs": render_js, + }, + force_text=format == "html", + ) + + +__all__ = ["MrScraperCrawlWebsiteUrlsTool", "MrScraperSearchGoogleSerpTool"] diff --git a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/extraction.py b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/extraction.py new file mode 100644 index 0000000000..229756f871 --- /dev/null +++ b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/extraction.py @@ -0,0 +1,186 @@ +"""MrScraper immediate extraction tools.""" + +from functools import lru_cache +import json +from pathlib import Path +from typing import Any, Literal, cast + +from pydantic import BaseModel + +from crewai_tools.tools.mrscraper.base import MrScraperBaseTool +from crewai_tools.tools.mrscraper.payloads import ( + general_payload, + listing_payload, + rendered_request, +) +from crewai_tools.tools.mrscraper.schemas import ( + ExtractStructuredDataInput, + FetchRenderedHtmlInput, + GeneralScraperInput, + ListingScraperInput, + ScrapingMode, + StructuredDataCategory, +) + + +@lru_cache(maxsize=1) +def load_structured_data_prompts() -> dict[str, str]: + """Load the byte-preserved n8n structured extraction presets.""" + path = Path(__file__).with_name("structured_data_prompts.json") + with path.open(encoding="utf-8") as preset_file: + return cast(dict[str, str], json.load(preset_file)) + + +class MrScraperExtractPageByPromptTool(MrScraperBaseTool): + """Perform immediate General extraction.""" + + name: str = "mrscraper_extract_page_by_prompt" + description: str = ( + "Use this for immediate AI extraction from one page using a prompt. " + "Use create_prompt_scraper when the primary intent is reusable scraper creation." + ) + args_schema: type[BaseModel] = GeneralScraperInput + + def _run( + self, + url: str, + prompt: str | None = None, + output_schema: dict[str, Any] | None = None, + mode: ScrapingMode = "Super", + proxy_country: str | None = None, + ) -> str: + return self._client.request( + "POST", + "primary", + "/api/v1/scrapers-ai", + json_body=general_payload( + url=url, + prompt=prompt, + output_schema=output_schema, + mode=mode, + proxy_country=proxy_country, + ), + ) + + +class MrScraperExtractListingsTool(MrScraperBaseTool): + """Perform immediate Listing extraction.""" + + name: str = "mrscraper_extract_listings" + description: str = ( + "Use this potentially multi-page immediate extraction for repeated listings or " + "paginated content. Use create_listing_scraper for reusable scraper creation." + ) + args_schema: type[BaseModel] = ListingScraperInput + + def _run( + self, + url: str, + prompt: str | None = None, + output_schema: dict[str, Any] | None = None, + max_pages: int = 1, + proxy_country: str | None = None, + ) -> str: + return self._client.request( + "POST", + "primary", + "/api/v1/scrapers-ai", + json_body=listing_payload( + url=url, + prompt=prompt, + output_schema=output_schema, + max_pages=max_pages, + proxy_country=proxy_country, + ), + ) + + +class MrScraperExtractStructuredDataTool(MrScraperBaseTool): + """Extract one of the bundled structured-data presets.""" + + name: str = "mrscraper_extract_structured_data" + description: str = ( + "Use this immediate extraction tool when a page matches one supported structured " + "category, such as article, product, hotel, job, property, restaurant, or tour." + ) + args_schema: type[BaseModel] = ExtractStructuredDataInput + + def _run( + self, + url: str, + category: StructuredDataCategory = "article", + mode: ScrapingMode = "Super", + proxy_country: str | None = None, + ) -> str: + payload: dict[str, Any] = { + "graph": "general", + "url": url, + "message": load_structured_data_prompts()[category], + "mode": mode, + } + if proxy_country is not None: + payload["proxyCountry"] = proxy_country + return self._client.request( + "POST", "primary", "/api/v1/scrapers-ai", json_body=payload + ) + + +class MrScraperFetchRenderedHtmlTool(MrScraperBaseTool): + """Fetch a browser-rendered page.""" + + name: str = "mrscraper_fetch_rendered_html" + description: str = ( + "Use this immediate stealth-browser call when JavaScript-rendered HTML, Markdown, " + "cookies, or a screenshot is needed. Keep the requested outputs narrow to control cost." + ) + args_schema: type[BaseModel] = FetchRenderedHtmlInput + + def _run( + self, + url: str, + max_retries: int = 3, + timeout: int = 300, + geo_code: str = "us", + proxy_country: str = "us", + screenshot: bool = False, + screenshot_mode: Literal["full", "top"] | None = None, + html: bool = True, + markdown: bool = False, + token_cap: int | None = None, + wait_for_selector: str | None = None, + wait_until: Literal["domcontentloaded", "load", "networkidle"] | None = None, + block_resources: bool = False, + home_page: bool = False, + return_cookie: bool = False, + super_mode: bool = False, + ) -> str: + params, body = rendered_request( + url=url, + max_retries=max_retries, + timeout=timeout, + geo_code=geo_code, + proxy_country=proxy_country, + screenshot=screenshot, + screenshot_mode=screenshot_mode, + html=html, + markdown=markdown, + token_cap=token_cap, + wait_for_selector=wait_for_selector, + wait_until=wait_until, + block_resources=block_resources, + home_page=home_page, + return_cookie=return_cookie, + super_mode=super_mode, + ) + return self._client.request( + "POST", "rendered", "/", params=params, json_body=body + ) + + +__all__ = [ + "MrScraperExtractListingsTool", + "MrScraperExtractPageByPromptTool", + "MrScraperExtractStructuredDataTool", + "MrScraperFetchRenderedHtmlTool", + "load_structured_data_prompts", +] diff --git a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/payloads.py b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/payloads.py new file mode 100644 index 0000000000..0c12cf05fc --- /dev/null +++ b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/payloads.py @@ -0,0 +1,212 @@ +"""Request payload builders shared by MrScraper operations.""" + +import json +from typing import Any + + +def _include_if_present( + payload: dict[str, Any], key: str, value: Any +) -> dict[str, Any]: + if value is not None: + payload[key] = value + return payload + + +def append_output_schema( + prompt: str | None, output_schema: dict[str, Any] | None, label: str +) -> str | None: + """Append a compact output schema exactly once using the API's label.""" + if output_schema is None: + return prompt + schema = json.dumps(output_schema, ensure_ascii=False, separators=(",", ":")) + schema_instruction = f"{label}\n{schema}" + return ( + f"{prompt}\n\n{schema_instruction}" + if prompt is not None + else schema_instruction + ) + + +def general_payload( + *, + url: str, + prompt: str | None, + output_schema: dict[str, Any] | None, + mode: str, + proxy_country: str | None, +) -> dict[str, Any]: + payload: dict[str, Any] = {"graph": "general", "url": url, "mode": mode} + message = append_output_schema( + prompt, output_schema, "Return the output as JSON matching this schema:" + ) + _include_if_present(payload, "message", message) + _include_if_present(payload, "proxyCountry", proxy_country) + return payload + + +def listing_payload( + *, + url: str, + prompt: str | None, + output_schema: dict[str, Any] | None, + max_pages: int, + proxy_country: str | None, +) -> dict[str, Any]: + payload: dict[str, Any] = { + "graph": "listing", + "url": url, + "maxPages": max_pages, + } + message = append_output_schema( + prompt, output_schema, "Return each item as JSON matching this schema:" + ) + _include_if_present(payload, "message", message) + _include_if_present(payload, "proxyCountry", proxy_country) + return payload + + +def map_payload( + *, + url: str, + max_depth: int, + max_pages: int, + limit: int, + include_patterns: str | None, + exclude_patterns: str | None, +) -> dict[str, Any]: + payload: dict[str, Any] = { + "graph": "map", + "url": url, + "maxDepth": max_depth, + "maxPages": max_pages, + "limit": limit, + } + _include_if_present(payload, "includePatterns", include_patterns) + _include_if_present(payload, "excludePatterns", exclude_patterns) + return payload + + +def rendered_request( + *, + url: str, + max_retries: int, + timeout: int, + geo_code: str, + proxy_country: str, + screenshot: bool, + screenshot_mode: str | None, + html: bool, + markdown: bool, + token_cap: int | None, + wait_for_selector: str | None, + wait_until: str | None, + block_resources: bool, + home_page: bool, + return_cookie: bool, + super_mode: bool, +) -> tuple[dict[str, Any], dict[str, Any]]: + def bool_text(value: bool) -> str: + return "true" if value else "false" + + params: dict[str, Any] = { + "timeout": timeout, + "geoCode": geo_code, + "html": bool_text(html), + "markdown": bool_text(markdown), + "proxyCountry": proxy_country, + } + if screenshot: + params["screenshot"] = screenshot_mode or "full" + _include_if_present(params, "waitForSelector", wait_for_selector) + _include_if_present(params, "waitUntil", wait_until) + if block_resources: + params["blockResources"] = bool_text(block_resources) + if return_cookie: + params["returnCookie"] = bool_text(return_cookie) + if super_mode: + params["super"] = bool_text(super_mode) + body = { + "url": url, + "maxRetries": max_retries, + } + _include_if_present(body, "tokenCap", token_cap) + if home_page: + body["homePage"] = home_page + return params, body + + +def existing_run_payload(values: dict[str, Any]) -> dict[str, Any]: + """Build an agent-specific rerun payload from validated public inputs.""" + payload: dict[str, Any] = { + "scraperId": values["scraper_id"], + "url": values["url"], + "maxRetry": values["max_retry"], + } + _include_if_present(payload, "proxyCountry", values.get("proxy_country")) + + if values["scraper_type"] == "manual": + mapping = { + "bypass_proxy": "bypassProxy", + "cookie_jar": "cookieJar", + "cookies": "cookies", + "home_page": "homePage", + "home_page_timeout": "homePageTimeout", + "html": "html", + "markdown": "markdown", + "paginator": "paginator", + "proxy": "proxy", + "record": "record", + "return_cookie": "returnCookie", + "stream": "stream", + "timeout": "timeout", + "token_cap": "tokenCap", + } + for source, target in mapping.items(): + _include_if_present(payload, target, values.get(source)) + screenshot = values.get("screenshot") + if screenshot is not None: + payload["screenshot"] = "true" if screenshot else "false" + return payload + + agent_type = values["agent_type"] + if agent_type == "map": + for source, target in { + "max_depth": "maxDepth", + "max_pages": "maxPages", + "limit": "limit", + "include_patterns": "includePatterns", + "exclude_patterns": "excludePatterns", + }.items(): + _include_if_present(payload, target, values.get(source)) + return payload + + ai_mapping = { + "bypass_proxy": "bypassProxy", + "html": "html", + "markdown": "markdown", + "render_javascript": "renderJavascript", + "return_cookies": "returnCookies", + "screenshot": "screenshot", + "use_home_page": "useHomePage", + "wait_for_selector": "waitForSelector", + } + for source, target in ai_mapping.items(): + _include_if_present(payload, target, values.get(source)) + if agent_type == "listing": + for source, target in { + "max_pages": "maxPages", + "timeout": "timeout", + "stream": "stream", + }.items(): + _include_if_present(payload, target, values.get(source)) + return payload + + +__all__ = [ + "append_output_schema", + "existing_run_payload", + "general_payload", + "listing_payload", + "map_payload", + "rendered_request", +] diff --git a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/results.py b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/results.py new file mode 100644 index 0000000000..1a74a44023 --- /dev/null +++ b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/results.py @@ -0,0 +1,92 @@ +"""MrScraper result retrieval tools.""" + +from typing import Literal +from urllib.parse import quote + +from pydantic import BaseModel + +from crewai_tools.tools.mrscraper.base import MrScraperBaseTool +from crewai_tools.tools.mrscraper.schemas import ( + GetLatestResultsInput, + GetResultDetailInput, + GetResultsInput, +) + + +class MrScraperGetResultsTool(MrScraperBaseTool): + """Retrieve a configurable page of results.""" + + name: str = "mrscraper_get_results" + description: str = ( + "Use this to page through results for one scraper with explicit paging and sort " + "controls. Use get_latest_results when only the newest N records are needed." + ) + args_schema: type[BaseModel] = GetResultsInput + + def _run( + self, + scraper_id: str, + page: int = 1, + page_size: int = 10, + sort_by: Literal["createdAt"] = "createdAt", + sort_order: Literal["ASC", "DESC"] = "DESC", + ) -> str: + return self._client.request( + "GET", + "primary", + "/api/v1/results", + params={ + "filters[scraperId]": scraper_id, + "page": page, + "pageSize": page_size, + "sort": sort_by, + "sortOrder": sort_order, + }, + ) + + +class MrScraperGetLatestResultsTool(MrScraperBaseTool): + """Retrieve the newest N results.""" + + name: str = "mrscraper_get_latest_results" + description: str = ( + "Use this shortcut for the newest N results from one scraper. Use get_results " + "instead when page navigation or ascending order is required." + ) + args_schema: type[BaseModel] = GetLatestResultsInput + + def _run(self, scraper_id: str, count: int = 10) -> str: + return self._client.request( + "GET", + "primary", + "/api/v1/results", + params={ + "filters[scraperId]": scraper_id, + "page": 1, + "pageSize": count, + "sort": "createdAt", + "sortOrder": "DESC", + }, + ) + + +class MrScraperGetResultDetailTool(MrScraperBaseTool): + """Retrieve one result by ID.""" + + name: str = "mrscraper_get_result_detail" + description: str = ( + "Use this narrow lookup when a specific MrScraper result ID is already known. " + "It returns that record rather than a paginated collection." + ) + args_schema: type[BaseModel] = GetResultDetailInput + + def _run(self, result_id: str) -> str: + encoded_id = quote(result_id, safe="") + return self._client.request("GET", "primary", f"/api/v1/results/{encoded_id}") + + +__all__ = [ + "MrScraperGetLatestResultsTool", + "MrScraperGetResultDetailTool", + "MrScraperGetResultsTool", +] diff --git a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/schemas.py b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/schemas.py new file mode 100644 index 0000000000..ea23df3e78 --- /dev/null +++ b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/schemas.py @@ -0,0 +1,427 @@ +"""Pydantic input schemas for MrScraper tools.""" + +from __future__ import annotations + +from typing import Annotated, Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, StringConstraints, model_validator + + +NonBlankStr = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] +TwoLetterCode = Annotated[ + str, StringConstraints(strip_whitespace=True, pattern=r"^[A-Za-z]{2}$") +] +StrictInt = Annotated[int, Field(strict=True)] +NonNegativeInt = Annotated[int, Field(strict=True, ge=0)] +PositiveInt = Annotated[int, Field(strict=True, ge=1)] + +ScrapingMode = Literal["Super", "Cheap"] +StructuredDataCategory = Literal[ + "article", + "forumThread", + "hotel", + "jobPosting", + "post", + "product", + "property", + "restaurant", + "socialMediaProfile", + "tourAttraction", +] + + +class MrScraperInput(BaseModel): + """Base schema that rejects undocumented arguments.""" + + model_config = ConfigDict(extra="forbid") + + +class GetAccountInfoInput(MrScraperInput): + """The account operation has no model-supplied inputs.""" + + +class MapScraperInput(MrScraperInput): + """Inputs shared by immediate and reusable website crawl tools.""" + + url: NonBlankStr = Field(description="Required starting URL to crawl.") + max_depth: StrictInt = Field( + default=2, description="Maximum link depth to crawl; defaults to 2." + ) + max_pages: StrictInt = Field( + default=50, description="Maximum pages to evaluate; defaults to 50." + ) + limit: PositiveInt = Field( + default=50, description="Maximum URLs to return; defaults to 50; minimum 1." + ) + include_patterns: NonBlankStr | None = Field( + default=None, + description="Optional pipe-separated regular expressions for URLs to include.", + ) + exclude_patterns: NonBlankStr | None = Field( + default=None, + description="Optional pipe-separated regular expressions for URLs to exclude.", + ) + + +class SearchGoogleSerpInput(MrScraperInput): + """Inputs for synchronous Google SERP search.""" + + query: NonBlankStr = Field(description="Required Google search query.") + region: TwoLetterCode = Field( + default="us", description="Two-letter result region code; defaults to 'us'." + ) + language: TwoLetterCode = Field( + default="en", description="Two-letter result language code; defaults to 'en'." + ) + page: PositiveInt = Field( + default=1, description="Google result page number; defaults to 1; minimum 1." + ) + format: Literal["json", "html"] = Field( + default="json", + description="Response format: 'json' or 'html'; defaults to 'json'.", + ) + render_js: bool = Field( + default=False, + description="Whether to render JavaScript before collecting results; defaults to false.", + ) + + +class GeneralScraperInput(MrScraperInput): + """Inputs shared by immediate and reusable prompt scrapers.""" + + url: NonBlankStr = Field(description="Required page URL to scrape.") + prompt: NonBlankStr | None = Field( + default=None, description="Optional extraction instructions for the AI scraper." + ) + output_schema: dict[str, Any] | None = Field( + default=None, + description="Optional JSON object describing the expected output shape.", + ) + mode: ScrapingMode = Field( + default="Super", + description="Scraping mode, 'Super' or 'Cheap'; defaults to 'Super'.", + ) + proxy_country: TwoLetterCode | None = Field( + default=None, description="Optional ISO country code for the proxy." + ) + + +class ListingScraperInput(MrScraperInput): + """Inputs shared by immediate and reusable listing scrapers.""" + + url: NonBlankStr = Field(description="Required listing page URL to scrape.") + prompt: NonBlankStr | None = Field( + default=None, + description="Optional instructions describing each listing item to extract.", + ) + output_schema: dict[str, Any] | None = Field( + default=None, + description="Optional JSON object describing each expected listing item.", + ) + max_pages: PositiveInt = Field( + default=1, + description="Maximum pagination pages to scrape; defaults to 1; minimum 1.", + ) + proxy_country: TwoLetterCode | None = Field( + default=None, description="Optional ISO country code for the proxy." + ) + + +class ExtractStructuredDataInput(MrScraperInput): + """Inputs for preset structured-data extraction.""" + + url: NonBlankStr = Field(description="Required page URL to scrape.") + category: StructuredDataCategory = Field( + default="article", + description="Structured extraction preset category; defaults to 'article'.", + ) + mode: ScrapingMode = Field( + default="Super", + description="Scraping mode, 'Super' or 'Cheap'; defaults to 'Super'.", + ) + proxy_country: TwoLetterCode | None = Field( + default=None, description="Optional ISO country code for the proxy." + ) + + +class FetchRenderedHtmlInput(MrScraperInput): + """Inputs for the rendered-page API.""" + + url: NonBlankStr = Field(description="Required target URL to render.") + max_retries: NonNegativeInt = Field( + default=3, description="Maximum retry attempts; defaults to 3; minimum 0." + ) + timeout: PositiveInt = Field( + default=300, + description="Maximum page-load time in seconds; defaults to 300; minimum 1.", + ) + geo_code: TwoLetterCode = Field( + default="us", description="Geolocation country code; defaults to 'us'." + ) + proxy_country: TwoLetterCode = Field( + default="us", description="Proxy country code; defaults to 'us'." + ) + screenshot: bool = Field( + default=False, description="Whether to capture a screenshot; defaults to false." + ) + screenshot_mode: Literal["full", "top"] | None = Field( + default=None, + description="Optional screenshot mode; used only when screenshot is true.", + ) + html: bool = Field( + default=True, description="Whether to include rendered HTML; defaults to true." + ) + markdown: bool = Field( + default=False, description="Whether to include Markdown; defaults to false." + ) + token_cap: PositiveInt | None = Field( + default=None, + description="Optional maximum processing token allowance; minimum 1.", + ) + wait_for_selector: NonBlankStr | None = Field( + default=None, description="Optional CSS selector to await before returning." + ) + wait_until: Literal["domcontentloaded", "load", "networkidle"] | None = Field( + default=None, + description="Optional browser lifecycle event to await.", + ) + block_resources: bool = Field( + default=False, + description="Whether to block images, fonts, and stylesheets; defaults to false.", + ) + home_page: bool = Field( + default=False, + description="Whether to visit the site home page first; defaults to false.", + ) + return_cookie: bool = Field( + default=False, + description="Whether to include browser cookies; defaults to false.", + ) + super_mode: bool = Field( + default=False, + description="Whether to use stronger device mode; defaults to false.", + ) + + +class GetResultsInput(MrScraperInput): + """Inputs for paginated scraper results.""" + + scraper_id: NonBlankStr = Field( + description="Required scraper ID whose results to list." + ) + page: StrictInt = Field( + default=1, description="Results page number; defaults to 1." + ) + page_size: StrictInt = Field( + default=10, description="Number of results per page; defaults to 10." + ) + sort_by: Literal["createdAt"] = Field( + default="createdAt", description="Sort field; only 'createdAt' is supported." + ) + sort_order: Literal["ASC", "DESC"] = Field( + default="DESC", + description="Sort direction, 'ASC' or 'DESC'; defaults to 'DESC'.", + ) + + +class GetLatestResultsInput(MrScraperInput): + """Inputs for the newest scraper results.""" + + scraper_id: NonBlankStr = Field( + description="Required scraper ID whose newest results to list." + ) + count: StrictInt = Field( + default=10, description="Number of newest results; defaults to 10." + ) + + +class GetResultDetailInput(MrScraperInput): + """Inputs for one result record.""" + + result_id: NonBlankStr = Field(description="Required result ID to retrieve.") + + +class RunExistingScraperInput(MrScraperInput): + """Stable conditional schema for AI and manual single scraper runs.""" + + scraper_type: Literal["ai", "manual"] = Field( + description="Required scraper kind selecting the AI or manual endpoint." + ) + scraper_id: NonBlankStr = Field(description="Required existing scraper ID.") + url: NonBlankStr = Field(description="Required URL to process in this run.") + max_retry: NonNegativeInt = Field( + default=3, description="Maximum retry attempts; defaults to 3; minimum 0." + ) + proxy_country: TwoLetterCode | None = Field( + default=None, description="Optional proxy country code." + ) + agent_type: Literal["general", "listing", "map"] = Field( + default="general", + description="AI agent type; defaults to 'general' for AI and is forbidden for manual runs.", + ) + + bypass_proxy: bool | None = Field( + default=None, + description="General/Listing default false; Manual default true; forbidden for Map.", + ) + html: bool | None = Field( + default=None, + description="Optional General, Listing, or Manual HTML output flag.", + ) + markdown: bool | None = Field( + default=None, + description="Optional General, Listing, or Manual Markdown output flag.", + ) + screenshot: bool | None = Field( + default=None, + description="Optional General, Listing, or Manual screenshot flag.", + ) + stream: bool | None = Field( + default=None, + description="Optional Listing or Manual streaming flag.", + ) + timeout: PositiveInt | None = Field( + default=None, + description="Listing timeout defaults to 300; Manual timeout defaults to 600; minimum 1.", + ) + + render_javascript: bool | None = Field( + default=None, + description="Optional General/Listing JavaScript rendering flag.", + ) + return_cookies: bool | None = Field( + default=None, + description="Optional General/Listing cookie-return flag.", + ) + use_home_page: bool | None = Field( + default=None, + description="Optional General/Listing home-page visit flag.", + ) + wait_for_selector: NonBlankStr | None = Field( + default=None, description="Optional General/Listing CSS selector to await." + ) + + max_pages: PositiveInt | None = Field( + default=None, + description="Listing defaults to 5; Map defaults to 50; minimum 1.", + ) + + max_depth: NonNegativeInt | None = Field( + default=None, description="Optional Map crawl depth; minimum 0." + ) + limit: PositiveInt | None = Field( + default=None, description="Optional Map result limit; minimum 1." + ) + include_patterns: NonBlankStr | None = Field( + default=None, description="Optional Map include-pattern expressions." + ) + exclude_patterns: NonBlankStr | None = Field( + default=None, description="Optional Map exclude-pattern expressions." + ) + + cookie_jar: NonBlankStr | None = Field( + default=None, description="Optional Manual cookie-jar identifier or value." + ) + cookies: list[dict[str, Any]] | None = Field( + default=None, + description="Optional Manual browser-cookie objects.", + ) + home_page: bool | None = Field( + default=None, description="Optional Manual home-page visit flag." + ) + home_page_timeout: PositiveInt | None = Field( + default=None, description="Optional Manual home-page timeout; minimum 1." + ) + paginator: dict[str, Any] | None = Field( + default=None, + description="Optional Manual paginator configuration.", + ) + proxy: NonBlankStr | None = Field( + default=None, description="Optional Manual proxy URL." + ) + record: bool | None = Field( + default=None, + description="Optional Manual browser-session recording flag.", + ) + return_cookie: bool | None = Field( + default=None, description="Optional Manual cookie-return flag." + ) + token_cap: NonNegativeInt | None = Field( + default=None, description="Optional Manual token cap; minimum 0." + ) + + @model_validator(mode="after") + def validate_conditional_fields(self) -> RunExistingScraperInput: + """Reject explicitly supplied fields that do not apply to the selected run.""" + supplied = self.model_fields_set + common = {"scraper_type", "scraper_id", "url", "max_retry", "proxy_country"} + general = { + "agent_type", + "bypass_proxy", + "html", + "markdown", + "render_javascript", + "return_cookies", + "screenshot", + "use_home_page", + "wait_for_selector", + } + listing = general | {"max_pages", "timeout", "stream"} + mapping = { + "agent_type", + "max_depth", + "max_pages", + "limit", + "include_patterns", + "exclude_patterns", + } + manual = { + "bypass_proxy", + "cookie_jar", + "cookies", + "home_page", + "home_page_timeout", + "html", + "markdown", + "paginator", + "proxy", + "record", + "return_cookie", + "screenshot", + "stream", + "timeout", + "token_cap", + } + + if self.scraper_type == "manual": + incompatible = supplied - common - manual + if incompatible: + names = ", ".join(sorted(incompatible)) + raise ValueError(f"Manual scraper runs do not accept: {names}") + return self + + allowed = { + "general": general, + "listing": listing, + "map": mapping, + }[self.agent_type] + incompatible = supplied - common - allowed + if incompatible: + names = ", ".join(sorted(incompatible)) + raise ValueError( + f"AI {self.agent_type} scraper runs do not accept: {names}" + ) + return self + + +class RunExistingScraperBatchInput(MrScraperInput): + """Inputs for batch runs of an existing scraper.""" + + scraper_type: Literal["ai", "manual"] = Field( + description="Required scraper kind selecting the AI or manual bulk endpoint." + ) + scraper_id: NonBlankStr = Field(description="Required existing scraper ID.") + urls: list[NonBlankStr] = Field( + min_length=1, + description="Required nonempty array of nonblank URLs to process in this batch.", + ) diff --git a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/scraper_creation.py b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/scraper_creation.py new file mode 100644 index 0000000000..435ee39faf --- /dev/null +++ b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/scraper_creation.py @@ -0,0 +1,123 @@ +"""MrScraper reusable scraper creation tools.""" + +from typing import Any + +from pydantic import BaseModel + +from crewai_tools.tools.mrscraper.base import MrScraperBaseTool +from crewai_tools.tools.mrscraper.payloads import ( + general_payload, + listing_payload, + map_payload, +) +from crewai_tools.tools.mrscraper.schemas import ( + GeneralScraperInput, + ListingScraperInput, + MapScraperInput, + ScrapingMode, +) + + +class MrScraperCreatePromptScraperTool(MrScraperBaseTool): + """Create a reusable General AI scraper.""" + + name: str = "mrscraper_create_prompt_scraper" + description: str = ( + "Use this to create a reusable General AI scraper from a page, prompt, and optional " + "output schema. Use extract_page_by_prompt for immediate one-page extraction intent." + ) + args_schema: type[BaseModel] = GeneralScraperInput + + def _run( + self, + url: str, + prompt: str | None = None, + output_schema: dict[str, Any] | None = None, + mode: ScrapingMode = "Super", + proxy_country: str | None = None, + ) -> str: + return self._client.request( + "POST", + "primary", + "/api/v1/scrapers-ai", + json_body=general_payload( + url=url, + prompt=prompt, + output_schema=output_schema, + mode=mode, + proxy_country=proxy_country, + ), + ) + + +class MrScraperCreateListingScraperTool(MrScraperBaseTool): + """Create a reusable Listing AI scraper.""" + + name: str = "mrscraper_create_listing_scraper" + description: str = ( + "Use this to create a reusable Listing AI scraper for repeated or paginated items. " + "Use extract_listings when the intent is immediate extraction." + ) + args_schema: type[BaseModel] = ListingScraperInput + + def _run( + self, + url: str, + prompt: str | None = None, + output_schema: dict[str, Any] | None = None, + max_pages: int = 1, + proxy_country: str | None = None, + ) -> str: + return self._client.request( + "POST", + "primary", + "/api/v1/scrapers-ai", + json_body=listing_payload( + url=url, + prompt=prompt, + output_schema=output_schema, + max_pages=max_pages, + proxy_country=proxy_country, + ), + ) + + +class MrScraperCreateWebsiteCrawlScraperTool(MrScraperBaseTool): + """Create a reusable Map AI scraper.""" + + name: str = "mrscraper_create_website_crawl_scraper" + description: str = ( + "Use this potentially expensive operation to create a reusable Map scraper for URL " + "discovery. Use crawl_website_urls for immediate crawl intent." + ) + args_schema: type[BaseModel] = MapScraperInput + + def _run( + self, + url: str, + max_depth: int = 2, + max_pages: int = 50, + limit: int = 50, + include_patterns: str | None = None, + exclude_patterns: str | None = None, + ) -> str: + return self._client.request( + "POST", + "primary", + "/api/v1/scrapers-ai", + json_body=map_payload( + url=url, + max_depth=max_depth, + max_pages=max_pages, + limit=limit, + include_patterns=include_patterns, + exclude_patterns=exclude_patterns, + ), + ) + + +__all__ = [ + "MrScraperCreateListingScraperTool", + "MrScraperCreatePromptScraperTool", + "MrScraperCreateWebsiteCrawlScraperTool", +] diff --git a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/scraper_runs.py b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/scraper_runs.py new file mode 100644 index 0000000000..976f65874f --- /dev/null +++ b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/scraper_runs.py @@ -0,0 +1,72 @@ +"""MrScraper existing-scraper run tools.""" + +from typing import Any, Literal + +from pydantic import BaseModel + +from crewai_tools.tools.mrscraper.base import MrScraperBaseTool +from crewai_tools.tools.mrscraper.payloads import existing_run_payload +from crewai_tools.tools.mrscraper.schemas import ( + RunExistingScraperBatchInput, + RunExistingScraperInput, +) + + +class MrScraperRunExistingScraperTool(MrScraperBaseTool): + """Run an existing AI or manual scraper on one URL.""" + + name: str = "mrscraper_run_existing_scraper" + description: str = ( + "Use this to run one URL through an existing AI or manual scraper. Choose the AI " + "agent type carefully; conditional options are validated before any request." + ) + args_schema: type[BaseModel] = RunExistingScraperInput + + def _run(self, **values: Any) -> str: + scraper_type = values["scraper_type"] + endpoint = ( + "/api/v1/scrapers-manual-rerun" + if scraper_type == "manual" + else "/api/v1/scrapers-ai-rerun" + ) + return self._client.request( + "POST", + "primary", + endpoint, + json_body=existing_run_payload(values), + ) + + +class MrScraperRunExistingScraperBatchTool(MrScraperBaseTool): + """Run an existing AI or manual scraper on a URL batch.""" + + name: str = "mrscraper_run_existing_scraper_batch" + description: str = ( + "Use this potentially expensive batch operation to run multiple URLs through one " + "existing AI or manual scraper. Use run_existing_scraper for a single URL." + ) + args_schema: type[BaseModel] = RunExistingScraperBatchInput + + def _run( + self, + scraper_type: Literal["ai", "manual"], + scraper_id: str, + urls: list[str], + ) -> str: + base = ( + "/api/v1/scrapers-manual-rerun" + if scraper_type == "manual" + else "/api/v1/scrapers-ai-rerun" + ) + return self._client.request( + "POST", + "primary", + f"{base}/bulk", + json_body={"scraperId": scraper_id, "urls": urls}, + ) + + +__all__ = [ + "MrScraperRunExistingScraperBatchTool", + "MrScraperRunExistingScraperTool", +] diff --git a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/structured_data_prompts.json b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/structured_data_prompts.json new file mode 100644 index 0000000000..bf00cccc1e --- /dev/null +++ b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/structured_data_prompts.json @@ -0,0 +1 @@ +{"article":"\nPlease extract the data by following the schema:\n{\n \"fields\": [\n {\n \"name\": \"headline\",\n \"type\": \"string\",\n \"description\": \"Article headline or title\"\n },\n {\n \"name\": \"articleBody\",\n \"type\": \"string\",\n \"description\": \"Full text content of the article\"\n },\n {\n \"name\": \"articleBodyHtml\",\n \"type\": \"string\",\n \"description\": \"HTML markup of the article body\"\n },\n {\n \"name\": \"description\",\n \"type\": \"string\",\n \"description\": \"Short summary or description of the article\"\n },\n {\n \"name\": \"datePublished\",\n \"type\": \"string\",\n \"description\": \"Publication date in ISO 8601 format\"\n },\n {\n \"name\": \"datePublishedRaw\",\n \"type\": \"string\",\n \"description\": \"Publication date as displayed on the page\"\n },\n {\n \"name\": \"dateModified\",\n \"type\": \"string\",\n \"description\": \"Last modified date in ISO 8601 format\"\n },\n {\n \"name\": \"dateModifiedRaw\",\n \"type\": \"string\",\n \"description\": \"Last modified date as displayed on the page\"\n },\n {\n \"name\": \"authors\",\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"List of article authors\",\n \"item_structure\": {\n \"name\": {\n \"type\": \"string\",\n \"description\": \"Parsed individual author name\"\n },\n \"nameRaw\": {\n \"type\": \"string\",\n \"description\": \"Raw author name as displayed on the page\"\n }\n }\n },\n {\n \"name\": \"inLanguage\",\n \"type\": \"string\",\n \"description\": \"Language code of the article (e.g., en)\"\n },\n {\n \"name\": \"breadcrumbs\",\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"Navigation breadcrumb trail\",\n \"item_structure\": {\n \"url\": {\n \"type\": \"string\",\n \"description\": \"Breadcrumb link URL\"\n },\n \"name\": {\n \"type\": \"string\",\n \"description\": \"Breadcrumb label\"\n }\n }\n },\n {\n \"name\": \"mainImage\",\n \"type\": \"object\",\n \"structure\": {\n \"url\": {\n \"type\": \"string\",\n \"description\": \"URL of the main image\"\n }\n },\n \"description\": \"Primary image of the article\"\n },\n {\n \"name\": \"images\",\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"All images found in the article\",\n \"item_structure\": {\n \"url\": {\n \"type\": \"string\",\n \"description\": \"Image URL\"\n }\n }\n },\n {\n \"name\": \"videos\",\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"All videos found in the article\",\n \"item_structure\": {\n \"url\": {\n \"type\": \"string\",\n \"description\": \"Video URL\"\n }\n }\n },\n {\n \"name\": \"audios\",\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"All audio files found in the article\",\n \"item_structure\": {\n \"url\": {\n \"type\": \"string\",\n \"description\": \"Audio URL\"\n }\n }\n },\n {\n \"name\": \"url\",\n \"type\": \"string\",\n \"description\": \"URL of the article page\"\n },\n {\n \"name\": \"canonicalUrl\",\n \"type\": \"string\",\n \"description\": \"Canonical URL of the article\"\n }\n ]\n}\n","forumThread":"\nPlease extract the data by following the schema:\n{\n \"fields\": [\n {\n \"name\": \"topic\",\n \"type\": \"object\",\n \"structure\": {\n \"name\": {\n \"type\": \"string\",\n \"description\": \"Topic title or question\"\n }\n },\n \"description\": \"Forum thread topic\"\n },\n {\n \"name\": \"posts\",\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"List of posts in the thread\",\n \"item_structure\": {\n \"text\": {\n \"type\": \"string\",\n \"description\": \"Post text content\"\n },\n \"reactions\": {\n \"type\": \"object\",\n \"structure\": {\n \"likes\": {\n \"type\": \"number\",\n \"description\": \"Number of likes\"\n },\n \"replies\": {\n \"type\": \"number\",\n \"description\": \"Number of replies\"\n }\n },\n \"description\": \"Reaction counts for the post\"\n },\n \"datePublished\": {\n \"type\": \"string\",\n \"description\": \"Post date in ISO 8601 format\"\n },\n \"datePublishedRaw\": {\n \"type\": \"string\",\n \"description\": \"Post date as displayed on the page\"\n }\n }\n },\n {\n \"name\": \"url\",\n \"type\": \"string\",\n \"description\": \"URL of the forum thread\"\n }\n ]\n}\n","hotel":"\nPlease extract the data by following the schema:\n{\n \"fields\": [\n {\n \"name\": \"name\",\n \"type\": \"string\",\n \"description\": \"Hotel or accommodation name\"\n },\n {\n \"name\": \"about\",\n \"type\": \"string\",\n \"description\": \"Description and overview of the property\"\n },\n {\n \"name\": \"address\",\n \"type\": \"string\",\n \"description\": \"Full street address\"\n },\n {\n \"name\": \"location\",\n \"type\": \"string\",\n \"description\": \"City, area, or region name\"\n },\n {\n \"name\": \"coordinates\",\n \"type\": \"object\",\n \"structure\": {\n \"latitude\": {\n \"type\": \"number\",\n \"description\": \"Latitude\"\n },\n \"longitude\": {\n \"type\": \"number\",\n \"description\": \"Longitude\"\n }\n },\n \"description\": \"Geographic coordinates\"\n },\n {\n \"name\": \"stars\",\n \"type\": \"number\",\n \"description\": \"Star rating of the property (1-5)\"\n },\n {\n \"name\": \"year_opened\",\n \"type\": \"number\",\n \"description\": \"Year the property was opened\"\n },\n {\n \"name\": \"check_in\",\n \"type\": \"string\",\n \"description\": \"Check-in time instructions\"\n },\n {\n \"name\": \"check_out\",\n \"type\": \"string\",\n \"description\": \"Check-out time instructions\"\n },\n {\n \"name\": \"contact\",\n \"type\": \"object\",\n \"structure\": {\n \"email\": {\n \"type\": \"string\",\n \"description\": \"Contact email\"\n },\n \"phone\": {\n \"type\": \"string\",\n \"description\": \"Contact phone number\"\n }\n },\n \"description\": \"Property contact information\"\n },\n {\n \"name\": \"host\",\n \"type\": \"object\",\n \"structure\": {\n \"name\": {\n \"type\": \"string\",\n \"description\": \"Host name\"\n },\n \"joined\": {\n \"type\": \"string\",\n \"description\": \"Date joined\"\n },\n \"response_time\": {\n \"type\": \"string\",\n \"description\": \"Average response time\"\n }\n },\n \"description\": \"Host information (for homestays/apartments)\"\n },\n {\n \"name\": \"pricing\",\n \"type\": \"object\",\n \"structure\": {\n \"currency\": {\n \"type\": \"string\",\n \"description\": \"Currency code\"\n },\n \"original_price\": {\n \"type\": \"string\",\n \"description\": \"Price before discount\"\n },\n \"starting_price\": {\n \"type\": \"string\",\n \"description\": \"Lowest price available\"\n },\n \"discounted_price\": {\n \"type\": \"string\",\n \"description\": \"Price after discount\"\n }\n },\n \"description\": \"General pricing info\"\n },\n {\n \"name\": \"review_stats\",\n \"type\": \"object\",\n \"structure\": {\n \"overall_score\": {\n \"type\": \"number\",\n \"description\": \"Overall numeric rating\"\n },\n \"total_reviews\": {\n \"type\": \"number\",\n \"description\": \"Total count of reviews\"\n },\n \"score_breakdown\": {\n \"type\": \"object\",\n \"structure\": {},\n \"description\": \"Scores by category\"\n },\n \"category_summaries\": {\n \"type\": \"object\",\n \"structure\": {},\n \"description\": \"Textual summary of reviews by category\"\n }\n },\n \"description\": \"Aggregated review statistics\"\n },\n {\n \"name\": \"highlights\",\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Key selling points or property highlights\"\n },\n {\n \"name\": \"review_highlights\",\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Snippets of positive feedback from reviews\"\n },\n {\n \"name\": \"reviews\",\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"Individual guest reviews\",\n \"item_structure\": {\n \"date\": {\n \"type\": \"string\",\n \"description\": \"Date of stay or review\"\n },\n \"title\": {\n \"type\": \"string\",\n \"description\": \"Review title\"\n },\n \"rating\": {\n \"type\": \"number\",\n \"description\": \"Rating given\"\n },\n \"content\": {\n \"type\": \"string\",\n \"description\": \"Review content\"\n },\n \"country\": {\n \"type\": \"string\",\n \"description\": \"Reviewer'''s country\"\n },\n \"reviewer\": {\n \"type\": \"string\",\n \"description\": \"Name of reviewer\"\n },\n \"room_type\": {\n \"type\": \"string\",\n \"description\": \"Room type stayed in\"\n },\n \"trip_type\": {\n \"type\": \"string\",\n \"description\": \"Type of trip (Business, Couple, etc.)\"\n },\n \"stay_duration\": {\n \"type\": \"string\",\n \"description\": \"Duration of stay\"\n }\n }\n },\n {\n \"name\": \"popular_facilities\",\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Flat list of popular facilities or amenities\"\n },\n {\n \"name\": \"facilities\",\n \"type\": \"object\",\n \"structure\": {\n \"dining\": {\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Dining options\"\n },\n \"internet\": {\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Internet amenities\"\n },\n \"services\": {\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Hotel services\"\n },\n \"recreation\": {\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Recreation and sports\"\n },\n \"room_amenities\": {\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"In-room features\"\n },\n \"transportation\": {\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Transport facilities\"\n },\n \"access_and_security\": {\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Security features\"\n },\n \"cleanliness_and_safety\": {\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Sanitation protocols\"\n }\n },\n \"description\": \"Facilities grouped by category\"\n },\n {\n \"name\": \"policies\",\n \"type\": \"object\",\n \"structure\": {\n \"pets\": {\n \"type\": \"string\",\n \"description\": \"Pet policy\"\n },\n \"deposit\": {\n \"type\": \"string\",\n \"description\": \"Deposit requirements\"\n },\n \"smoking\": {\n \"type\": \"string\",\n \"description\": \"Smoking policy\"\n },\n \"children\": {\n \"type\": \"string\",\n \"description\": \"Child policy\"\n },\n \"breakfast\": {\n \"type\": \"string\",\n \"description\": \"Breakfast availability/hours\"\n },\n \"cancellation\": {\n \"type\": \"string\",\n \"description\": \"Cancellation rules\"\n },\n \"check_in_instructions\": {\n \"type\": \"string\",\n \"description\": \"Detailed check-in rules\"\n }\n },\n \"description\": \"Property rules and policies\"\n },\n {\n \"name\": \"room_types\",\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"Available room configurations\",\n \"item_structure\": {\n \"name\": {\n \"type\": \"string\",\n \"description\": \"Room type name\"\n },\n \"size\": {\n \"type\": \"string\",\n \"description\": \"Room dimensions\"\n },\n \"prices\": {\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"Price options\",\n \"item_structure\": {\n \"kids\": {\n \"type\": \"number\",\n \"description\": \"Child capacity\"\n },\n \"adults\": {\n \"type\": \"number\",\n \"description\": \"Adult capacity\"\n },\n \"cashback\": {\n \"type\": \"string\",\n \"description\": \"Potential cashback\"\n },\n \"price_total\": {\n \"type\": \"string\",\n \"description\": \"Final price\"\n },\n \"price_breakdown\": {\n \"type\": \"string\",\n \"description\": \"Price before tax/fees\"\n },\n \"breakfast_included\": {\n \"type\": \"boolean\",\n \"description\": \"Is breakfast included\"\n },\n \"cancellation_policy\": {\n \"type\": \"string\",\n \"description\": \"Cancellation terms for this price\"\n }\n }\n },\n \"bed_type\": {\n \"type\": \"string\",\n \"description\": \"Bed configuration\"\n },\n \"features\": {\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"List of room features\"\n }\n }\n },\n {\n \"name\": \"location_nearby\",\n \"type\": \"object\",\n \"structure\": {\n \"airports\": {\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"item_structure\": {\n \"name\": {\n \"type\": \"string\"\n },\n \"distance\": {\n \"type\": \"string\"\n }\n }\n },\n \"culinary\": {\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"item_structure\": {\n \"name\": {\n \"type\": \"string\"\n },\n \"distance\": {\n \"type\": \"string\"\n }\n }\n },\n \"shopping\": {\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"item_structure\": {\n \"name\": {\n \"type\": \"string\"\n },\n \"distance\": {\n \"type\": \"string\"\n }\n }\n },\n \"hospitals\": {\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"item_structure\": {\n \"name\": {\n \"type\": \"string\"\n },\n \"distance\": {\n \"type\": \"string\"\n }\n }\n },\n \"landmarks\": {\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"item_structure\": {\n \"name\": {\n \"type\": \"string\"\n },\n \"distance\": {\n \"type\": \"string\"\n }\n }\n },\n \"attractions\": {\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"item_structure\": {\n \"name\": {\n \"type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\"\n },\n \"distance\": {\n \"type\": \"string\"\n }\n }\n },\n \"cash_withdrawal\": {\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"item_structure\": {\n \"name\": {\n \"type\": \"string\"\n },\n \"distance\": {\n \"type\": \"string\"\n }\n }\n },\n \"public_transport\": {\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"item_structure\": {\n \"name\": {\n \"type\": \"string\"\n },\n \"distance\": {\n \"type\": \"string\"\n }\n }\n },\n \"convenience_stores\": {\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"item_structure\": {\n \"name\": {\n \"type\": \"string\"\n },\n \"distance\": {\n \"type\": \"string\"\n }\n }\n }\n },\n \"description\": \"Points of interest near the property\"\n }\n ]\n}\n","jobPosting":"\nPlease extract the data by following the schema:\n{\n \"fields\": [\n {\n \"name\": \"jobTitle\",\n \"type\": \"string\",\n \"description\": \"Title of the job posting\"\n },\n {\n \"name\": \"datePublished\",\n \"type\": \"string\",\n \"description\": \"Publication date in ISO 8601 format\"\n },\n {\n \"name\": \"datePublishedRaw\",\n \"type\": \"string\",\n \"description\": \"Publication date as displayed on the page\"\n },\n {\n \"name\": \"validThrough\",\n \"type\": \"string\",\n \"description\": \"Expiration date of the job posting in ISO 8601 format\"\n },\n {\n \"name\": \"description\",\n \"type\": \"string\",\n \"description\": \"Full job description text\"\n },\n {\n \"name\": \"descriptionHtml\",\n \"type\": \"string\",\n \"description\": \"HTML markup of the job description\"\n },\n {\n \"name\": \"employmentType\",\n \"type\": \"string\",\n \"description\": \"Type of employment (e.g., Full-time, Part-time)\"\n },\n {\n \"name\": \"hiringOrganization\",\n \"type\": \"object\",\n \"structure\": {\n \"name\": {\n \"type\": \"string\",\n \"description\": \"Company or organization name\"\n }\n },\n \"description\": \"Organization offering the job\"\n },\n {\n \"name\": \"baseSalary\",\n \"type\": \"object\",\n \"structure\": {\n \"raw\": {\n \"type\": \"string\",\n \"description\": \"Raw salary string as displayed\"\n },\n \"currency\": {\n \"type\": \"string\",\n \"description\": \"ISO 4217 currency code\"\n },\n \"valueMax\": {\n \"type\": \"string\",\n \"description\": \"Maximum salary value\"\n },\n \"currencyRaw\": {\n \"type\": \"string\",\n \"description\": \"Currency symbol as displayed\"\n }\n },\n \"description\": \"Salary information for the job\"\n },\n {\n \"name\": \"jobLocation\",\n \"type\": \"object\",\n \"structure\": {\n \"raw\": {\n \"type\": \"string\",\n \"description\": \"Raw location string as displayed\"\n }\n },\n \"description\": \"Location of the job\"\n },\n {\n \"name\": \"url\",\n \"type\": \"string\",\n \"description\": \"URL of the job posting page\"\n }\n ]\n}\n","post":"\nPlease extract the data by following the schema:\n{\n \"fields\": [\n {\n \"name\": \"id\",\n \"type\": \"string\",\n \"description\": \"Unique post identifier\"\n },\n {\n \"name\": \"platform\",\n \"type\": \"string\",\n \"description\": \"Social media platform (e.g., Instagram, Twitter, Facebook, LinkedIn, TikTok)\"\n },\n {\n \"name\": \"post_type\",\n \"type\": \"string\",\n \"description\": \"Type of post (text, photo, video, reel, story, live, poll, event, article, link)\"\n },\n {\n \"name\": \"status\",\n \"type\": \"string\",\n \"description\": \"Post status (published, draft, scheduled, archived, deleted)\"\n },\n {\n \"name\": \"content\",\n \"type\": \"object\",\n \"structure\": {\n \"text\": {\n \"type\": \"string\",\n \"description\": \"Main post text/caption\"\n },\n \"links\": {\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"URLs in the post\",\n \"item_structure\": {\n \"url\": {\n \"type\": \"string\",\n \"description\": \"Link URL\"\n },\n \"image\": {\n \"type\": \"string\",\n \"description\": \"Link preview image URL\"\n },\n \"title\": {\n \"type\": \"string\",\n \"description\": \"Link preview title\"\n },\n \"domain\": {\n \"type\": \"string\",\n \"description\": \"Domain of the link\"\n },\n \"description\": {\n \"type\": \"string\",\n \"description\": \"Link preview description\"\n }\n }\n },\n \"title\": {\n \"type\": \"string\",\n \"description\": \"Post title or headline\"\n },\n \"hashtags\": {\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Hashtags used in the post\"\n },\n \"language\": {\n \"type\": \"string\",\n \"description\": \"Content language code (e.g., en, es, fr)\"\n },\n \"mentions\": {\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"User mentions (@username)\"\n }\n },\n \"description\": \"Post content details\"\n },\n {\n \"name\": \"author\",\n \"type\": \"object\",\n \"structure\": {\n \"id\": {\n \"type\": \"string\",\n \"description\": \"Author user ID\"\n },\n \"username\": {\n \"type\": \"string\",\n \"description\": \"Author username/handle\"\n },\n \"verified\": {\n \"type\": \"boolean\",\n \"description\": \"Is the author verified\"\n },\n \"avatar_url\": {\n \"type\": \"string\",\n \"description\": \"Author profile picture URL\"\n },\n \"profile_url\": {\n \"type\": \"string\",\n \"description\": \"Link to author profile\"\n },\n \"display_name\": {\n \"type\": \"string\",\n \"description\": \"Author display name\"\n },\n \"follower_count\": {\n \"type\": \"number\",\n \"description\": \"Author'''s follower count at time of post\"\n }\n },\n \"description\": \"Post author information\"\n },\n {\n \"name\": \"media\",\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"Media attachments (photos, videos, etc.)\",\n \"item_structure\": {\n \"id\": {\n \"type\": \"string\",\n \"description\": \"Media file ID\"\n },\n \"url\": {\n \"type\": \"string\",\n \"description\": \"Media URL\"\n },\n \"type\": {\n \"type\": \"string\",\n \"description\": \"Media type (image, video, gif, audio)\"\n },\n \"order\": {\n \"type\": \"number\",\n \"description\": \"Display order in carousel\"\n },\n \"width\": {\n \"type\": \"number\",\n \"description\": \"Media width in pixels\"\n },\n \"height\": {\n \"type\": \"number\",\n \"description\": \"Media height in pixels\"\n },\n \"alt_text\": {\n \"type\": \"string\",\n \"description\": \"Accessibility description\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"description\": \"Video/audio duration in seconds\"\n },\n \"file_size\": {\n \"type\": \"number\",\n \"description\": \"File size in bytes\"\n },\n \"mime_type\": {\n \"type\": \"string\",\n \"description\": \"MIME type (e.g., image/jpeg, video/mp4)\"\n },\n \"thumbnail_url\": {\n \"type\": \"string\",\n \"description\": \"Thumbnail/preview image URL\"\n }\n }\n },\n {\n \"name\": \"location\",\n \"type\": \"object\",\n \"structure\": {\n \"city\": {\n \"type\": \"string\",\n \"description\": \"City name\"\n },\n \"name\": {\n \"type\": \"string\",\n \"description\": \"Location name/venue\"\n },\n \"state\": {\n \"type\": \"string\",\n \"description\": \"State/province\"\n },\n \"address\": {\n \"type\": \"string\",\n \"description\": \"Full address\"\n },\n \"country\": {\n \"type\": \"string\",\n \"description\": \"Country name\"\n },\n \"latitude\": {\n \"type\": \"number\",\n \"description\": \"Geographic latitude\"\n },\n \"place_id\": {\n \"type\": \"string\",\n \"description\": \"Platform-specific place ID\"\n },\n \"zip_code\": {\n \"type\": \"string\",\n \"description\": \"Postal/ZIP code\"\n },\n \"longitude\": {\n \"type\": \"number\",\n \"description\": \"Geographic longitude\"\n },\n \"country_code\": {\n \"type\": \"string\",\n \"description\": \"Country code (ISO 3166-1 alpha-2)\"\n }\n },\n \"description\": \"Geographic location data\"\n },\n {\n \"name\": \"engagement\",\n \"type\": \"object\",\n \"structure\": {\n \"reactions\": {\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"Detailed reaction breakdown\",\n \"item_structure\": {\n \"type\": {\n \"type\": \"string\",\n \"description\": \"Reaction type (like, love, haha, wow, sad, angry)\"\n },\n \"count\": {\n \"type\": \"number\",\n \"description\": \"Number of reactions of this type\"\n }\n }\n },\n \"likes_count\": {\n \"type\": \"number\",\n \"description\": \"Number of likes\"\n },\n \"saves_count\": {\n \"type\": \"number\",\n \"description\": \"Number of saves/bookmarks\"\n },\n \"views_count\": {\n \"type\": \"number\",\n \"description\": \"Number of views/impressions\"\n },\n \"clicks_count\": {\n \"type\": \"number\",\n \"description\": \"Number of link clicks\"\n },\n \"shares_count\": {\n \"type\": \"number\",\n \"description\": \"Number of shares/reposts\"\n },\n \"comments_count\": {\n \"type\": \"number\",\n \"description\": \"Number of comments\"\n },\n \"dislikes_count\": {\n \"type\": \"number\",\n \"description\": \"Number of dislikes (if applicable)\"\n }\n },\n \"description\": \"Engagement metrics\"\n },\n {\n \"name\": \"comments\",\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"Top comments on the post\",\n \"item_structure\": {\n \"id\": {\n \"type\": \"string\",\n \"description\": \"Comment ID\"\n },\n \"text\": {\n \"type\": \"string\",\n \"description\": \"Comment text\"\n },\n \"author\": {\n \"type\": \"object\",\n \"structure\": {\n \"id\": {\n \"type\": \"string\",\n \"description\": \"Author user ID\"\n },\n \"username\": {\n \"type\": \"string\",\n \"description\": \"Author username\"\n },\n \"verified\": {\n \"type\": \"boolean\",\n \"description\": \"Is author verified\"\n },\n \"avatar_url\": {\n \"type\": \"string\",\n \"description\": \"Author avatar URL\"\n },\n \"display_name\": {\n \"type\": \"string\",\n \"description\": \"Author display name\"\n }\n },\n \"description\": \"Comment author info\"\n },\n \"is_pinned\": {\n \"type\": \"boolean\",\n \"description\": \"Is comment pinned by author\"\n },\n \"created_at\": {\n \"type\": \"string\",\n \"description\": \"Comment timestamp\"\n },\n \"likes_count\": {\n \"type\": \"number\",\n \"description\": \"Comment likes\"\n },\n \"replies_count\": {\n \"type\": \"number\",\n \"description\": \"Number of replies to this comment\"\n },\n \"is_liked_by_author\": {\n \"type\": \"boolean\",\n \"description\": \"Is comment liked by post author\"\n }\n }\n },\n {\n \"name\": \"timestamps\",\n \"type\": \"object\",\n \"structure\": {\n \"edited\": {\n \"type\": \"boolean\",\n \"description\": \"Has post been edited\"\n },\n \"created_at\": {\n \"type\": \"string\",\n \"description\": \"When post was created\"\n },\n \"expires_at\": {\n \"type\": \"string\",\n \"description\": \"Story/expiration time\"\n },\n \"updated_at\": {\n \"type\": \"string\",\n \"description\": \"When post was last edited\"\n },\n \"published_at\": {\n \"type\": \"string\",\n \"description\": \"When post was published\"\n },\n \"scheduled_for\": {\n \"type\": \"string\",\n \"description\": \"Scheduled publish time\"\n }\n },\n \"description\": \"Post timing information\"\n },\n {\n \"name\": \"tags\",\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"People, products, and other tags in the post\",\n \"item_structure\": {\n \"name\": {\n \"type\": \"string\",\n \"description\": \"Tag name/label\"\n },\n \"type\": {\n \"type\": \"string\",\n \"description\": \"Tag type (person, product, location, topic)\"\n },\n \"user_id\": {\n \"type\": \"string\",\n \"description\": \"Tagged user ID (if applicable)\"\n },\n \"position_x\": {\n \"type\": \"number\",\n \"description\": \"X position in image (0-1)\"\n },\n \"position_y\": {\n \"type\": \"number\",\n \"description\": \"Y position in image (0-1)\"\n }\n }\n },\n {\n \"name\": \"audio\",\n \"type\": \"object\",\n \"structure\": {\n \"id\": {\n \"type\": \"string\",\n \"description\": \"Audio track ID\"\n },\n \"url\": {\n \"type\": \"string\",\n \"description\": \"Audio preview URL\"\n },\n \"album\": {\n \"type\": \"string\",\n \"description\": \"Album name\"\n },\n \"title\": {\n \"type\": \"string\",\n \"description\": \"Song/audio title\"\n },\n \"artist\": {\n \"type\": \"string\",\n \"description\": \"Artist name\"\n },\n \"duration\": {\n \"type\": \"number\",\n \"description\": \"Audio duration in seconds\"\n },\n \"provider\": {\n \"type\": \"string\",\n \"description\": \"Audio provider (Spotify, Apple Music, etc.)\"\n },\n \"start_time\": {\n \"type\": \"number\",\n \"description\": \"Start time in audio (for trimmed clips)\"\n },\n \"original_audio\": {\n \"type\": \"boolean\",\n \"description\": \"Is original user-created audio\"\n }\n },\n \"description\": \"Audio information (for video posts with music/sound)\"\n },\n {\n \"name\": \"collaboration\",\n \"type\": \"object\",\n \"structure\": {\n \"collaborators\": {\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"Co-authors\",\n \"item_structure\": {\n \"id\": {\n \"type\": \"string\",\n \"description\": \"Collaborator user ID\"\n },\n \"username\": {\n \"type\": \"string\",\n \"description\": \"Collaborator username\"\n },\n \"avatar_url\": {\n \"type\": \"string\",\n \"description\": \"Collaborator avatar URL\"\n },\n \"display_name\": {\n \"type\": \"string\",\n \"description\": \"Collaborator display name\"\n }\n }\n },\n \"invited_users\": {\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Users invited to collaborate\"\n },\n \"is_collaborative\": {\n \"type\": \"boolean\",\n \"description\": \"Is this a collaborative post\"\n }\n },\n \"description\": \"Collaborative post details\"\n },\n {\n \"name\": \"parent_post\",\n \"type\": \"object\",\n \"structure\": {\n \"id\": {\n \"type\": \"string\",\n \"description\": \"Parent post ID\"\n },\n \"type\": {\n \"type\": \"string\",\n \"description\": \"Relationship type (reply, repost, quote, thread)\"\n },\n \"author\": {\n \"type\": \"object\",\n \"structure\": {\n \"id\": {\n \"type\": \"string\",\n \"description\": \"Author user ID\"\n },\n \"username\": {\n \"type\": \"string\",\n \"description\": \"Author username\"\n },\n \"display_name\": {\n \"type\": \"string\",\n \"description\": \"Author display name\"\n }\n },\n \"description\": \"Parent post author\"\n }\n },\n \"description\": \"Parent post info (for replies, reposts, quotes)\"\n },\n {\n \"name\": \"more_posts\",\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"More posts URLs\"\n },\n {\n \"name\": \"metadata\",\n \"type\": \"object\",\n \"structure\": {\n \"post_url\": {\n \"type\": \"string\",\n \"description\": \"Direct URL to the post\"\n },\n \"embed_url\": {\n \"type\": \"string\",\n \"description\": \"Embed iframe URL\"\n },\n \"shortcode\": {\n \"type\": \"string\",\n \"description\": \"Platform shortcode for sharing\"\n },\n \"source_app\": {\n \"type\": \"string\",\n \"description\": \"App used to create post\"\n },\n \"created_via\": {\n \"type\": \"string\",\n \"description\": \"Method of creation (web, mobile, api, third_party)\"\n },\n \"auto_translated\": {\n \"type\": \"boolean\",\n \"description\": \"Was content auto-translated\"\n },\n \"translation_source\": {\n \"type\": \"string\",\n \"description\": \"Original language if translated\"\n }\n },\n \"description\": \"Additional metadata\"\n }\n ]\n}\n","product":"\nPlease extract the data by following the schema:\n{\n \"fields\": [\n {\n \"name\": \"product_info\",\n \"type\": \"object\",\n \"structure\": {\n \"name\": {\n \"type\": \"string\",\n \"description\": \"Full product title\"\n },\n \"brand\": {\n \"type\": \"string\",\n \"description\": \"Normalized brand name\"\n },\n \"category\": {\n \"type\": \"string\",\n \"description\": \"Category name\"\n },\n \"condition\": {\n \"type\": \"string\",\n \"description\": \"New, Used, Refurbished\"\n },\n \"breadcrumbs\": {\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"Navigation breadcrumb trail\",\n \"item_structure\": {\n \"url\": {\n \"type\": \"string\",\n \"description\": \"Breadcrumb link URL\"\n },\n \"name\": {\n \"type\": \"string\",\n \"description\": \"Breadcrumb label\"\n }\n }\n }\n },\n \"description\": \"Core product details\"\n },\n {\n \"name\": \"sku\",\n \"type\": \"string\",\n \"description\": \"Stock Keeping Unit identifier\"\n },\n {\n \"name\": \"mpn\",\n \"type\": \"string\",\n \"description\": \"Manufacturer Part Number\"\n },\n {\n \"name\": \"gtin\",\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"Global Trade Item Numbers\",\n \"item_structure\": {\n \"type\": {\n \"type\": \"string\",\n \"description\": \"GTIN type (e.g., isbn13, ean13, upc)\"\n },\n \"value\": {\n \"type\": \"number\",\n \"description\": \"GTIN value\"\n }\n }\n },\n {\n \"name\": \"price\",\n \"type\": \"object\",\n \"structure\": {\n \"currency\": {\n \"type\": \"string\",\n \"description\": \"ISO 4217 code (e.g., USD, IDR)\"\n },\n \"current_price\": {\n \"type\": \"number\",\n \"description\": \"Active selling price\"\n },\n \"is_flash_sale\": {\n \"type\": \"boolean\",\n \"description\": \"If currently in a time-limited deal\"\n },\n \"original_price\": {\n \"type\": \"number\",\n \"description\": \"MSRP or price before discount\"\n },\n \"currency_symbol\": {\n \"type\": \"string\",\n \"description\": \"Display symbol (e.g., $, Rp)\"\n },\n \"discount_percentage\": {\n \"type\": \"number\",\n \"description\": \"Calculated off percentage\"\n }\n },\n \"description\": \"Price, currency, and discounts\"\n },\n {\n \"name\": \"wholesale_tiers\",\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"Bulk pricing (common in Tokopedia/Shopee)\",\n \"item_structure\": {\n \"min_qty\": {\n \"type\": \"number\",\n \"description\": \"Minimum quantity\"\n },\n \"price_per_unit\": {\n \"type\": \"number\",\n \"description\": \"Discounted unit price\"\n }\n }\n },\n {\n \"name\": \"stock_status\",\n \"type\": \"object\",\n \"structure\": {\n \"status\": {\n \"type\": \"string\",\n \"description\": \"e.g., in_stock, out_of_stock, pre_order\"\n },\n \"quantity\": {\n \"type\": \"number\",\n \"description\": \"Exact stock count if available\"\n }\n },\n \"description\": \"Inventory availability\"\n },\n {\n \"name\": \"media\",\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"Images and videos\",\n \"item_structure\": {\n \"url\": {\n \"type\": \"string\",\n \"description\": \"Resource URL\"\n },\n \"type\": {\n \"type\": \"string\",\n \"description\": \"'''image''' or '''video'''\"\n },\n \"is_thumbnail\": {\n \"type\": \"boolean\",\n \"description\": \"True if main display image\"\n }\n }\n },\n {\n \"name\": \"description\",\n \"type\": \"object\",\n \"structure\": {\n \"full_text\": {\n \"type\": \"string\",\n \"description\": \"Plain text description\"\n },\n \"html_content\": {\n \"type\": \"string\",\n \"description\": \"Raw HTML if preserved\"\n },\n \"key_features\": {\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Bullet points/Highlights\"\n }\n },\n \"description\": \"Product text content\"\n },\n {\n \"name\": \"specifications\",\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"Technical specs (Material, Dimensions, Tech Specs)\",\n \"item_structure\": {\n \"name\": {\n \"type\": \"string\",\n \"description\": \"Attribute name (e.g., Material)\"\n },\n \"value\": {\n \"type\": \"string\",\n \"description\": \"Attribute value (e.g., Cotton)\"\n }\n }\n },\n {\n \"name\": \"variants\",\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"Product variants (e.g., different colors, sizes)\",\n \"item_structure\": {\n \"mpn\": {\n \"type\": \"string\",\n \"description\": \"Manufacturer Part Number\"\n },\n \"sku\": {\n \"type\": \"string\",\n \"description\": \"Stock Keeping Unit identifier\"\n },\n \"url\": {\n \"type\": \"string\",\n \"description\": \"URL of the variant\"\n },\n \"gtin\": {\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"Global Trade Item Numbers\",\n \"item_structure\": {\n \"type\": {\n \"type\": \"string\",\n \"description\": \"GTIN type\"\n },\n \"value\": {\n \"type\": \"number\",\n \"description\": \"GTIN value\"\n }\n }\n },\n \"name\": {\n \"type\": \"string\",\n \"description\": \"Variant name\"\n },\n \"media\": {\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"Images and videos\",\n \"item_structure\": {\n \"url\": {\n \"type\": \"string\",\n \"description\": \"Resource URL\"\n },\n \"type\": {\n \"type\": \"string\",\n \"description\": \"'''image''' or '''video'''\"\n }\n }\n },\n \"price\": {\n \"type\": \"object\",\n \"structure\": {\n \"currency\": {\n \"type\": \"string\",\n \"description\": \"ISO 4217 code (e.g., USD, IDR)\"\n },\n \"current_price\": {\n \"type\": \"number\",\n \"description\": \"Active selling price\"\n },\n \"original_price\": {\n \"type\": \"number\",\n \"description\": \"Price before discount\"\n },\n \"currency_symbol\": {\n \"type\": \"string\",\n \"description\": \"Display symbol (e.g., $, Rp)\"\n },\n \"discount_percentage\": {\n \"type\": \"number\",\n \"description\": \"Calculated off percentage\"\n }\n },\n \"description\": \"Price, currency, and discounts of the variant\"\n },\n \"option\": {\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Variant option (array of string)\"\n },\n \"product_url\": {\n \"type\": \"string\",\n \"description\": \"URL of the variant\"\n },\n \"stock_status\": {\n \"type\": \"object\",\n \"structure\": {\n \"status\": {\n \"type\": \"string\",\n \"description\": \"e.g., in_stock, out_of_stock, pre_order\"\n },\n \"quantity\": {\n \"type\": \"number\",\n \"description\": \"Exact stock count if available\"\n }\n },\n \"description\": \"Inventory availability\"\n },\n \"additionalProperties\": {\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"Additional variant properties\",\n \"item_structure\": {\n \"name\": {\n \"type\": \"string\",\n \"description\": \"Property name\"\n },\n \"value\": {\n \"type\": \"string\",\n \"description\": \"Property value\"\n }\n }\n }\n }\n },\n {\n \"name\": \"seller\",\n \"type\": \"object\",\n \"structure\": {\n \"id\": {\n \"type\": \"string\",\n \"description\": \"Store ID\"\n },\n \"name\": {\n \"type\": \"string\",\n \"description\": \"Store name\"\n },\n \"type\": {\n \"type\": \"string\",\n \"description\": \"e.g., Official Store, Power Merchant, Mall\"\n },\n \"badges\": {\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Trust badges\"\n },\n \"rating\": {\n \"type\": \"number\",\n \"description\": \"Seller aggregate rating (0-5)\"\n },\n \"location\": {\n \"type\": \"string\",\n \"description\": \"City/Region of origin\"\n },\n \"sold_count\": {\n \"type\": \"number\",\n \"description\": \"Total units sold by the seller/store\"\n },\n \"review_count\": {\n \"type\": \"number\",\n \"description\": \"Total number of reviews received by the seller\"\n },\n \"product_count\": {\n \"type\": \"number\",\n \"description\": \"Total number of products listed by the seller/store\"\n }\n },\n \"description\": \"Merchant information\"\n },\n {\n \"name\": \"rating_summary\",\n \"type\": \"object\",\n \"structure\": {\n \"sold_count\": {\n \"type\": \"number\",\n \"description\": \"Total units sold\"\n },\n \"rating_count\": {\n \"type\": \"number\",\n \"description\": \"Total number of ratings\"\n },\n \"review_count\": {\n \"type\": \"number\",\n \"description\": \"Total textual/media reviews\"\n },\n \"average_rating\": {\n \"type\": \"number\",\n \"description\": \"0-5 scale\"\n }\n },\n \"description\": \"Product reviews\"\n },\n {\n \"name\": \"rating_distribution\",\n \"type\": \"object\",\n \"structure\": {\n \"1\": {\n \"type\": \"object\",\n \"structure\": {\n \"count\": {\n \"type\": \"number\",\n \"description\": \"Number of 1-star ratings\"\n },\n \"percentage\": {\n \"type\": \"number\",\n \"description\": \"Percentage of 1-star ratings\"\n }\n }\n },\n \"2\": {\n \"type\": \"object\",\n \"structure\": {\n \"count\": {\n \"type\": \"number\",\n \"description\": \"Number of 2-star ratings\"\n },\n \"percentage\": {\n \"type\": \"number\",\n \"description\": \"Percentage of 2-star ratings\"\n }\n }\n },\n \"3\": {\n \"type\": \"object\",\n \"structure\": {\n \"count\": {\n \"type\": \"number\",\n \"description\": \"Number of 3-star ratings\"\n },\n \"percentage\": {\n \"type\": \"number\",\n \"description\": \"Percentage of 3-star ratings\"\n }\n }\n },\n \"4\": {\n \"type\": \"object\",\n \"structure\": {\n \"count\": {\n \"type\": \"number\",\n \"description\": \"Number of 4-star ratings\"\n },\n \"percentage\": {\n \"type\": \"number\",\n \"description\": \"Percentage of 4-star ratings\"\n }\n }\n },\n \"5\": {\n \"type\": \"object\",\n \"structure\": {\n \"count\": {\n \"type\": \"number\",\n \"description\": \"Number of 5-star ratings\"\n },\n \"percentage\": {\n \"type\": \"number\",\n \"description\": \"Percentage of 5-star ratings\"\n }\n }\n }\n },\n \"description\": \"Rating distribution\"\n },\n {\n \"name\": \"shipping_info\",\n \"type\": \"object\",\n \"structure\": {\n \"weight\": {\n \"type\": \"object\",\n \"structure\": {\n \"unit\": \"string\",\n \"value\": \"number\"\n }\n },\n \"dimensions\": {\n \"type\": \"string\",\n \"description\": \"LxWxH format\"\n },\n \"free_shipping\": {\n \"type\": \"boolean\",\n \"description\": \"Is free shipping available?\"\n },\n \"fulfillment_type\": {\n \"type\": \"string\",\n \"description\": \"e.g., Pickup, Standard, Express\"\n }\n },\n \"description\": \"Logistics details\"\n },\n {\n \"name\": \"warranty_and_returns\",\n \"type\": \"object\",\n \"structure\": {\n \"has_warranty\": {\n \"type\": \"boolean\",\n \"description\": \"True/False\"\n },\n \"warranty_type\": {\n \"type\": \"string\",\n \"description\": \"e.g., Manufacturer, Seller\"\n },\n \"warranty_period\": {\n \"type\": \"string\",\n \"description\": \"e.g., 1 Year, 7 Days\"\n },\n \"return_policy_text\": {\n \"type\": \"string\",\n \"description\": \"Brief return conditions\"\n }\n },\n \"description\": \"Post-purchase policies\"\n }\n ]\n}\n","property":"\nPlease extract the data by following the schema:\n{\n \"fields\": [\n {\n \"name\": \"title\",\n \"type\": \"string\",\n \"description\": \"Listing title or marketing headline (e.g., '''Castillo Caribe''')\"\n },\n {\n \"name\": \"mls_number\",\n \"type\": \"string\",\n \"description\": \"Multiple Listing Service identifier (MLS ID/Number)\"\n },\n {\n \"name\": \"status\",\n \"type\": \"string\",\n \"description\": \"Current listing status (e.g., For Sale, Active, Pending, Sold, Current, New, etc.)\"\n },\n {\n \"name\": \"price\",\n \"type\": \"object\",\n \"structure\": {\n \"amount\": {\n \"type\": \"string\",\n \"description\": \"Listing price value\"\n },\n \"currency\": {\n \"type\": \"string\",\n \"description\": \"Currency code (e.g., USD, CAD)\"\n },\n \"display_price\": {\n \"type\": \"string\",\n \"description\": \"Formatted price string (e.g., $1,250,000)\"\n },\n \"price_per_sqft\": {\n \"type\": \"string\",\n \"description\": \"Price per square foot\"\n }\n },\n \"description\": \"Price information\"\n },\n {\n \"name\": \"address\",\n \"type\": \"object\",\n \"structure\": {\n \"city\": {\n \"type\": \"string\",\n \"description\": \"City or locality\"\n },\n \"unit\": {\n \"type\": \"string\",\n \"description\": \"Apartment or unit number\"\n },\n \"state\": {\n \"type\": \"string\",\n \"description\": \"State, region, or province\"\n },\n \"county\": {\n \"type\": \"string\",\n \"description\": \"County or parish\"\n },\n \"island\": {\n \"type\": \"string\",\n \"description\": \"Island name (for Caribbean/island properties)\"\n },\n \"street\": {\n \"type\": \"string\",\n \"description\": \"Street number and name\"\n },\n \"country\": {\n \"type\": \"string\",\n \"description\": \"Country code or name\"\n },\n \"latitude\": {\n \"type\": \"number\",\n \"description\": \"Geographic latitude\"\n },\n \"zip_code\": {\n \"type\": \"string\",\n \"description\": \"Postal or ZIP code\"\n },\n \"longitude\": {\n \"type\": \"number\",\n \"description\": \"Geographic longitude\"\n },\n \"maps_link\": {\n \"type\": \"string\",\n \"description\": \"URL to map location\"\n },\n \"full_address\": {\n \"type\": \"string\",\n \"description\": \"Full formatted address\"\n }\n },\n \"description\": \"Location details\"\n },\n {\n \"name\": \"property_details\",\n \"type\": \"object\",\n \"structure\": {\n \"sq_ft\": {\n \"type\": \"number\",\n \"description\": \"Total living area in square feet\"\n },\n \"stories\": {\n \"type\": \"number\",\n \"description\": \"Number of stories/levels\"\n },\n \"bedrooms\": {\n \"type\": \"number\",\n \"description\": \"Total number of bedrooms\"\n },\n \"bathrooms\": {\n \"type\": \"number\",\n \"description\": \"Total number of bathrooms\"\n },\n \"furnished\": {\n \"type\": \"boolean\",\n \"description\": \"Is the property furnished\"\n },\n \"year_built\": {\n \"type\": \"number\",\n \"description\": \"Year of construction\"\n },\n \"property_type\": {\n \"type\": \"string\",\n \"description\": \"Type of property (e.g., Single Family, Condo)\"\n },\n \"bathrooms_full\": {\n \"type\": \"number\",\n \"description\": \"Number of full bathrooms\"\n },\n \"property_style\": {\n \"type\": \"string\",\n \"description\": \"Architectural style (e.g., 2 Story, Ranch)\"\n },\n \"bedrooms_details\": {\n \"type\": \"string\",\n \"description\": \"Details like above/below grade\"\n },\n \"bathrooms_partial\": {\n \"type\": \"number\",\n \"description\": \"Number of half/partial bathrooms\"\n },\n \"construction_materials\": {\n \"type\": \"string\",\n \"description\": \"Materials used (e.g., Stone, Brick)\"\n }\n },\n \"description\": \"Core property specifications\"\n },\n {\n \"name\": \"lot_info\",\n \"type\": \"object\",\n \"structure\": {\n \"views\": {\n \"type\": \"string\",\n \"description\": \"Description of views (e.g., Mountain, Panoramic)\"\n },\n \"zoning\": {\n \"type\": \"string\",\n \"description\": \"Zoning classification\"\n },\n \"waterfront\": {\n \"type\": \"boolean\",\n \"description\": \"Does the property have water frontage\"\n },\n \"lot_size_sqft\": {\n \"type\": \"number\",\n \"description\": \"Lot size converted to square feet\"\n },\n \"lot_size_text\": {\n \"type\": \"string\",\n \"description\": \"Lot size string (e.g., '''0.51 Acres''')\"\n },\n \"lot_size_acres\": {\n \"type\": \"number\",\n \"description\": \"Lot size converted to acres\"\n }\n },\n \"description\": \"Land and lot details\"\n },\n {\n \"name\": \"description\",\n \"type\": \"string\",\n \"description\": \"Full marketing description of the property\"\n },\n {\n \"name\": \"dates\",\n \"type\": \"object\",\n \"structure\": {\n \"sold\": {\n \"type\": \"string\",\n \"description\": \"Date sold\"\n },\n \"posted\": {\n \"type\": \"string\",\n \"description\": \"Date listed/posted\"\n },\n \"updated\": {\n \"type\": \"string\",\n \"description\": \"Last updated date\"\n },\n \"days_on_market\": {\n \"type\": \"number\",\n \"description\": \"Number of days listed\"\n }\n },\n \"description\": \"Relevant listing dates\"\n },\n {\n \"name\": \"interior_features\",\n \"type\": \"object\",\n \"structure\": {\n \"cooling\": {\n \"type\": \"string\",\n \"description\": \"Cooling system details\"\n },\n \"heating\": {\n \"type\": \"string\",\n \"description\": \"Heating system details\"\n },\n \"kitchen\": {\n \"type\": \"string\",\n \"description\": \"Kitchen features\"\n },\n \"laundry\": {\n \"type\": \"string\",\n \"description\": \"Laundry features\"\n },\n \"basement\": {\n \"type\": \"string\",\n \"description\": \"Basement description\"\n },\n \"flooring\": {\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Types of flooring\"\n },\n \"appliances\": {\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"List of appliances included\"\n },\n \"fireplaces\": {\n \"type\": \"number\",\n \"description\": \"Number of fireplaces\"\n },\n \"other_interior\": {\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Additional interior features (e.g., Vaulted Ceilings)\"\n },\n \"security_features\": {\n \"type\": \"string\",\n \"description\": \"Interior security systems\"\n },\n \"fireplace_features\": {\n \"type\": \"string\",\n \"description\": \"Details about fireplaces\"\n }\n },\n \"description\": \"Interior amenities and systems\"\n },\n {\n \"name\": \"rooms\",\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"Detailed room information\",\n \"item_structure\": {\n \"name\": {\n \"type\": \"string\",\n \"description\": \"Room name (e.g., Kitchen, Master Bed)\"\n },\n \"level\": {\n \"type\": \"string\",\n \"description\": \"Floor level\"\n },\n \"features\": {\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Specific features of this room\"\n },\n \"dimensions\": {\n \"type\": \"string\",\n \"description\": \"Room dimensions\"\n }\n }\n },\n {\n \"name\": \"exterior_features\",\n \"type\": \"object\",\n \"structure\": {\n \"pool\": {\n \"type\": \"string\",\n \"description\": \"Pool features\"\n },\n \"roof\": {\n \"type\": \"string\",\n \"description\": \"Roofing material\"\n },\n \"fencing\": {\n \"type\": \"string\",\n \"description\": \"Fence details\"\n },\n \"patio_porch\": {\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Patio, deck, or porch features\"\n },\n \"other_structures\": {\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Sheds, workshops, etc.\"\n }\n },\n \"description\": \"Exterior amenities and structures\"\n },\n {\n \"name\": \"parking\",\n \"type\": \"object\",\n \"structure\": {\n \"type\": {\n \"type\": \"string\",\n \"description\": \"Type of parking (Garage, Carport, Off-street)\"\n },\n \"spaces\": {\n \"type\": \"number\",\n \"description\": \"Number of parking spaces\"\n },\n \"has_garage\": {\n \"type\": \"boolean\",\n \"description\": \"Is there a garage\"\n },\n \"description\": {\n \"type\": \"string\",\n \"description\": \"Parking features description\"\n }\n },\n \"description\": \"Parking and garage details\"\n },\n {\n \"name\": \"utilities\",\n \"type\": \"object\",\n \"structure\": {\n \"sewer\": {\n \"type\": \"string\",\n \"description\": \"Sewer system\"\n },\n \"water\": {\n \"type\": \"string\",\n \"description\": \"Water source\"\n },\n \"electricity\": {\n \"type\": \"string\",\n \"description\": \"Electric provider or details\"\n },\n \"energy_info\": {\n \"type\": \"string\",\n \"description\": \"Solar or energy efficiency details\"\n },\n \"available_utilities\": {\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"List of available utility connections\"\n }\n },\n \"description\": \"Utility connections\"\n },\n {\n \"name\": \"financial\",\n \"type\": \"object\",\n \"structure\": {\n \"hoa_fee\": {\n \"type\": \"string\",\n \"description\": \"HOA or maintenance monthly fee\"\n },\n \"annual_tax\": {\n \"type\": \"string\",\n \"description\": \"Most recent annual tax amount\"\n },\n \"hoa_includes\": {\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"What the HOA fee covers\"\n },\n \"tax_property_id\": {\n \"type\": \"string\",\n \"description\": \"Tax ID / APN\"\n },\n \"tax_assessed_value\": {\n \"type\": \"string\",\n \"description\": \"Total assessed tax value\"\n },\n \"estimated_monthly_payment\": {\n \"type\": \"string\",\n \"description\": \"Estimated mortgage payment\"\n }\n },\n \"description\": \"Financial data and recurring costs\"\n },\n {\n \"name\": \"tax_history\",\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"Historical tax records\",\n \"item_structure\": {\n \"year\": {\n \"type\": \"number\",\n \"description\": \"Tax year\"\n },\n \"amount\": {\n \"type\": \"string\",\n \"description\": \"Tax amount paid\"\n },\n \"assessment\": {\n \"type\": \"string\",\n \"description\": \"Assessed value\"\n },\n \"change_percentage\": {\n \"type\": \"string\",\n \"description\": \"Year-over-year change\"\n }\n }\n },\n {\n \"name\": \"price_history\",\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"History of listing prices and sales\",\n \"item_structure\": {\n \"date\": {\n \"type\": \"string\",\n \"description\": \"Date of event\"\n },\n \"event\": {\n \"type\": \"string\",\n \"description\": \"Event type (Listed, Sold, Price Change)\"\n },\n \"price\": {\n \"type\": \"string\",\n \"description\": \"Price at time of event\"\n },\n \"source\": {\n \"type\": \"string\",\n \"description\": \"Source of data\"\n }\n }\n },\n {\n \"name\": \"agents\",\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"Listing agents and brokerages\",\n \"item_structure\": {\n \"name\": {\n \"type\": \"string\",\n \"description\": \"Agent name\"\n },\n \"role\": {\n \"type\": \"string\",\n \"description\": \"Role (Listing Agent, Buying Agent)\"\n },\n \"email\": {\n \"type\": \"string\",\n \"description\": \"Contact email\"\n },\n \"phone\": {\n \"type\": \"string\",\n \"description\": \"Contact phone\"\n },\n \"agency\": {\n \"type\": \"string\",\n \"description\": \"Agency or Brokerage name\"\n },\n \"profile_url\": {\n \"type\": \"string\",\n \"description\": \"Link to agent profile\"\n }\n }\n },\n {\n \"name\": \"school_district\",\n \"type\": \"string\",\n \"description\": \"School district name\"\n },\n {\n \"name\": \"nearby_schools\",\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"List of nearby schools\",\n \"item_structure\": {\n \"name\": {\n \"type\": \"string\",\n \"description\": \"School name\"\n },\n \"type\": {\n \"type\": \"string\",\n \"description\": \"Public or Private\"\n },\n \"grades\": {\n \"type\": \"string\",\n \"description\": \"Grades served\"\n },\n \"distance\": {\n \"type\": \"string\",\n \"description\": \"Distance from property\"\n }\n }\n },\n {\n \"name\": \"community\",\n \"type\": \"object\",\n \"structure\": {\n \"name\": {\n \"type\": \"string\",\n \"description\": \"Name of neighborhood or subdivision\"\n },\n \"features\": {\n \"type\": \"string\",\n \"description\": \"Community amenities (Park, Clubhouse)\"\n },\n \"security\": {\n \"type\": \"string\",\n \"description\": \"Community security features\"\n }\n },\n \"description\": \"Neighborhood and community features\"\n },\n {\n \"name\": \"area_statistics\",\n \"type\": \"object\",\n \"structure\": {\n \"people\": {\n \"type\": \"object\",\n \"structure\": {\n \"median_age\": {\n \"type\": \"object\",\n \"structure\": {\n \"zip\": \"number\",\n \"city\": \"number\",\n \"county\": \"number\",\n \"national\": \"number\"\n },\n \"description\": \"Median age in years (zip, city, county, national)\"\n },\n \"population\": {\n \"type\": \"object\",\n \"structure\": {\n \"zip\": \"number\",\n \"city\": \"number\",\n \"county\": \"number\",\n \"national\": \"number\"\n },\n \"description\": \"Population counts (zip, city, county, national)\"\n },\n \"average_income\": {\n \"type\": \"object\",\n \"structure\": {\n \"zip\": \"string\",\n \"city\": \"string\",\n \"county\": \"string\",\n \"national\": \"string\"\n },\n \"description\": \"Average income (zip, city, county, national)\"\n },\n \"population_density\": {\n \"type\": \"object\",\n \"structure\": {\n \"zip\": \"number\",\n \"city\": \"number\",\n \"county\": \"number\",\n \"national\": \"number\"\n },\n \"description\": \"People per sq mi (zip, city, county, national)\"\n },\n \"cost_of_living_index\": {\n \"type\": \"object\",\n \"structure\": {\n \"zip\": \"number\",\n \"city\": \"number\",\n \"county\": \"number\",\n \"national\": \"number\"\n },\n \"description\": \"Cost of living index, 100=national (zip, city, county, national)\"\n },\n \"people_per_household\": {\n \"type\": \"object\",\n \"structure\": {\n \"zip\": \"number\",\n \"city\": \"number\",\n \"county\": \"number\",\n \"national\": \"number\"\n },\n \"description\": \"Avg household size (zip, city, county, national)\"\n },\n \"median_household_income\": {\n \"type\": \"object\",\n \"structure\": {\n \"zip\": \"string\",\n \"city\": \"string\",\n \"county\": \"string\",\n \"national\": \"string\"\n },\n \"description\": \"Median income (zip, city, county, national)\"\n }\n },\n \"description\": \"Population and income statistics for the area\"\n },\n \"sun_exposure\": {\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"Sun hours by month\"\n },\n \"annual_precipitation\": {\n \"type\": \"object\",\n \"structure\": {\n \"zip\": \"number\",\n \"city\": \"number\",\n \"county\": \"number\",\n \"national\": \"number\"\n },\n \"description\": \"Annual precipitation in inches (zip, city, county, national)\"\n }\n },\n \"description\": \"Demographics and environment stats\"\n },\n {\n \"name\": \"climate_risks\",\n \"type\": \"object\",\n \"structure\": {\n \"air_quality\": {\n \"type\": \"string\",\n \"description\": \"Air quality factor\"\n },\n \"fire_factor\": {\n \"type\": \"string\",\n \"description\": \"Wildfire risk level\"\n },\n \"heat_factor\": {\n \"type\": \"string\",\n \"description\": \"Heat risk level\"\n },\n \"wind_factor\": {\n \"type\": \"string\",\n \"description\": \"Wind risk level\"\n },\n \"flood_factor\": {\n \"type\": \"string\",\n \"description\": \"Flood risk level\"\n }\n },\n \"description\": \"Environmental risk factors\"\n },\n {\n \"name\": \"lifestyle\",\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"Lifestyle scores and attributes\",\n \"item_structure\": {\n \"score\": {\n \"type\": \"number\",\n \"description\": \"Score value\"\n },\n \"category\": {\n \"type\": \"string\",\n \"description\": \"Category (e.g., Walkability, Quiet)\"\n },\n \"description\": {\n \"type\": \"string\",\n \"description\": \"Description of score\"\n }\n }\n },\n {\n \"name\": \"nearby_amenities\",\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"Points of interest nearby\",\n \"item_structure\": {\n \"name\": {\n \"type\": \"string\",\n \"description\": \"Name of place\"\n },\n \"category\": {\n \"type\": \"string\",\n \"description\": \"Type (Grocery, Park, Hospital)\"\n },\n \"distance\": {\n \"type\": \"string\",\n \"description\": \"Distance from property\"\n }\n }\n },\n {\n \"name\": \"metadata\",\n \"type\": \"object\",\n \"structure\": {\n \"url\": {\n \"type\": \"string\",\n \"description\": \"Listing URL\"\n },\n \"views\": {\n \"type\": \"number\",\n \"description\": \"Number of views\"\n },\n \"source\": {\n \"type\": \"string\",\n \"description\": \"Data source name\"\n },\n \"favorites\": {\n \"type\": \"number\",\n \"description\": \"Number of favorites/saves\"\n }\n },\n \"description\": \"Listing metadata\"\n }\n ]\n}\n","restaurant":"\nPlease extract the data by following the schema:\n{\n \"fields\": [\n {\n \"name\": \"name\",\n \"type\": \"string\",\n \"description\": \"Restaurant name\"\n },\n {\n \"name\": \"rank\",\n \"type\": \"string\",\n \"description\": \"Ranking among restaurants in the area (e.g., #14 of 769 Restaurants in Kuta)\"\n },\n {\n \"name\": \"cuisine\",\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Cuisine types served\"\n },\n {\n \"name\": \"address\",\n \"type\": \"string\",\n \"description\": \"Full street address\"\n },\n {\n \"name\": \"map_link\",\n \"type\": \"string\",\n \"description\": \"Google Maps directions link\"\n },\n {\n \"name\": \"phone\",\n \"type\": \"string\",\n \"description\": \"Contact phone number\"\n },\n {\n \"name\": \"email\",\n \"type\": \"string\",\n \"description\": \"Contact email address\"\n },\n {\n \"name\": \"website\",\n \"type\": \"string\",\n \"description\": \"Official website URL\"\n },\n {\n \"name\": \"hours\",\n \"type\": \"object\",\n \"structure\": {\n \"Friday\": {\n \"type\": \"string\",\n \"description\": \"Opening hours\"\n },\n \"Monday\": {\n \"type\": \"string\",\n \"description\": \"Opening hours\"\n },\n \"Sunday\": {\n \"type\": \"string\",\n \"description\": \"Opening hours\"\n },\n \"Tuesday\": {\n \"type\": \"string\",\n \"description\": \"Opening hours\"\n },\n \"Saturday\": {\n \"type\": \"string\",\n \"description\": \"Opening hours\"\n },\n \"Thursday\": {\n \"type\": \"string\",\n \"description\": \"Opening hours\"\n },\n \"Wednesday\": {\n \"type\": \"string\",\n \"description\": \"Opening hours\"\n }\n },\n \"description\": \"Operating hours by day of week\"\n },\n {\n \"name\": \"features\",\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Restaurant features and amenities (e.g., dietary options, meal types, payment methods)\"\n },\n {\n \"name\": \"description\",\n \"type\": \"string\",\n \"description\": \"Restaurant description and overview\"\n },\n {\n \"name\": \"rating\",\n \"type\": \"number\",\n \"description\": \"Overall rating (0-5 scale)\"\n },\n {\n \"name\": \"rating_distribution\",\n \"type\": \"object\",\n \"structure\": {\n \"Food\": {\n \"type\": \"number\",\n \"description\": \"Food rating (0-5)\"\n },\n \"Value\": {\n \"type\": \"number\",\n \"description\": \"Value rating (0-5)\"\n },\n \"Service\": {\n \"type\": \"number\",\n \"description\": \"Service rating (0-5)\"\n },\n \"Atmosphere\": {\n \"type\": \"number\",\n \"description\": \"Atmosphere rating (0-5)\"\n }\n },\n \"description\": \"Ratings by category\"\n },\n {\n \"name\": \"reviews_count\",\n \"type\": \"number\",\n \"description\": \"Total number of reviews\"\n },\n {\n \"name\": \"reviews_distribution\",\n \"type\": \"object\",\n \"structure\": {\n \"Good\": {\n \"type\": \"number\",\n \"description\": \"Number of Good reviews\"\n },\n \"Poor\": {\n \"type\": \"number\",\n \"description\": \"Number of Poor reviews\"\n },\n \"Average\": {\n \"type\": \"number\",\n \"description\": \"Number of Average reviews\"\n },\n \"Terrible\": {\n \"type\": \"number\",\n \"description\": \"Number of Terrible reviews\"\n },\n \"Excellent\": {\n \"type\": \"number\",\n \"description\": \"Number of Excellent reviews\"\n }\n },\n \"description\": \"Review count by rating level\"\n },\n {\n \"name\": \"review_summary\",\n \"type\": \"string\",\n \"description\": \"AI-generated summary of guest reviews\"\n },\n {\n \"name\": \"review_summary_distribution\",\n \"type\": \"object\",\n \"structure\": {\n \"Food\": {\n \"type\": \"string\",\n \"description\": \"Food sentiment keyword\"\n },\n \"Value\": {\n \"type\": \"string\",\n \"description\": \"Value sentiment keyword\"\n },\n \"Service\": {\n \"type\": \"string\",\n \"description\": \"Service sentiment keyword\"\n },\n \"Location\": {\n \"type\": \"string\",\n \"description\": \"Location sentiment keyword\"\n },\n \"Atmosphere\": {\n \"type\": \"string\",\n \"description\": \"Atmosphere sentiment keyword\"\n }\n },\n \"description\": \"One-word sentiment summary by category\"\n },\n {\n \"name\": \"reviews\",\n \"type\": \"object\",\n \"structure\": {\n \"title\": {\n \"type\": \"string\",\n \"description\": \"Review title\"\n },\n \"rating\": {\n \"type\": \"string\",\n \"description\": \"\"\n },\n \"content\": {\n \"type\": \"string\",\n \"description\": \"\"\n },\n \"review_date\": {\n \"type\": \"number\",\n \"description\": \"For example Feb 5, 2026\"\n },\n \"reviewer_name\": {\n \"type\": \"string\",\n \"description\": \"\"\n },\n \"rating_distribution\": {\n \"type\": \"object\",\n \"structure\": {},\n \"description\": \"Key value rating distribution (for example value: 4, service: 3, etc.)\"\n }\n },\n \"description\": \"Restaurant reviews\"\n }\n ]\n}\n","socialMediaProfile":"\nPlease extract the data by following the schema:\n{\n \"fields\": [\n {\n \"name\": \"username\",\n \"type\": \"string\",\n \"description\": \"\"\n },\n {\n \"name\": \"display_name\",\n \"type\": \"string\",\n \"description\": \"Profile display name\"\n },\n {\n \"name\": \"type\",\n \"type\": \"string\",\n \"description\": \"Profile type (e.g. Internet Service Provider, Sport and recreation, community, etc.)\"\n },\n {\n \"name\": \"email\",\n \"type\": \"string\",\n \"description\": \"\"\n },\n {\n \"name\": \"phone\",\n \"type\": \"string\",\n \"description\": \"\"\n },\n {\n \"name\": \"phone\",\n \"type\": \"string\",\n \"description\": \"\"\n },\n {\n \"name\": \"address\",\n \"type\": \"object\",\n \"structure\": {\n \"city\": {\n \"type\": \"string\",\n \"description\": \"\"\n },\n \"street\": {\n \"type\": \"string\",\n \"description\": \"\"\n },\n \"country\": {\n \"type\": \"string\",\n \"description\": \"\"\n },\n \"district\": {\n \"type\": \"string\",\n \"description\": \"\"\n }\n },\n \"description\": \"\"\n },\n {\n \"name\": \"description\",\n \"type\": \"string\",\n \"description\": \"Profile bio or description\"\n },\n {\n \"name\": \"verified\",\n \"type\": \"string\",\n \"description\": \"true or false\"\n },\n {\n \"name\": \"followers\",\n \"type\": \"number\",\n \"description\": \"\"\n },\n {\n \"name\": \"following\",\n \"type\": \"number\",\n \"description\": \"\"\n },\n {\n \"name\": \"subscribers\",\n \"type\": \"number\",\n \"description\": \"\"\n },\n {\n \"name\": \"num_posts\",\n \"type\": \"number\",\n \"description\": \"\"\n },\n {\n \"name\": \"social_links\",\n \"type\": \"object\",\n \"structure\": {\n \"x\": {\n \"type\": \"string\",\n \"description\": \"\"\n },\n \"tiktok\": {\n \"type\": \"string\",\n \"description\": \"\"\n },\n \"threads\": {\n \"type\": \"string\",\n \"description\": \"\"\n },\n \"youtube\": {\n \"type\": \"string\",\n \"description\": \"\"\n },\n \"telegram\": {\n \"type\": \"string\",\n \"description\": \"\"\n },\n \"instagram\": {\n \"type\": \"string\",\n \"description\": \"\"\n }\n },\n \"description\": \"\"\n },\n {\n \"name\": \"website\",\n \"type\": \"string\",\n \"description\": \"\"\n },\n {\n \"name\": \"profile_picture\",\n \"type\": \"string\",\n \"description\": \"\"\n },\n {\n \"name\": \"recent_posts\",\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"\",\n \"item_structure\": {\n \"title\": {\n \"type\": \"string\",\n \"description\": \"\"\n },\n \"posted\": {\n \"type\": \"string\",\n \"description\": \"Can be date or for example 1 day ago, 4 months ago.\"\n },\n \"post_url\": {\n \"type\": \"string\",\n \"description\": \"Full url of the post\"\n },\n \"num_views\": {\n \"type\": \"string\",\n \"description\": \"\"\n },\n \"thumbnail\": {\n \"type\": \"string\",\n \"description\": \"Thumbnail URL\"\n },\n \"description\": {\n \"type\": \"string\",\n \"description\": \"\"\n }\n }\n }\n ]\n}\n","tourAttraction":"\nPlease extract the data by following the schema:\n{\n \"fields\": [\n {\n \"name\": \"title\",\n \"type\": \"string\",\n \"description\": \"Tour title as displayed on the page\"\n },\n {\n \"name\": \"about\",\n \"type\": \"string\",\n \"description\": \"Full tour description and overview\"\n },\n {\n \"name\": \"provider\",\n \"type\": \"object\",\n \"structure\": {\n \"logo\": {\n \"type\": \"string\",\n \"description\": \"Logo URL\"\n },\n \"name\": {\n \"type\": \"string\",\n \"description\": \"Company name\"\n },\n \"contact\": {\n \"type\": \"object\",\n \"structure\": {\n \"email\": {\n \"type\": \"string\"\n },\n \"phone\": {\n \"type\": \"string\"\n },\n \"website\": {\n \"type\": \"string\"\n }\n }\n }\n },\n \"description\": \"Tour operator information\"\n },\n {\n \"name\": \"location\",\n \"type\": \"object\",\n \"structure\": {\n \"city\": {\n \"type\": \"string\"\n },\n \"country\": {\n \"type\": \"string\"\n },\n \"coordinates\": {\n \"type\": \"object\",\n \"structure\": {\n \"latitude\": {\n \"type\": \"number\"\n },\n \"longitude\": {\n \"type\": \"number\"\n }\n }\n },\n \"meeting_point\": {\n \"type\": \"string\"\n }\n },\n \"description\": \"Primary tour location\"\n },\n {\n \"name\": \"age_range\",\n \"type\": \"object\",\n \"structure\": {\n \"max_age\": {\n \"type\": \"number\"\n },\n \"min_age\": {\n \"type\": \"number\"\n }\n },\n \"description\": \"Age range for the tour\"\n },\n {\n \"name\": \"duration\",\n \"type\": \"string\",\n \"description\": \"Total tour duration (e.g., 3-4 hours)\"\n },\n {\n \"name\": \"start_time\",\n \"type\": \"string\",\n \"description\": \"Start time for the tour\"\n },\n {\n \"name\": \"mobile_ticket\",\n \"type\": \"boolean\",\n \"description\": \"Whether the tour requires a mobile ticket\"\n },\n {\n \"name\": \"languages\",\n \"type\": \"object\",\n \"structure\": {\n \"live_guide\": {\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Languages of the live guide\"\n },\n \"audio_guide\": {\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Languages of the audio guide\"\n },\n \"written_guide\": {\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Languages of the written guide\"\n }\n },\n \"description\": \"Languages of the tour\"\n },\n {\n \"name\": \"price\",\n \"type\": \"object\",\n \"structure\": {\n \"currency\": {\n \"type\": \"string\"\n },\n \"original_price\": {\n \"type\": \"string\"\n },\n \"starting_price\": {\n \"type\": \"string\"\n },\n \"discounted_price\": {\n \"type\": \"string\"\n },\n \"price_per_person\": {\n \"type\": \"boolean\"\n }\n },\n \"description\": \"Tour pricing information\"\n },\n {\n \"name\": \"availability\",\n \"type\": \"object\",\n \"structure\": {\n \"max_group_size\": {\n \"type\": \"number\"\n },\n \"available_dates\": {\n \"type\": \"array\",\n \"item_type\": \"string\"\n },\n \"free_cancellation\": {\n \"type\": \"boolean\"\n },\n \"instant_confirmation\": {\n \"type\": \"boolean\"\n },\n \"reserve_now_pay_later\": {\n \"type\": \"boolean\"\n }\n },\n \"description\": \"Availability and booking info\"\n },\n {\n \"name\": \"highlights\",\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Key selling points and highlights\"\n },\n {\n \"name\": \"included\",\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Items included in the tour\"\n },\n {\n \"name\": \"excluded\",\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Items not included\"\n },\n {\n \"name\": \"meeting_and_pickup\",\n \"type\": \"object\",\n \"structure\": {\n \"end_point\": {\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"item_structure\": {\n \"address\": {\n \"type\": \"string\",\n \"description\": \"Address of the end point\"\n },\n \"details\": {\n \"type\": \"string\",\n \"description\": \"Details of the end point address\"\n }\n }\n },\n \"start_point\": {\n \"type\": \"object\",\n \"structure\": {\n \"type\": {\n \"type\": \"string\",\n \"description\": \"Type of start point (hotel, airport, multiple, etc.)\"\n },\n \"address\": {\n \"type\": \"string\",\n \"description\": \"Address of the start point\"\n },\n \"details\": {\n \"type\": \"string\",\n \"description\": \"Details of the start point address\"\n },\n \"pickup_details\": {\n \"type\": \"string\",\n \"description\": \"Pickup details\"\n },\n \"pickup_offered\": {\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"item_structure\": {\n \"name\": {\n \"type\": \"string\",\n \"description\": \"Name of the pickup point\"\n },\n \"details\": {\n \"type\": \"string\",\n \"description\": \"Details of the pickup point\"\n }\n }\n }\n }\n }\n },\n \"description\": \"Meeting point and pickup information\"\n },\n {\n \"name\": \"itinerary\",\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"Step-by-step itinerary\",\n \"item_structure\": {\n \"duration\": {\n \"type\": \"string\"\n },\n \"stop_name\": {\n \"type\": \"string\"\n },\n \"description\": {\n \"type\": \"string\"\n },\n \"admission_included\": {\n \"type\": \"boolean\"\n }\n }\n },\n {\n \"name\": \"additional_info\",\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Important notes and additional details\"\n },\n {\n \"name\": \"accessibility\",\n \"type\": \"object\",\n \"structure\": {\n \"stroller_accessible\": {\n \"type\": \"boolean\"\n },\n \"near_public_transport\": {\n \"type\": \"boolean\"\n },\n \"wheelchair_accessible\": {\n \"type\": \"boolean\"\n }\n },\n \"description\": \"Accessibility information\"\n },\n {\n \"name\": \"review_stats\",\n \"type\": \"object\",\n \"structure\": {\n \"overall_score\": {\n \"type\": \"number\"\n },\n \"total_reviews\": {\n \"type\": \"number\"\n },\n \"rating_breakdown\": {\n \"type\": \"object\",\n \"structure\": {\n \"1_star\": {\n \"type\": \"number\"\n },\n \"2_star\": {\n \"type\": \"number\"\n },\n \"3_star\": {\n \"type\": \"number\"\n },\n \"4_star\": {\n \"type\": \"number\"\n },\n \"5_star\": {\n \"type\": \"number\"\n }\n }\n }\n },\n \"description\": \"Aggregated review statistics\"\n },\n {\n \"name\": \"reviews\",\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"Individual traveler reviews\",\n \"item_structure\": {\n \"date\": {\n \"type\": \"string\"\n },\n \"title\": {\n \"type\": \"string\"\n },\n \"rating\": {\n \"type\": \"number\"\n },\n \"content\": {\n \"type\": \"string\"\n },\n \"country\": {\n \"type\": \"string\"\n },\n \"reviewer\": {\n \"type\": \"string\"\n },\n \"trip_type\": {\n \"type\": \"string\"\n }\n }\n },\n {\n \"name\": \"photos\",\n \"type\": \"array\",\n \"item_type\": \"string\",\n \"description\": \"Image URLs\"\n },\n {\n \"name\": \"faq\",\n \"type\": \"array\",\n \"item_type\": \"object\",\n \"description\": \"Frequently asked questions\",\n \"item_structure\": {\n \"answer\": {\n \"type\": \"string\"\n },\n \"question\": {\n \"type\": \"string\"\n }\n }\n },\n {\n \"name\": \"policies\",\n \"type\": \"object\",\n \"structure\": {\n \"child_policy\": {\n \"type\": \"string\"\n },\n \"refund_policy\": {\n \"type\": \"string\"\n },\n \"weather_policy\": {\n \"type\": \"string\"\n },\n \"cancellation_policy\": {\n \"type\": \"string\"\n }\n },\n \"description\": \"Tour policies\"\n }\n ]\n}\n"} \ No newline at end of file diff --git a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/toolkit.py b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/toolkit.py new file mode 100644 index 0000000000..48bce6e50a --- /dev/null +++ b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/toolkit.py @@ -0,0 +1,113 @@ +"""Convenience factory for independent MrScraper tool instances.""" + +from collections.abc import Iterable + +from crewai.tools import BaseTool + +from crewai_tools.tools.mrscraper.account import MrScraperGetAccountInfoTool +from crewai_tools.tools.mrscraper.base import MrScraperBaseTool, resolve_api_token +from crewai_tools.tools.mrscraper.client import MrScraperClient +from crewai_tools.tools.mrscraper.discovery import ( + MrScraperCrawlWebsiteUrlsTool, + MrScraperSearchGoogleSerpTool, +) +from crewai_tools.tools.mrscraper.extraction import ( + MrScraperExtractListingsTool, + MrScraperExtractPageByPromptTool, + MrScraperExtractStructuredDataTool, + MrScraperFetchRenderedHtmlTool, +) +from crewai_tools.tools.mrscraper.results import ( + MrScraperGetLatestResultsTool, + MrScraperGetResultDetailTool, + MrScraperGetResultsTool, +) +from crewai_tools.tools.mrscraper.scraper_creation import ( + MrScraperCreateListingScraperTool, + MrScraperCreatePromptScraperTool, + MrScraperCreateWebsiteCrawlScraperTool, +) +from crewai_tools.tools.mrscraper.scraper_runs import ( + MrScraperRunExistingScraperBatchTool, + MrScraperRunExistingScraperTool, +) + + +ToolClass = type[MrScraperBaseTool] + +_GROUPS: dict[str, tuple[ToolClass, ...]] = { + "account": (MrScraperGetAccountInfoTool,), + "discovery": ( + MrScraperCrawlWebsiteUrlsTool, + MrScraperSearchGoogleSerpTool, + ), + "extraction": ( + MrScraperExtractPageByPromptTool, + MrScraperExtractListingsTool, + MrScraperExtractStructuredDataTool, + MrScraperFetchRenderedHtmlTool, + ), + "results": ( + MrScraperGetResultsTool, + MrScraperGetLatestResultsTool, + MrScraperGetResultDetailTool, + ), + "scraper creation": ( + MrScraperCreatePromptScraperTool, + MrScraperCreateListingScraperTool, + MrScraperCreateWebsiteCrawlScraperTool, + ), + "scraper runs": ( + MrScraperRunExistingScraperTool, + MrScraperRunExistingScraperBatchTool, + ), +} + + +def create_mrscraper_toolkit( + *, + groups: Iterable[str] | None = None, + tool_names: Iterable[str] | None = None, + api_token: str | None = None, +) -> list[BaseTool]: + """Create all 15 MrScraper tools or a selected group/name subset. + + Args: + groups: Optional case-insensitive group names: Account, Discovery, + Extraction, Results, Scraper Creation, or Scraper Runs. + tool_names: Optional exact public tool names to select. + api_token: Optional constructor-only credential override. The value remains + private and is excluded from schemas, serialization, reprs, and errors. + + Returns: + New independent tool instances sharing one configured HTTP client. + + Raises: + ValueError: If selection is ambiguous or contains an unknown group/name. + """ + if groups is not None and tool_names is not None: + raise ValueError("Select MrScraper tools by groups or tool_names, not both") + + all_classes = tuple(tool for group in _GROUPS.values() for tool in group) + selected: tuple[ToolClass, ...] + if groups is not None: + normalized = list(dict.fromkeys(group.strip().lower() for group in groups)) + unknown = sorted(set(normalized) - _GROUPS.keys()) + if unknown: + raise ValueError(f"Unknown MrScraper toolkit groups: {', '.join(unknown)}") + selected = tuple(tool for group in normalized for tool in _GROUPS[group]) + elif tool_names is not None: + by_name = {tool.model_fields["name"].default: tool for tool in all_classes} + requested = list(tool_names) + unknown = sorted(set(requested) - by_name.keys()) + if unknown: + raise ValueError(f"Unknown MrScraper tool names: {', '.join(unknown)}") + selected = tuple(by_name[name] for name in requested) + else: + selected = all_classes + + client = MrScraperClient(resolve_api_token(api_token)) + return [tool_class(client=client) for tool_class in selected] + + +__all__ = ["create_mrscraper_toolkit"] diff --git a/lib/crewai-tools/tests/tools/mrscraper/test_mrscraper_tools.py b/lib/crewai-tools/tests/tools/mrscraper/test_mrscraper_tools.py new file mode 100644 index 0000000000..cd9e738107 --- /dev/null +++ b/lib/crewai-tools/tests/tools/mrscraper/test_mrscraper_tools.py @@ -0,0 +1,625 @@ +"""Contract tests for the native MrScraper integration.""" + +import hashlib +import json +from pathlib import Path +from typing import Any + +from crewai import Agent +from crewai.tools import BaseTool +from crewai_tools import ( + MrScraperCrawlWebsiteUrlsTool, + MrScraperCreateListingScraperTool, + MrScraperCreatePromptScraperTool, + MrScraperCreateWebsiteCrawlScraperTool, + MrScraperExtractListingsTool, + MrScraperExtractPageByPromptTool, + MrScraperExtractStructuredDataTool, + MrScraperFetchRenderedHtmlTool, + MrScraperGetAccountInfoTool, + MrScraperGetLatestResultsTool, + MrScraperGetResultDetailTool, + MrScraperGetResultsTool, + MrScraperRunExistingScraperBatchTool, + MrScraperRunExistingScraperTool, + MrScraperSearchGoogleSerpTool, + create_mrscraper_toolkit, +) +from crewai_tools.generate_tool_specs import ToolSpecExtractor +from crewai_tools.tools.mrscraper.client import MrScraperClient +from crewai_tools.tools.mrscraper.extraction import load_structured_data_prompts +from crewai_tools.tools.mrscraper.payloads import ( + append_output_schema, + general_payload, + listing_payload, + map_payload, +) +from crewai_tools.tools.mrscraper.schemas import ( + FetchRenderedHtmlInput, + GetResultsInput, + ListingScraperInput, + MapScraperInput, + RunExistingScraperBatchInput, + RunExistingScraperInput, + SearchGoogleSerpInput, +) +from pydantic import ValidationError +import pytest +import requests + + +FAKE_TOKEN = "conspicuously-fake-mrscraper-token" + +TOOL_CLASSES = ( + MrScraperGetAccountInfoTool, + MrScraperCrawlWebsiteUrlsTool, + MrScraperSearchGoogleSerpTool, + MrScraperExtractPageByPromptTool, + MrScraperExtractListingsTool, + MrScraperExtractStructuredDataTool, + MrScraperFetchRenderedHtmlTool, + MrScraperGetResultsTool, + MrScraperGetLatestResultsTool, + MrScraperGetResultDetailTool, + MrScraperCreatePromptScraperTool, + MrScraperCreateListingScraperTool, + MrScraperCreateWebsiteCrawlScraperTool, + MrScraperRunExistingScraperTool, + MrScraperRunExistingScraperBatchTool, +) + +TOOL_NAMES = ( + "mrscraper_get_account_info", + "mrscraper_crawl_website_urls", + "mrscraper_search_google_serp", + "mrscraper_extract_page_by_prompt", + "mrscraper_extract_listings", + "mrscraper_extract_structured_data", + "mrscraper_fetch_rendered_html", + "mrscraper_get_results", + "mrscraper_get_latest_results", + "mrscraper_get_result_detail", + "mrscraper_create_prompt_scraper", + "mrscraper_create_listing_scraper", + "mrscraper_create_website_crawl_scraper", + "mrscraper_run_existing_scraper", + "mrscraper_run_existing_scraper_batch", +) + + +class FakeResponse: + def __init__( + self, + value: Any = None, + *, + text: str | None = None, + status_code: int = 200, + content_type: str = "application/json", + ) -> None: + self.value = value + self.text = text if text is not None else json.dumps(value) + self.status_code = status_code + self.headers = {"Content-Type": content_type} + + def json(self) -> Any: + return self.value + + +class FakeSession: + def __init__( + self, + response: FakeResponse | None = None, + error: requests.RequestException | None = None, + ) -> None: + self.response = response or FakeResponse({"ok": True}) + self.error = error + self.calls: list[dict[str, Any]] = [] + + def request(self, method: str, url: str, **kwargs: Any) -> FakeResponse: + self.calls.append({"method": method, "url": url, **kwargs}) + if self.error is not None: + raise self.error + return self.response + + +def make_client( + response: FakeResponse | None = None, + error: requests.RequestException | None = None, +) -> tuple[MrScraperClient, FakeSession]: + session = FakeSession(response=response, error=error) + return MrScraperClient(FAKE_TOKEN, session=session), session # type: ignore[arg-type] + + +def test_all_tools_are_public_independent_base_tools() -> None: + client, _ = make_client() + tools = [tool_class(client=client) for tool_class in TOOL_CLASSES] + + assert len(tools) == 15 + assert all(isinstance(tool, BaseTool) for tool in tools) + assert tuple(tool.name for tool in tools) == TOOL_NAMES + assert len({tool.description for tool in tools}) == 15 + assert all(tool.description.strip() for tool in tools) + assert all(tool.args_schema is not BaseTool._ArgsSchemaPlaceholder for tool in tools) + + +def test_toolkit_returns_all_groups_names_and_fresh_state(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("MRSCRAPER_API_TOKEN", FAKE_TOKEN) + first = create_mrscraper_toolkit() + second = create_mrscraper_toolkit() + + assert len(first) == 15 + assert [tool.name for tool in first] == list(TOOL_NAMES) + assert all(left is not right for left, right in zip(first, second, strict=True)) + assert [tool.name for tool in create_mrscraper_toolkit(groups=["Results"])] == [ + "mrscraper_get_results", + "mrscraper_get_latest_results", + "mrscraper_get_result_detail", + ] + assert len(create_mrscraper_toolkit(groups=["Discovery", "Extraction"])) == 6 + selected = create_mrscraper_toolkit( + tool_names=["mrscraper_get_account_info", "mrscraper_get_result_detail"] + ) + assert [tool.name for tool in selected] == [ + "mrscraper_get_account_info", + "mrscraper_get_result_detail", + ] + with pytest.raises(ValueError, match="groups or tool_names"): + create_mrscraper_toolkit(groups=["Account"], tool_names=[TOOL_NAMES[0]]) + with pytest.raises(ValueError, match="Unknown MrScraper toolkit groups"): + create_mrscraper_toolkit(groups=["unknown"]) + + +def test_agent_receives_15_independent_tools(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("MRSCRAPER_API_TOKEN", FAKE_TOKEN) + tools = create_mrscraper_toolkit() + agent = Agent( + role="MrScraper contract test", + goal="Verify independent tool discovery", + backstory="A deterministic test agent", + tools=tools, + ) + + assert len(agent.tools) == 15 + assert [tool.name for tool in agent.tools] == list(TOOL_NAMES) + assert "operation" not in { + field for tool in agent.tools for field in tool.args_schema.model_fields + } + + +def test_schema_required_defaults_enums_constraints_and_descriptions() -> None: + search_schema = SearchGoogleSerpInput.model_json_schema() + assert search_schema["required"] == ["query"] + assert search_schema["properties"]["page"]["default"] == 1 + assert search_schema["properties"]["page"]["minimum"] == 1 + assert search_schema["properties"]["format"]["enum"] == ["json", "html"] + + rendered = FetchRenderedHtmlInput.model_json_schema()["properties"] + assert rendered["max_retries"]["minimum"] == 0 + assert {item.get("minimum") for item in rendered["token_cap"]["anyOf"]} == { + 1, + None, + } + assert ["full", "top"] in [ + item.get("enum") for item in rendered["screenshot_mode"]["anyOf"] + ] + + listing = ListingScraperInput.model_json_schema() + assert listing["required"] == ["url"] + assert listing["properties"]["max_pages"]["default"] == 1 + assert listing["properties"]["max_pages"]["minimum"] == 1 + + for tool_class in TOOL_CLASSES: + schema = tool_class.model_fields["args_schema"].default.model_json_schema() + assert schema["type"] == "object" + for field in schema.get("properties", {}).values(): + assert field.get("description") + + +@pytest.mark.parametrize( + ("schema", "values"), + [ + (SearchGoogleSerpInput, {"query": "x", "page": True}), + (SearchGoogleSerpInput, {"query": "x", "region": "USA"}), + (MapScraperInput, {"url": "https://example.com", "limit": False}), + (ListingScraperInput, {"url": "https://example.com", "max_pages": 1.5}), + (FetchRenderedHtmlInput, {"url": "https://example.com", "max_retries": -1}), + (RunExistingScraperBatchInput, {"scraper_type": "ai", "scraper_id": "id", "urls": []}), + (RunExistingScraperBatchInput, {"scraper_type": "ai", "scraper_id": "id", "urls": [" "]}), + ], +) +def test_strict_schema_validation(schema: Any, values: dict[str, Any]) -> None: + with pytest.raises(ValidationError): + schema.model_validate(values) + + +def test_credentials_are_required_metadata_only_and_secret_safe( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("MRSCRAPER_API_TOKEN", raising=False) + with pytest.raises(ValueError, match="MRSCRAPER_API_TOKEN is required") as exc: + MrScraperGetAccountInfoTool() + assert FAKE_TOKEN not in str(exc.value) + + client, _ = make_client() + for tool_class in TOOL_CLASSES: + tool = tool_class(client=client) + assert [(item.name, item.description, item.required) for item in tool.env_vars] == [ + ("MRSCRAPER_API_TOKEN", "MrScraper API token", True) + ] + schema_text = json.dumps(tool.args_schema.model_json_schema()) + assert "MRSCRAPER_API_TOKEN" not in schema_text + assert FAKE_TOKEN not in schema_text + assert FAKE_TOKEN not in repr(tool) + assert FAKE_TOKEN not in tool.model_dump_json() + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ({"café": False, "zero": 0}, '{"café":false,"zero":0}'), + ([1, False, 0], "[1,false,0]"), + (False, "false"), + (0, "0"), + (None, "null"), + ], +) +def test_client_preserves_json_shapes_and_primary_auth(value: Any, expected: str) -> None: + client, session = make_client(FakeResponse(value)) + result = client.request("GET", "primary", "/api/v1/test") + + assert result == expected + call = session.calls[0] + assert call["url"] == "https://api.app.mrscraper.com/api/v1/test" + assert call["headers"]["x-api-token"] == FAKE_TOKEN + assert call["headers"]["Accept"] == "application/json" + assert call["timeout"] == (10, 660) + + +def test_serp_host_bearer_payload_and_html_preservation() -> None: + html = "\n

exact & unquoted

" + client, session = make_client( + FakeResponse(text=html, content_type="text/html; charset=utf-8") + ) + tool = MrScraperSearchGoogleSerpTool(client=client) + + result = tool.run(query="hotels", format="html", render_js=False) + + assert result == html + call = session.calls[0] + assert call["url"] == ( + "https://sync.scraper.mrscraper.com/api/google/serp/v2/sync" + ) + assert call["headers"]["Authorization"] == f"Bearer {FAKE_TOKEN}" + assert "x-api-token" not in call["headers"] + assert call["json"]["renderJs"] is False + + +def test_rendered_page_query_body_split_and_boolean_text() -> None: + client, session = make_client(FakeResponse(text="", content_type="text/html")) + tool = MrScraperFetchRenderedHtmlTool(client=client) + + assert tool.run( + url="https://target.example", + max_retries=0, + screenshot=False, + html=False, + markdown=False, + block_resources=False, + home_page=False, + return_cookie=False, + super_mode=False, + ) == "" + call = session.calls[0] + assert call["method"] == "POST" + assert call["url"] == "https://api.mrscraper.com/" + assert call["params"] == { + "token": FAKE_TOKEN, + "browserRendering": "true", + "timeout": 300, + "geoCode": "us", + "html": "false", + "markdown": "false", + "proxyCountry": "us", + } + assert call["json"] == { + "url": "https://target.example", + "maxRetries": 0, + } + + tool.run(url="https://target.example", screenshot=True, screenshot_mode="top") + assert session.calls[1]["params"]["screenshot"] == "top" + + tool.run( + url="https://target.example", + screenshot=True, + token_cap=30, + wait_for_selector="#ready", + wait_until="networkidle", + block_resources=True, + home_page=True, + return_cookie=True, + super_mode=True, + ) + advanced = session.calls[2] + assert advanced["params"]["screenshot"] == "full" + assert advanced["params"]["waitForSelector"] == "#ready" + assert advanced["params"]["waitUntil"] == "networkidle" + assert advanced["params"]["blockResources"] == "true" + assert advanced["params"]["returnCookie"] == "true" + assert advanced["params"]["super"] == "true" + assert advanced["json"]["tokenCap"] == 30 + assert advanced["json"]["homePage"] is True + + +def test_general_listing_map_payloads_and_schema_append_once() -> None: + schema = {"name": "string", "enabled": False, "count": 0} + general = general_payload( + url="https://example.com", + prompt="Extract", + output_schema=schema, + mode="Cheap", + proxy_country=None, + ) + listing = listing_payload( + url="https://example.com", + prompt=None, + output_schema={}, + max_pages=1, + proxy_country="ID", + ) + mapping = map_payload( + url="https://example.com", + max_depth=0, + max_pages=0, + limit=1, + include_patterns=None, + exclude_patterns=None, + ) + + assert general["graph"] == "general" + assert general["message"] == ( + "Extract\n\nReturn the output as JSON matching this schema:\n" + '{"name":"string","enabled":false,"count":0}' + ) + assert general["message"].count("Return the output as JSON matching this schema:") == 1 + assert "output_schema" not in general + assert listing["graph"] == "listing" + assert listing["message"] == "Return each item as JSON matching this schema:\n{}" + assert mapping == { + "graph": "map", + "url": "https://example.com", + "maxDepth": 0, + "maxPages": 0, + "limit": 1, + } + assert append_output_schema("p", None, "label") == "p" + + +def test_structured_presets_are_exact_and_selected_without_category() -> None: + preset_path = Path( + "lib/crewai-tools/src/crewai_tools/tools/mrscraper/structured_data_prompts.json" + ) + assert hashlib.sha256(preset_path.read_bytes()).hexdigest() == ( + "3d9c15e8ebe7ad8cb04281251311200c1d3413452f14f252dc9ed3a8aae8533a" + ) + prompts = load_structured_data_prompts() + assert set(prompts) == { + "article", + "forumThread", + "hotel", + "jobPosting", + "post", + "product", + "property", + "restaurant", + "socialMediaProfile", + "tourAttraction", + } + + client, session = make_client() + tool = MrScraperExtractStructuredDataTool(client=client) + for category, expected_prompt in prompts.items(): + tool.run(url="https://example.com", category=category) + body = session.calls[-1]["json"] + assert body["message"] == expected_prompt + assert "category" not in body + + +def test_result_filters_sort_and_encoded_detail_id() -> None: + client, session = make_client() + MrScraperGetResultsTool(client=client).run( + scraper_id="scraper/id", page=0, page_size=0, sort_order="ASC" + ) + assert session.calls[0]["params"] == { + "filters[scraperId]": "scraper/id", + "page": 0, + "pageSize": 0, + "sort": "createdAt", + "sortOrder": "ASC", + } + + MrScraperGetLatestResultsTool(client=client).run(scraper_id="id", count=0) + assert session.calls[1]["params"]["page"] == 1 + assert session.calls[1]["params"]["pageSize"] == 0 + assert session.calls[1]["params"]["sortOrder"] == "DESC" + + MrScraperGetResultDetailTool(client=client).run(result_id="a/b ?#") + assert session.calls[2]["url"].endswith("/api/v1/results/a%2Fb%20%3F%23") + + +@pytest.mark.parametrize( + "values", + [ + {"scraper_type": "manual", "scraper_id": "id", "url": "u", "agent_type": "general"}, + {"scraper_type": "manual", "scraper_id": "id", "url": "u", "max_depth": 2}, + {"scraper_type": "ai", "scraper_id": "id", "url": "u", "cookie_jar": "x"}, + {"scraper_type": "ai", "scraper_id": "id", "url": "u", "stream": False}, + {"scraper_type": "ai", "scraper_id": "id", "url": "u", "agent_type": "map", "screenshot": False}, + {"scraper_type": "ai", "scraper_id": "id", "url": "u", "agent_type": "general", "max_pages": 2}, + ], +) +def test_single_run_rejects_incompatible_conditional_fields(values: dict[str, Any]) -> None: + with pytest.raises(ValidationError, match="do not accept"): + RunExistingScraperInput.model_validate(values) + + +def test_ai_single_run_endpoints_defaults_and_zero_false_preservation() -> None: + client, session = make_client() + tool = MrScraperRunExistingScraperTool(client=client) + + tool.run( + scraper_type="ai", + scraper_id="id", + url="https://example.com", + agent_type="general", + bypass_proxy=False, + html=False, + ) + general = session.calls[0] + assert general["url"].endswith("/api/v1/scrapers-ai-rerun") + assert general["json"]["bypassProxy"] is False + assert general["json"]["html"] is False + assert "agent_type" not in general["json"] + + tool.run( + scraper_type="ai", + scraper_id="id", + url="https://example.com", + agent_type="listing", + max_pages=1, + timeout=1, + stream=False, + ) + listing = session.calls[1]["json"] + assert listing["maxPages"] == 1 + assert listing["timeout"] == 1 + assert listing["stream"] is False + + tool.run( + scraper_type="ai", + scraper_id="id", + url="https://example.com", + agent_type="map", + max_depth=0, + max_pages=1, + limit=1, + ) + mapping = session.calls[2]["json"] + assert mapping["maxDepth"] == 0 + assert mapping["maxPages"] == 1 + assert mapping["limit"] == 1 + assert "bypassProxy" not in mapping + + +def test_single_run_omits_unsupplied_advanced_options() -> None: + client, session = make_client() + tool = MrScraperRunExistingScraperTool(client=client) + + tool.run(scraper_type="ai", scraper_id="ai-id", url="https://example.com") + assert session.calls[0]["json"] == { + "scraperId": "ai-id", + "url": "https://example.com", + "maxRetry": 3, + } + + tool.run(scraper_type="manual", scraper_id="manual-id", url="https://example.com") + assert session.calls[1]["json"] == { + "scraperId": "manual-id", + "url": "https://example.com", + "maxRetry": 3, + } + + +def test_manual_single_run_endpoint_json_values_and_screenshot_string() -> None: + client, session = make_client() + tool = MrScraperRunExistingScraperTool(client=client) + cookies = [{"name": "session", "value": "x"}] + paginator = {"selector": "a.next", "maxPages": 0} + + tool.run( + scraper_type="manual", + scraper_id="id", + url="https://example.com", + max_retry=0, + bypass_proxy=False, + cookies=cookies, + paginator=paginator, + screenshot=False, + token_cap=0, + ) + + call = session.calls[0] + assert call["url"].endswith("/api/v1/scrapers-manual-rerun") + assert call["json"]["maxRetry"] == 0 + assert call["json"]["bypassProxy"] is False + assert call["json"]["cookies"] == cookies + assert call["json"]["paginator"] == paginator + assert call["json"]["screenshot"] == "false" + assert call["json"]["tokenCap"] == 0 + assert "agentType" not in call["json"] + + +@pytest.mark.parametrize( + ("scraper_type", "endpoint"), + [ + ("ai", "/api/v1/scrapers-ai-rerun/bulk"), + ("manual", "/api/v1/scrapers-manual-rerun/bulk"), + ], +) +def test_batch_endpoint_and_array_payload(scraper_type: str, endpoint: str) -> None: + client, session = make_client() + tool = MrScraperRunExistingScraperBatchTool(client=client) + urls = ["https://example.com/1", "https://example.com/2"] + + tool.run(scraper_type=scraper_type, scraper_id="id", urls=urls) + + assert session.calls[0]["url"].endswith(endpoint) + assert session.calls[0]["json"] == {"scraperId": "id", "urls": urls} + + +def test_non_2xx_and_transport_errors_are_truncated_and_redacted() -> None: + secret_body = f"token={FAKE_TOKEN} " + "x" * 2000 + client, _ = make_client(FakeResponse(text=secret_body, status_code=502)) + with pytest.raises(RuntimeError) as http_exc: + client.request("POST", "rendered", "/") + message = str(http_exc.value) + assert "HTTP 502" in message + assert FAKE_TOKEN not in message + assert "[REDACTED]" in message + assert len(message) < 1100 + + rendered_url = f"https://api.mrscraper.com/?token={FAKE_TOKEN}&browserRendering=true" + client, _ = make_client(error=requests.RequestException(rendered_url)) + with pytest.raises(RuntimeError) as transport_exc: + client.request("POST", "rendered", "/") + assert FAKE_TOKEN not in str(transport_exc.value) + assert "token=[REDACTED]" in str(transport_exc.value) + + +def test_success_response_cannot_echo_the_secret() -> None: + client, _ = make_client(FakeResponse({"echo": FAKE_TOKEN})) + result = client.request("GET", "primary", "/api/v1/test") + + assert result == '{"echo":"[REDACTED]"}' + assert FAKE_TOKEN not in result + + +def test_generated_discovery_specs_include_all_tools_without_secrets() -> None: + specs = ToolSpecExtractor().extract_all_tools() + by_class = {spec["name"]: spec for spec in specs} + + for tool_class in TOOL_CLASSES: + spec = by_class[tool_class.__name__] + assert spec["humanized_name"] in TOOL_NAMES + assert spec["run_params_schema"]["type"] == "object" + assert spec["env_vars"] == [ + { + "name": "MRSCRAPER_API_TOKEN", + "description": "MrScraper API token", + "required": True, + "default": None, + } + ] + rendered = json.dumps(spec) + assert FAKE_TOKEN not in rendered + assert "api_token" not in spec["run_params_schema"].get("properties", {}) diff --git a/lib/crewai-tools/tool.specs.json b/lib/crewai-tools/tool.specs.json index 2e540f8c13..0435a8e8b5 100644 --- a/lib/crewai-tools/tool.specs.json +++ b/lib/crewai-tools/tool.specs.json @@ -15212,6 +15212,2077 @@ "type": "object" } }, + { + "description": "Use this potentially expensive immediate Map crawl to discover website URLs. Use the website-crawl creation tool when the intent is to create a reusable scraper.", + "env_vars": [ + { + "default": null, + "description": "MrScraper API token", + "name": "MRSCRAPER_API_TOKEN", + "required": true + } + ], + "humanized_name": "mrscraper_crawl_website_urls", + "init_params_schema": { + "$defs": { + "EnvVar": { + "properties": { + "default": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Default" + }, + "description": { + "title": "Description", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "required": { + "default": true, + "title": "Required", + "type": "boolean" + } + }, + "required": [ + "name", + "description" + ], + "title": "EnvVar", + "type": "object" + }, + "ToolFailurePolicy": { + "description": "How an agent reacts when one of its tools reports a failure.", + "enum": [ + "ignore", + "warn", + "raise" + ], + "title": "ToolFailurePolicy", + "type": "string" + } + }, + "description": "Discover URLs from a starting page.", + "properties": {}, + "required": [], + "title": "MrScraperCrawlWebsiteUrlsTool", + "type": "object" + }, + "name": "MrScraperCrawlWebsiteUrlsTool", + "package_dependencies": [], + "run_params_schema": { + "additionalProperties": false, + "description": "Inputs shared by immediate and reusable website crawl tools.", + "properties": { + "exclude_patterns": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional pipe-separated regular expressions for URLs to exclude.", + "title": "Exclude Patterns" + }, + "include_patterns": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional pipe-separated regular expressions for URLs to include.", + "title": "Include Patterns" + }, + "limit": { + "default": 50, + "description": "Maximum URLs to return; defaults to 50; minimum 1.", + "minimum": 1, + "title": "Limit", + "type": "integer" + }, + "max_depth": { + "default": 2, + "description": "Maximum link depth to crawl; defaults to 2.", + "title": "Max Depth", + "type": "integer" + }, + "max_pages": { + "default": 50, + "description": "Maximum pages to evaluate; defaults to 50.", + "title": "Max Pages", + "type": "integer" + }, + "url": { + "description": "Required starting URL to crawl.", + "minLength": 1, + "title": "Url", + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "MapScraperInput", + "type": "object" + } + }, + { + "description": "Use this to create a reusable Listing AI scraper for repeated or paginated items. Use extract_listings when the intent is immediate extraction.", + "env_vars": [ + { + "default": null, + "description": "MrScraper API token", + "name": "MRSCRAPER_API_TOKEN", + "required": true + } + ], + "humanized_name": "mrscraper_create_listing_scraper", + "init_params_schema": { + "$defs": { + "EnvVar": { + "properties": { + "default": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Default" + }, + "description": { + "title": "Description", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "required": { + "default": true, + "title": "Required", + "type": "boolean" + } + }, + "required": [ + "name", + "description" + ], + "title": "EnvVar", + "type": "object" + }, + "ToolFailurePolicy": { + "description": "How an agent reacts when one of its tools reports a failure.", + "enum": [ + "ignore", + "warn", + "raise" + ], + "title": "ToolFailurePolicy", + "type": "string" + } + }, + "description": "Create a reusable Listing AI scraper.", + "properties": {}, + "required": [], + "title": "MrScraperCreateListingScraperTool", + "type": "object" + }, + "name": "MrScraperCreateListingScraperTool", + "package_dependencies": [], + "run_params_schema": { + "additionalProperties": false, + "description": "Inputs shared by immediate and reusable listing scrapers.", + "properties": { + "max_pages": { + "default": 1, + "description": "Maximum pagination pages to scrape; defaults to 1; minimum 1.", + "minimum": 1, + "title": "Max Pages", + "type": "integer" + }, + "output_schema": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional JSON object describing each expected listing item.", + "title": "Output Schema" + }, + "prompt": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional instructions describing each listing item to extract.", + "title": "Prompt" + }, + "proxy_country": { + "anyOf": [ + { + "pattern": "^[A-Za-z]{2}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional ISO country code for the proxy.", + "title": "Proxy Country" + }, + "url": { + "description": "Required listing page URL to scrape.", + "minLength": 1, + "title": "Url", + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "ListingScraperInput", + "type": "object" + } + }, + { + "description": "Use this to create a reusable General AI scraper from a page, prompt, and optional output schema. Use extract_page_by_prompt for immediate one-page extraction intent.", + "env_vars": [ + { + "default": null, + "description": "MrScraper API token", + "name": "MRSCRAPER_API_TOKEN", + "required": true + } + ], + "humanized_name": "mrscraper_create_prompt_scraper", + "init_params_schema": { + "$defs": { + "EnvVar": { + "properties": { + "default": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Default" + }, + "description": { + "title": "Description", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "required": { + "default": true, + "title": "Required", + "type": "boolean" + } + }, + "required": [ + "name", + "description" + ], + "title": "EnvVar", + "type": "object" + }, + "ToolFailurePolicy": { + "description": "How an agent reacts when one of its tools reports a failure.", + "enum": [ + "ignore", + "warn", + "raise" + ], + "title": "ToolFailurePolicy", + "type": "string" + } + }, + "description": "Create a reusable General AI scraper.", + "properties": {}, + "required": [], + "title": "MrScraperCreatePromptScraperTool", + "type": "object" + }, + "name": "MrScraperCreatePromptScraperTool", + "package_dependencies": [], + "run_params_schema": { + "additionalProperties": false, + "description": "Inputs shared by immediate and reusable prompt scrapers.", + "properties": { + "mode": { + "default": "Super", + "description": "Scraping mode, 'Super' or 'Cheap'; defaults to 'Super'.", + "enum": [ + "Super", + "Cheap" + ], + "title": "Mode", + "type": "string" + }, + "output_schema": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional JSON object describing the expected output shape.", + "title": "Output Schema" + }, + "prompt": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional extraction instructions for the AI scraper.", + "title": "Prompt" + }, + "proxy_country": { + "anyOf": [ + { + "pattern": "^[A-Za-z]{2}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional ISO country code for the proxy.", + "title": "Proxy Country" + }, + "url": { + "description": "Required page URL to scrape.", + "minLength": 1, + "title": "Url", + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "GeneralScraperInput", + "type": "object" + } + }, + { + "description": "Use this potentially expensive operation to create a reusable Map scraper for URL discovery. Use crawl_website_urls for immediate crawl intent.", + "env_vars": [ + { + "default": null, + "description": "MrScraper API token", + "name": "MRSCRAPER_API_TOKEN", + "required": true + } + ], + "humanized_name": "mrscraper_create_website_crawl_scraper", + "init_params_schema": { + "$defs": { + "EnvVar": { + "properties": { + "default": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Default" + }, + "description": { + "title": "Description", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "required": { + "default": true, + "title": "Required", + "type": "boolean" + } + }, + "required": [ + "name", + "description" + ], + "title": "EnvVar", + "type": "object" + }, + "ToolFailurePolicy": { + "description": "How an agent reacts when one of its tools reports a failure.", + "enum": [ + "ignore", + "warn", + "raise" + ], + "title": "ToolFailurePolicy", + "type": "string" + } + }, + "description": "Create a reusable Map AI scraper.", + "properties": {}, + "required": [], + "title": "MrScraperCreateWebsiteCrawlScraperTool", + "type": "object" + }, + "name": "MrScraperCreateWebsiteCrawlScraperTool", + "package_dependencies": [], + "run_params_schema": { + "additionalProperties": false, + "description": "Inputs shared by immediate and reusable website crawl tools.", + "properties": { + "exclude_patterns": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional pipe-separated regular expressions for URLs to exclude.", + "title": "Exclude Patterns" + }, + "include_patterns": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional pipe-separated regular expressions for URLs to include.", + "title": "Include Patterns" + }, + "limit": { + "default": 50, + "description": "Maximum URLs to return; defaults to 50; minimum 1.", + "minimum": 1, + "title": "Limit", + "type": "integer" + }, + "max_depth": { + "default": 2, + "description": "Maximum link depth to crawl; defaults to 2.", + "title": "Max Depth", + "type": "integer" + }, + "max_pages": { + "default": 50, + "description": "Maximum pages to evaluate; defaults to 50.", + "title": "Max Pages", + "type": "integer" + }, + "url": { + "description": "Required starting URL to crawl.", + "minLength": 1, + "title": "Url", + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "MapScraperInput", + "type": "object" + } + }, + { + "description": "Use this potentially multi-page immediate extraction for repeated listings or paginated content. Use create_listing_scraper for reusable scraper creation.", + "env_vars": [ + { + "default": null, + "description": "MrScraper API token", + "name": "MRSCRAPER_API_TOKEN", + "required": true + } + ], + "humanized_name": "mrscraper_extract_listings", + "init_params_schema": { + "$defs": { + "EnvVar": { + "properties": { + "default": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Default" + }, + "description": { + "title": "Description", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "required": { + "default": true, + "title": "Required", + "type": "boolean" + } + }, + "required": [ + "name", + "description" + ], + "title": "EnvVar", + "type": "object" + }, + "ToolFailurePolicy": { + "description": "How an agent reacts when one of its tools reports a failure.", + "enum": [ + "ignore", + "warn", + "raise" + ], + "title": "ToolFailurePolicy", + "type": "string" + } + }, + "description": "Perform immediate Listing extraction.", + "properties": {}, + "required": [], + "title": "MrScraperExtractListingsTool", + "type": "object" + }, + "name": "MrScraperExtractListingsTool", + "package_dependencies": [], + "run_params_schema": { + "additionalProperties": false, + "description": "Inputs shared by immediate and reusable listing scrapers.", + "properties": { + "max_pages": { + "default": 1, + "description": "Maximum pagination pages to scrape; defaults to 1; minimum 1.", + "minimum": 1, + "title": "Max Pages", + "type": "integer" + }, + "output_schema": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional JSON object describing each expected listing item.", + "title": "Output Schema" + }, + "prompt": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional instructions describing each listing item to extract.", + "title": "Prompt" + }, + "proxy_country": { + "anyOf": [ + { + "pattern": "^[A-Za-z]{2}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional ISO country code for the proxy.", + "title": "Proxy Country" + }, + "url": { + "description": "Required listing page URL to scrape.", + "minLength": 1, + "title": "Url", + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "ListingScraperInput", + "type": "object" + } + }, + { + "description": "Use this for immediate AI extraction from one page using a prompt. Use create_prompt_scraper when the primary intent is reusable scraper creation.", + "env_vars": [ + { + "default": null, + "description": "MrScraper API token", + "name": "MRSCRAPER_API_TOKEN", + "required": true + } + ], + "humanized_name": "mrscraper_extract_page_by_prompt", + "init_params_schema": { + "$defs": { + "EnvVar": { + "properties": { + "default": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Default" + }, + "description": { + "title": "Description", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "required": { + "default": true, + "title": "Required", + "type": "boolean" + } + }, + "required": [ + "name", + "description" + ], + "title": "EnvVar", + "type": "object" + }, + "ToolFailurePolicy": { + "description": "How an agent reacts when one of its tools reports a failure.", + "enum": [ + "ignore", + "warn", + "raise" + ], + "title": "ToolFailurePolicy", + "type": "string" + } + }, + "description": "Perform immediate General extraction.", + "properties": {}, + "required": [], + "title": "MrScraperExtractPageByPromptTool", + "type": "object" + }, + "name": "MrScraperExtractPageByPromptTool", + "package_dependencies": [], + "run_params_schema": { + "additionalProperties": false, + "description": "Inputs shared by immediate and reusable prompt scrapers.", + "properties": { + "mode": { + "default": "Super", + "description": "Scraping mode, 'Super' or 'Cheap'; defaults to 'Super'.", + "enum": [ + "Super", + "Cheap" + ], + "title": "Mode", + "type": "string" + }, + "output_schema": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional JSON object describing the expected output shape.", + "title": "Output Schema" + }, + "prompt": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional extraction instructions for the AI scraper.", + "title": "Prompt" + }, + "proxy_country": { + "anyOf": [ + { + "pattern": "^[A-Za-z]{2}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional ISO country code for the proxy.", + "title": "Proxy Country" + }, + "url": { + "description": "Required page URL to scrape.", + "minLength": 1, + "title": "Url", + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "GeneralScraperInput", + "type": "object" + } + }, + { + "description": "Use this immediate extraction tool when a page matches one supported structured category, such as article, product, hotel, job, property, restaurant, or tour.", + "env_vars": [ + { + "default": null, + "description": "MrScraper API token", + "name": "MRSCRAPER_API_TOKEN", + "required": true + } + ], + "humanized_name": "mrscraper_extract_structured_data", + "init_params_schema": { + "$defs": { + "EnvVar": { + "properties": { + "default": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Default" + }, + "description": { + "title": "Description", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "required": { + "default": true, + "title": "Required", + "type": "boolean" + } + }, + "required": [ + "name", + "description" + ], + "title": "EnvVar", + "type": "object" + }, + "ToolFailurePolicy": { + "description": "How an agent reacts when one of its tools reports a failure.", + "enum": [ + "ignore", + "warn", + "raise" + ], + "title": "ToolFailurePolicy", + "type": "string" + } + }, + "description": "Extract one of the bundled structured-data presets.", + "properties": {}, + "required": [], + "title": "MrScraperExtractStructuredDataTool", + "type": "object" + }, + "name": "MrScraperExtractStructuredDataTool", + "package_dependencies": [], + "run_params_schema": { + "additionalProperties": false, + "description": "Inputs for preset structured-data extraction.", + "properties": { + "category": { + "default": "article", + "description": "Structured extraction preset category; defaults to 'article'.", + "enum": [ + "article", + "forumThread", + "hotel", + "jobPosting", + "post", + "product", + "property", + "restaurant", + "socialMediaProfile", + "tourAttraction" + ], + "title": "Category", + "type": "string" + }, + "mode": { + "default": "Super", + "description": "Scraping mode, 'Super' or 'Cheap'; defaults to 'Super'.", + "enum": [ + "Super", + "Cheap" + ], + "title": "Mode", + "type": "string" + }, + "proxy_country": { + "anyOf": [ + { + "pattern": "^[A-Za-z]{2}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional ISO country code for the proxy.", + "title": "Proxy Country" + }, + "url": { + "description": "Required page URL to scrape.", + "minLength": 1, + "title": "Url", + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "ExtractStructuredDataInput", + "type": "object" + } + }, + { + "description": "Use this immediate stealth-browser call when JavaScript-rendered HTML, Markdown, cookies, or a screenshot is needed. Keep the requested outputs narrow to control cost.", + "env_vars": [ + { + "default": null, + "description": "MrScraper API token", + "name": "MRSCRAPER_API_TOKEN", + "required": true + } + ], + "humanized_name": "mrscraper_fetch_rendered_html", + "init_params_schema": { + "$defs": { + "EnvVar": { + "properties": { + "default": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Default" + }, + "description": { + "title": "Description", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "required": { + "default": true, + "title": "Required", + "type": "boolean" + } + }, + "required": [ + "name", + "description" + ], + "title": "EnvVar", + "type": "object" + }, + "ToolFailurePolicy": { + "description": "How an agent reacts when one of its tools reports a failure.", + "enum": [ + "ignore", + "warn", + "raise" + ], + "title": "ToolFailurePolicy", + "type": "string" + } + }, + "description": "Fetch a browser-rendered page.", + "properties": {}, + "required": [], + "title": "MrScraperFetchRenderedHtmlTool", + "type": "object" + }, + "name": "MrScraperFetchRenderedHtmlTool", + "package_dependencies": [], + "run_params_schema": { + "additionalProperties": false, + "description": "Inputs for the rendered-page API.", + "properties": { + "block_resources": { + "default": true, + "description": "Whether to block images, fonts, and stylesheets; defaults to true.", + "title": "Block Resources", + "type": "boolean" + }, + "geo_code": { + "default": "us", + "description": "Geolocation country code; defaults to 'us'.", + "pattern": "^[A-Za-z]{2}$", + "title": "Geo Code", + "type": "string" + }, + "home_page": { + "default": false, + "description": "Whether to visit the site home page first; defaults to false.", + "title": "Home Page", + "type": "boolean" + }, + "html": { + "default": true, + "description": "Whether to include rendered HTML; defaults to true.", + "title": "Html", + "type": "boolean" + }, + "markdown": { + "default": false, + "description": "Whether to include Markdown; defaults to false.", + "title": "Markdown", + "type": "boolean" + }, + "max_retries": { + "default": 3, + "description": "Maximum retry attempts; defaults to 3; minimum 0.", + "minimum": 0, + "title": "Max Retries", + "type": "integer" + }, + "proxy_country": { + "default": "us", + "description": "Proxy country code; defaults to 'us'.", + "pattern": "^[A-Za-z]{2}$", + "title": "Proxy Country", + "type": "string" + }, + "return_cookie": { + "default": true, + "description": "Whether to include browser cookies; defaults to true.", + "title": "Return Cookie", + "type": "boolean" + }, + "screenshot": { + "default": false, + "description": "Whether to capture a screenshot; defaults to false.", + "title": "Screenshot", + "type": "boolean" + }, + "screenshot_mode": { + "default": "full", + "description": "Screenshot mode, 'full' or 'top'; used only when screenshot is true.", + "enum": [ + "full", + "top" + ], + "title": "Screenshot Mode", + "type": "string" + }, + "super_mode": { + "default": true, + "description": "Whether to use stronger device mode; defaults to true.", + "title": "Super Mode", + "type": "boolean" + }, + "timeout": { + "default": 300, + "description": "Maximum page-load time in seconds; defaults to 300; minimum 1.", + "minimum": 1, + "title": "Timeout", + "type": "integer" + }, + "token_cap": { + "default": 30, + "description": "Maximum processing token allowance; defaults to 30; minimum 1.", + "minimum": 1, + "title": "Token Cap", + "type": "integer" + }, + "url": { + "description": "Required target URL to render.", + "minLength": 1, + "title": "Url", + "type": "string" + }, + "wait_for_selector": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional CSS selector to await before returning.", + "title": "Wait For Selector" + }, + "wait_until": { + "default": "domcontentloaded", + "description": "Browser lifecycle event to await; defaults to 'domcontentloaded'.", + "enum": [ + "domcontentloaded", + "load", + "networkidle" + ], + "title": "Wait Until", + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "FetchRenderedHtmlInput", + "type": "object" + } + }, + { + "description": "Use this narrow read-only tool to inspect MrScraper account details, token usage, and token limits. It does not scrape a page or create a job.", + "env_vars": [ + { + "default": null, + "description": "MrScraper API token", + "name": "MRSCRAPER_API_TOKEN", + "required": true + } + ], + "humanized_name": "mrscraper_get_account_info", + "init_params_schema": { + "$defs": { + "EnvVar": { + "properties": { + "default": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Default" + }, + "description": { + "title": "Description", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "required": { + "default": true, + "title": "Required", + "type": "boolean" + } + }, + "required": [ + "name", + "description" + ], + "title": "EnvVar", + "type": "object" + }, + "ToolFailurePolicy": { + "description": "How an agent reacts when one of its tools reports a failure.", + "enum": [ + "ignore", + "warn", + "raise" + ], + "title": "ToolFailurePolicy", + "type": "string" + } + }, + "description": "Retrieve subscription and token usage information.", + "properties": {}, + "required": [], + "title": "MrScraperGetAccountInfoTool", + "type": "object" + }, + "name": "MrScraperGetAccountInfoTool", + "package_dependencies": [], + "run_params_schema": { + "additionalProperties": false, + "description": "The account operation has no model-supplied inputs.", + "properties": {}, + "title": "GetAccountInfoInput", + "type": "object" + } + }, + { + "description": "Use this shortcut for the newest N results from one scraper. Use get_results instead when page navigation or ascending order is required.", + "env_vars": [ + { + "default": null, + "description": "MrScraper API token", + "name": "MRSCRAPER_API_TOKEN", + "required": true + } + ], + "humanized_name": "mrscraper_get_latest_results", + "init_params_schema": { + "$defs": { + "EnvVar": { + "properties": { + "default": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Default" + }, + "description": { + "title": "Description", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "required": { + "default": true, + "title": "Required", + "type": "boolean" + } + }, + "required": [ + "name", + "description" + ], + "title": "EnvVar", + "type": "object" + }, + "ToolFailurePolicy": { + "description": "How an agent reacts when one of its tools reports a failure.", + "enum": [ + "ignore", + "warn", + "raise" + ], + "title": "ToolFailurePolicy", + "type": "string" + } + }, + "description": "Retrieve the newest N results.", + "properties": {}, + "required": [], + "title": "MrScraperGetLatestResultsTool", + "type": "object" + }, + "name": "MrScraperGetLatestResultsTool", + "package_dependencies": [], + "run_params_schema": { + "additionalProperties": false, + "description": "Inputs for the newest scraper results.", + "properties": { + "count": { + "default": 10, + "description": "Number of newest results; defaults to 10.", + "title": "Count", + "type": "integer" + }, + "scraper_id": { + "description": "Required scraper ID whose newest results to list.", + "minLength": 1, + "title": "Scraper Id", + "type": "string" + } + }, + "required": [ + "scraper_id" + ], + "title": "GetLatestResultsInput", + "type": "object" + } + }, + { + "description": "Use this narrow lookup when a specific MrScraper result ID is already known. It returns that record rather than a paginated collection.", + "env_vars": [ + { + "default": null, + "description": "MrScraper API token", + "name": "MRSCRAPER_API_TOKEN", + "required": true + } + ], + "humanized_name": "mrscraper_get_result_detail", + "init_params_schema": { + "$defs": { + "EnvVar": { + "properties": { + "default": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Default" + }, + "description": { + "title": "Description", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "required": { + "default": true, + "title": "Required", + "type": "boolean" + } + }, + "required": [ + "name", + "description" + ], + "title": "EnvVar", + "type": "object" + }, + "ToolFailurePolicy": { + "description": "How an agent reacts when one of its tools reports a failure.", + "enum": [ + "ignore", + "warn", + "raise" + ], + "title": "ToolFailurePolicy", + "type": "string" + } + }, + "description": "Retrieve one result by ID.", + "properties": {}, + "required": [], + "title": "MrScraperGetResultDetailTool", + "type": "object" + }, + "name": "MrScraperGetResultDetailTool", + "package_dependencies": [], + "run_params_schema": { + "additionalProperties": false, + "description": "Inputs for one result record.", + "properties": { + "result_id": { + "description": "Required result ID to retrieve.", + "minLength": 1, + "title": "Result Id", + "type": "string" + } + }, + "required": [ + "result_id" + ], + "title": "GetResultDetailInput", + "type": "object" + } + }, + { + "description": "Use this to page through results for one scraper with explicit paging and sort controls. Use get_latest_results when only the newest N records are needed.", + "env_vars": [ + { + "default": null, + "description": "MrScraper API token", + "name": "MRSCRAPER_API_TOKEN", + "required": true + } + ], + "humanized_name": "mrscraper_get_results", + "init_params_schema": { + "$defs": { + "EnvVar": { + "properties": { + "default": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Default" + }, + "description": { + "title": "Description", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "required": { + "default": true, + "title": "Required", + "type": "boolean" + } + }, + "required": [ + "name", + "description" + ], + "title": "EnvVar", + "type": "object" + }, + "ToolFailurePolicy": { + "description": "How an agent reacts when one of its tools reports a failure.", + "enum": [ + "ignore", + "warn", + "raise" + ], + "title": "ToolFailurePolicy", + "type": "string" + } + }, + "description": "Retrieve a configurable page of results.", + "properties": {}, + "required": [], + "title": "MrScraperGetResultsTool", + "type": "object" + }, + "name": "MrScraperGetResultsTool", + "package_dependencies": [], + "run_params_schema": { + "additionalProperties": false, + "description": "Inputs for paginated scraper results.", + "properties": { + "page": { + "default": 1, + "description": "Results page number; defaults to 1.", + "title": "Page", + "type": "integer" + }, + "page_size": { + "default": 10, + "description": "Number of results per page; defaults to 10.", + "title": "Page Size", + "type": "integer" + }, + "scraper_id": { + "description": "Required scraper ID whose results to list.", + "minLength": 1, + "title": "Scraper Id", + "type": "string" + }, + "sort_by": { + "const": "createdAt", + "default": "createdAt", + "description": "Sort field; only 'createdAt' is supported.", + "title": "Sort By", + "type": "string" + }, + "sort_order": { + "default": "DESC", + "description": "Sort direction, 'ASC' or 'DESC'; defaults to 'DESC'.", + "enum": [ + "ASC", + "DESC" + ], + "title": "Sort Order", + "type": "string" + } + }, + "required": [ + "scraper_id" + ], + "title": "GetResultsInput", + "type": "object" + } + }, + { + "description": "Use this potentially expensive batch operation to run multiple URLs through one existing AI or manual scraper. Use run_existing_scraper for a single URL.", + "env_vars": [ + { + "default": null, + "description": "MrScraper API token", + "name": "MRSCRAPER_API_TOKEN", + "required": true + } + ], + "humanized_name": "mrscraper_run_existing_scraper_batch", + "init_params_schema": { + "$defs": { + "EnvVar": { + "properties": { + "default": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Default" + }, + "description": { + "title": "Description", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "required": { + "default": true, + "title": "Required", + "type": "boolean" + } + }, + "required": [ + "name", + "description" + ], + "title": "EnvVar", + "type": "object" + }, + "ToolFailurePolicy": { + "description": "How an agent reacts when one of its tools reports a failure.", + "enum": [ + "ignore", + "warn", + "raise" + ], + "title": "ToolFailurePolicy", + "type": "string" + } + }, + "description": "Run an existing AI or manual scraper on a URL batch.", + "properties": {}, + "required": [], + "title": "MrScraperRunExistingScraperBatchTool", + "type": "object" + }, + "name": "MrScraperRunExistingScraperBatchTool", + "package_dependencies": [], + "run_params_schema": { + "additionalProperties": false, + "description": "Inputs for batch runs of an existing scraper.", + "properties": { + "scraper_id": { + "description": "Required existing scraper ID.", + "minLength": 1, + "title": "Scraper Id", + "type": "string" + }, + "scraper_type": { + "description": "Required scraper kind selecting the AI or manual bulk endpoint.", + "enum": [ + "ai", + "manual" + ], + "title": "Scraper Type", + "type": "string" + }, + "urls": { + "description": "Required nonempty array of nonblank URLs to process in this batch.", + "items": { + "minLength": 1, + "type": "string" + }, + "minItems": 1, + "title": "Urls", + "type": "array" + } + }, + "required": [ + "scraper_type", + "scraper_id", + "urls" + ], + "title": "RunExistingScraperBatchInput", + "type": "object" + } + }, + { + "description": "Use this to run one URL through an existing AI or manual scraper. Choose the AI agent type carefully; conditional options are validated before any request.", + "env_vars": [ + { + "default": null, + "description": "MrScraper API token", + "name": "MRSCRAPER_API_TOKEN", + "required": true + } + ], + "humanized_name": "mrscraper_run_existing_scraper", + "init_params_schema": { + "$defs": { + "EnvVar": { + "properties": { + "default": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Default" + }, + "description": { + "title": "Description", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "required": { + "default": true, + "title": "Required", + "type": "boolean" + } + }, + "required": [ + "name", + "description" + ], + "title": "EnvVar", + "type": "object" + }, + "ToolFailurePolicy": { + "description": "How an agent reacts when one of its tools reports a failure.", + "enum": [ + "ignore", + "warn", + "raise" + ], + "title": "ToolFailurePolicy", + "type": "string" + } + }, + "description": "Run an existing AI or manual scraper on one URL.", + "properties": {}, + "required": [], + "title": "MrScraperRunExistingScraperTool", + "type": "object" + }, + "name": "MrScraperRunExistingScraperTool", + "package_dependencies": [], + "run_params_schema": { + "additionalProperties": false, + "description": "Stable conditional schema for AI and manual single scraper runs.", + "properties": { + "agent_type": { + "default": "general", + "description": "AI agent type; defaults to 'general' for AI and is forbidden for manual runs.", + "enum": [ + "general", + "listing", + "map" + ], + "title": "Agent Type", + "type": "string" + }, + "bypass_proxy": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "General/Listing default false; Manual default true; forbidden for Map.", + "title": "Bypass Proxy" + }, + "cookie_jar": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional Manual cookie-jar identifier or value.", + "title": "Cookie Jar" + }, + "cookies": { + "description": "Manual browser-cookie objects; defaults to an empty array.", + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Cookies", + "type": "array" + }, + "exclude_patterns": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional Map exclude-pattern expressions.", + "title": "Exclude Patterns" + }, + "home_page": { + "default": false, + "description": "Manual home-page visit flag; defaults to false.", + "title": "Home Page", + "type": "boolean" + }, + "home_page_timeout": { + "default": 10, + "description": "Manual home-page timeout; defaults to 10; minimum 1.", + "minimum": 1, + "title": "Home Page Timeout", + "type": "integer" + }, + "html": { + "default": false, + "description": "General, Listing, or Manual HTML output flag; defaults to false.", + "title": "Html", + "type": "boolean" + }, + "include_patterns": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional Map include-pattern expressions.", + "title": "Include Patterns" + }, + "limit": { + "default": 50, + "description": "Map result limit; defaults to 50; minimum 1.", + "minimum": 1, + "title": "Limit", + "type": "integer" + }, + "markdown": { + "default": false, + "description": "General, Listing, or Manual Markdown output flag; defaults to false.", + "title": "Markdown", + "type": "boolean" + }, + "max_depth": { + "default": 2, + "description": "Map crawl depth; defaults to 2; minimum 0.", + "minimum": 0, + "title": "Max Depth", + "type": "integer" + }, + "max_pages": { + "anyOf": [ + { + "minimum": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Listing defaults to 5; Map defaults to 50; minimum 1.", + "title": "Max Pages" + }, + "max_retry": { + "default": 3, + "description": "Maximum retry attempts; defaults to 3; minimum 0.", + "minimum": 0, + "title": "Max Retry", + "type": "integer" + }, + "paginator": { + "additionalProperties": true, + "description": "Manual paginator configuration; defaults to an empty object.", + "title": "Paginator", + "type": "object" + }, + "proxy": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional Manual proxy URL.", + "title": "Proxy" + }, + "proxy_country": { + "anyOf": [ + { + "pattern": "^[A-Za-z]{2}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional proxy country code.", + "title": "Proxy Country" + }, + "record": { + "default": false, + "description": "Manual browser-session recording flag; defaults to false.", + "title": "Record", + "type": "boolean" + }, + "render_javascript": { + "default": false, + "description": "General/Listing JavaScript rendering flag; defaults to false.", + "title": "Render Javascript", + "type": "boolean" + }, + "return_cookie": { + "default": false, + "description": "Manual cookie-return flag; defaults to false.", + "title": "Return Cookie", + "type": "boolean" + }, + "return_cookies": { + "default": false, + "description": "General/Listing cookie-return flag; defaults to false.", + "title": "Return Cookies", + "type": "boolean" + }, + "scraper_id": { + "description": "Required existing scraper ID.", + "minLength": 1, + "title": "Scraper Id", + "type": "string" + }, + "scraper_type": { + "description": "Required scraper kind selecting the AI or manual endpoint.", + "enum": [ + "ai", + "manual" + ], + "title": "Scraper Type", + "type": "string" + }, + "screenshot": { + "default": false, + "description": "General, Listing, or Manual screenshot flag; defaults to false.", + "title": "Screenshot", + "type": "boolean" + }, + "stream": { + "default": false, + "description": "Listing or Manual streaming flag; defaults to false.", + "title": "Stream", + "type": "boolean" + }, + "timeout": { + "anyOf": [ + { + "minimum": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Listing timeout defaults to 300; Manual timeout defaults to 600; minimum 1.", + "title": "Timeout" + }, + "token_cap": { + "default": 0, + "description": "Manual token cap; defaults to 0; minimum 0.", + "minimum": 0, + "title": "Token Cap", + "type": "integer" + }, + "url": { + "description": "Required URL to process in this run.", + "minLength": 1, + "title": "Url", + "type": "string" + }, + "use_home_page": { + "default": false, + "description": "General/Listing home-page visit flag; defaults to false.", + "title": "Use Home Page", + "type": "boolean" + }, + "wait_for_selector": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional General/Listing CSS selector to await.", + "title": "Wait For Selector" + } + }, + "required": [ + "scraper_type", + "scraper_id", + "url" + ], + "title": "RunExistingScraperInput", + "type": "object" + } + }, + { + "description": "Use this for one narrow synchronous Google search. It returns compact JSON text for JSON format or the exact upstream HTML string for HTML format.", + "env_vars": [ + { + "default": null, + "description": "MrScraper API token", + "name": "MRSCRAPER_API_TOKEN", + "required": true + } + ], + "humanized_name": "mrscraper_search_google_serp", + "init_params_schema": { + "$defs": { + "EnvVar": { + "properties": { + "default": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Default" + }, + "description": { + "title": "Description", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "required": { + "default": true, + "title": "Required", + "type": "boolean" + } + }, + "required": [ + "name", + "description" + ], + "title": "EnvVar", + "type": "object" + }, + "ToolFailurePolicy": { + "description": "How an agent reacts when one of its tools reports a failure.", + "enum": [ + "ignore", + "warn", + "raise" + ], + "title": "ToolFailurePolicy", + "type": "string" + } + }, + "description": "Search Google synchronously through MrScraper.", + "properties": {}, + "required": [], + "title": "MrScraperSearchGoogleSerpTool", + "type": "object" + }, + "name": "MrScraperSearchGoogleSerpTool", + "package_dependencies": [], + "run_params_schema": { + "additionalProperties": false, + "description": "Inputs for synchronous Google SERP search.", + "properties": { + "format": { + "default": "json", + "description": "Response format: 'json' or 'html'; defaults to 'json'.", + "enum": [ + "json", + "html" + ], + "title": "Format", + "type": "string" + }, + "language": { + "default": "en", + "description": "Two-letter result language code; defaults to 'en'.", + "pattern": "^[A-Za-z]{2}$", + "title": "Language", + "type": "string" + }, + "page": { + "default": 1, + "description": "Google result page number; defaults to 1; minimum 1.", + "minimum": 1, + "title": "Page", + "type": "integer" + }, + "query": { + "description": "Required Google search query.", + "minLength": 1, + "title": "Query", + "type": "string" + }, + "region": { + "default": "us", + "description": "Two-letter result region code; defaults to 'us'.", + "pattern": "^[A-Za-z]{2}$", + "title": "Region", + "type": "string" + }, + "render_js": { + "default": false, + "description": "Whether to render JavaScript before collecting results; defaults to false.", + "title": "Render Js", + "type": "boolean" + } + }, + "required": [ + "query" + ], + "title": "SearchGoogleSerpInput", + "type": "object" + } + }, { "description": "Multion gives the ability for LLMs to control web browsers using natural language instructions.\n If the status is 'CONTINUE', reissue the same instruction to continue execution", "env_vars": [ diff --git a/scripts/test_mrscraper_real.py b/scripts/test_mrscraper_real.py new file mode 100644 index 0000000000..62c77259db --- /dev/null +++ b/scripts/test_mrscraper_real.py @@ -0,0 +1,280 @@ +"""Run opt-in, real API smoke tests for every MrScraper CrewAI tool. + +Examples: + uv run python scripts/test_mrscraper_real.py --list + uv run python scripts/test_mrscraper_real.py --test account + uv run python scripts/test_mrscraper_real.py --test rendered_html + uv run python scripts/test_mrscraper_real.py --test crew_agent + +These tests call real services and may consume MrScraper or LLM credits. Run one +test at a time. Scraper creation tests create persistent scraper records. +""" + +# ruff: noqa: T201 - This is an intentionally interactive command-line script. + +from __future__ import annotations + +import argparse +from collections.abc import Callable +import os +from typing import Any + +from crewai import Agent, Crew, Task +from crewai_tools import ( + MrScraperCrawlWebsiteUrlsTool, + MrScraperCreateListingScraperTool, + MrScraperCreatePromptScraperTool, + MrScraperCreateWebsiteCrawlScraperTool, + MrScraperExtractListingsTool, + MrScraperExtractPageByPromptTool, + MrScraperExtractStructuredDataTool, + MrScraperFetchRenderedHtmlTool, + MrScraperGetAccountInfoTool, + MrScraperGetLatestResultsTool, + MrScraperGetResultDetailTool, + MrScraperGetResultsTool, + MrScraperRunExistingScraperBatchTool, + MrScraperRunExistingScraperTool, + MrScraperSearchGoogleSerpTool, +) + + +# Read credentials from the environment so they cannot be committed accidentally. +MRSCRAPER_API_TOKEN = os.getenv("MRSCRAPER_API_TOKEN", "") +OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "") + +# Safe public targets for a first smoke test. Replace them as needed. +TARGET_URL = "https://www.cireba.com/property-detail/south-sound/residential-properties-for-sale-in-cayman-islands/castillo-caribe-3" +LISTING_URL = "https://www.cireba.com/cayman-islands-real-estate-listings/" +SEARCH_QUERY = "CrewAI framework" + +# Fill these IDs before running result/rerun tests. +AI_SCRAPER_ID = "0bc62b79-e314-4d70-a6c8-7f0bd58ae221" +MANUAL_SCRAPER_ID = "" +RESULT_ID = "" + +MAX_OUTPUT_CHARS = 6_000 +Test = Callable[[], Any] + + +def tool(tool_class: type[Any]) -> Any: + """Construct a tool with the configured token.""" + require(MRSCRAPER_API_TOKEN, "MRSCRAPER_API_TOKEN") + return tool_class(api_token=MRSCRAPER_API_TOKEN) + + +def require(value: str, name: str) -> str: + """Require a configured value without printing secret contents.""" + if not value.strip(): + raise RuntimeError( + f"{name} belum diisi. Set environment variable atau isi konstanta " + "di bagian atas file ini." + ) + return value + + +def account() -> str: + return tool(MrScraperGetAccountInfoTool).run() + + +def crawl_urls() -> str: + return tool(MrScraperCrawlWebsiteUrlsTool).run( + url=TARGET_URL, max_depth=1, max_pages=2, limit=5 + ) + + +def google_serp() -> str: + return tool(MrScraperSearchGoogleSerpTool).run( + query=SEARCH_QUERY, region="us", language="en", page=1, format="json" + ) + + +def extract_prompt() -> str: + return tool(MrScraperExtractPageByPromptTool).run( + url=TARGET_URL, + prompt="Extract the page title and main description.", + output_schema={"title": "string", "description": "string"}, + mode="Cheap", + ) + + +def extract_listings() -> str: + return tool(MrScraperExtractListingsTool).run( + url=LISTING_URL, + prompt="Extract book title and price from the first page.", + output_schema={"title": "string", "price": "string"}, + max_pages=1, + ) + + +def extract_structured() -> str: + return tool(MrScraperExtractStructuredDataTool).run( + url=TARGET_URL, category="article", mode="Cheap" + ) + + +def rendered_html() -> str: + # Advanced options that remain False/None are intentionally not sent. + return tool(MrScraperFetchRenderedHtmlTool).run( + url=TARGET_URL, + html=True, + home_page=True, + markdown=False, + screenshot=False, + wait_until=None, + wait_for_selector=None, + ) + + +def get_results() -> str: + return tool(MrScraperGetResultsTool).run( + scraper_id=require(AI_SCRAPER_ID, "AI_SCRAPER_ID"), + page=1, + page_size=5, + sort_order="DESC", + ) + + +def get_latest_results() -> str: + return tool(MrScraperGetLatestResultsTool).run( + scraper_id=require(AI_SCRAPER_ID, "AI_SCRAPER_ID"), count=5 + ) + + +def get_result_detail() -> str: + return tool(MrScraperGetResultDetailTool).run( + result_id=require(RESULT_ID, "RESULT_ID") + ) + + +def create_prompt_scraper() -> str: + return tool(MrScraperCreatePromptScraperTool).run( + url=TARGET_URL, + prompt="Extract the property name and price, number of bedroom and bathroom, and mls ID.", + output_schema={"title": "string", "description": "string"}, + mode="Cheap", + ) + + +def create_listing_scraper() -> str: + return tool(MrScraperCreateListingScraperTool).run( + url=LISTING_URL, + prompt="Extract book title and price.", + output_schema={"title": "string", "price": "string"}, + max_pages=1, + ) + + +def create_crawl_scraper() -> str: + return tool(MrScraperCreateWebsiteCrawlScraperTool).run( + url=TARGET_URL, max_depth=1, max_pages=2, limit=5 + ) + + +def run_ai_scraper() -> str: + return tool(MrScraperRunExistingScraperTool).run( + scraper_type="ai", + scraper_id=require(AI_SCRAPER_ID, "AI_SCRAPER_ID"), + url=TARGET_URL, + agent_type="general", + ) + + +def run_manual_scraper() -> str: + return tool(MrScraperRunExistingScraperTool).run( + scraper_type="manual", + scraper_id=require(MANUAL_SCRAPER_ID, "MANUAL_SCRAPER_ID"), + url=TARGET_URL, + ) + + +def run_ai_batch() -> str: + return tool(MrScraperRunExistingScraperBatchTool).run( + scraper_type="ai", + scraper_id=require(AI_SCRAPER_ID, "AI_SCRAPER_ID"), + urls=[TARGET_URL, f"{TARGET_URL}/?second=1"], + ) + + +def crew_agent() -> Any: + """Test MrScraper through a real CrewAI Agent and OpenAI model.""" + require(OPENAI_API_KEY, "OPENAI_API_KEY") + os.environ["OPENAI_API_KEY"] = OPENAI_API_KEY + fetch_tool = tool(MrScraperFetchRenderedHtmlTool) + researcher = Agent( + role="Web content tester", + goal="Fetch one public page and report its title accurately", + backstory="You test web extraction tools with narrow, low-cost calls.", + tools=[fetch_tool], + verbose=True, + ) + task = Task( + description=( + f"Use the MrScraper rendered HTML tool to fetch {TARGET_URL}. " + "Return the page title and a one-sentence summary." + ), + expected_output="The source URL, page title, and one-sentence summary.", + agent=researcher, + ) + return Crew(agents=[researcher], tasks=[task], verbose=True).kickoff() + + +TESTS: dict[str, Test] = { + "account": account, + "crawl_urls": crawl_urls, + "google_serp": google_serp, + "extract_prompt": extract_prompt, + "extract_listings": extract_listings, + "extract_structured": extract_structured, + "rendered_html": rendered_html, + "get_results": get_results, + "get_latest_results": get_latest_results, + "get_result_detail": get_result_detail, + "create_prompt_scraper": create_prompt_scraper, + "create_listing_scraper": create_listing_scraper, + "create_crawl_scraper": create_crawl_scraper, + "run_ai_scraper": run_ai_scraper, + "run_manual_scraper": run_manual_scraper, + "run_ai_batch": run_ai_batch, + "crew_agent": crew_agent, +} + +CREATES_RECORDS = { + "create_prompt_scraper", + "create_listing_scraper", + "create_crawl_scraper", +} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--list", action="store_true", help="List available tests") + parser.add_argument("--test", choices=sorted(TESTS), help="Run one real test") + args = parser.parse_args() + + if args.list or args.test is None: + print("Available real tests:") + for name in TESTS: + warning = " [CREATES A SCRAPER]" if name in CREATES_RECORDS else "" + print(f" {name}{warning}") + if args.test is None: + print("\nRun one with: --test ") + return 0 + + print(f"Running real test: {args.test}") + try: + result = TESTS[args.test]() + except Exception as exc: # This CLI should show upstream integration failures. + print(f"FAILED: {type(exc).__name__}: {exc}") + return 1 + + output = str(result) + if len(output) > MAX_OUTPUT_CHARS: + output = f"{output[:MAX_OUTPUT_CHARS]}\n... [output truncated]" + print("SUCCESS") + print(output) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 2e7f3a005ceb004d2c892304a301b0ea5767ded4 Mon Sep 17 00:00:00 2001 From: riandradiva Date: Mon, 31 Aug 2026 11:48:15 +0700 Subject: [PATCH 2/3] feat: add mrscraper docs --- docs/docs.json | 10 +- .../ar/tools/web-scraping/mrscraper-tools.mdx | 128 ++++++++++++++++++ docs/edge/ar/tools/web-scraping/overview.mdx | 6 +- .../en/tools/web-scraping/mrscraper-tools.mdx | 128 ++++++++++++++++++ docs/edge/en/tools/web-scraping/overview.mdx | 5 + .../ko/tools/web-scraping/mrscraper-tools.mdx | 128 ++++++++++++++++++ docs/edge/ko/tools/web-scraping/overview.mdx | 6 +- .../tools/web-scraping/mrscraper-tools.mdx | 128 ++++++++++++++++++ .../pt-BR/tools/web-scraping/overview.mdx | 4 + 9 files changed, 538 insertions(+), 5 deletions(-) create mode 100644 docs/edge/ar/tools/web-scraping/mrscraper-tools.mdx create mode 100644 docs/edge/en/tools/web-scraping/mrscraper-tools.mdx create mode 100644 docs/edge/ko/tools/web-scraping/mrscraper-tools.mdx create mode 100644 docs/edge/pt-BR/tools/web-scraping/mrscraper-tools.mdx diff --git a/docs/docs.json b/docs/docs.json index 22a2c963fe..fde9bb9b9b 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -250,6 +250,7 @@ "edge/en/tools/web-scraping/firecrawlscrapewebsitetool", "edge/en/tools/web-scraping/oxylabsscraperstool", "edge/en/tools/web-scraping/brightdata-tools", + "edge/en/tools/web-scraping/mrscraper-tools", "edge/en/tools/web-scraping/youai-contents" ] }, @@ -13903,7 +13904,8 @@ "edge/pt-BR/tools/web-scraping/stagehandtool", "edge/pt-BR/tools/web-scraping/firecrawlcrawlwebsitetool", "edge/pt-BR/tools/web-scraping/firecrawlscrapewebsitetool", - "edge/pt-BR/tools/web-scraping/oxylabsscraperstool" + "edge/pt-BR/tools/web-scraping/oxylabsscraperstool", + "edge/pt-BR/tools/web-scraping/mrscraper-tools" ] }, { @@ -26621,7 +26623,8 @@ "edge/ko/tools/web-scraping/firecrawlcrawlwebsitetool", "edge/ko/tools/web-scraping/firecrawlscrapewebsitetool", "edge/ko/tools/web-scraping/oxylabsscraperstool", - "edge/ko/tools/web-scraping/brightdata-tools" + "edge/ko/tools/web-scraping/brightdata-tools", + "edge/ko/tools/web-scraping/mrscraper-tools" ] }, { @@ -39768,7 +39771,8 @@ "edge/ar/tools/web-scraping/firecrawlcrawlwebsitetool", "edge/ar/tools/web-scraping/firecrawlscrapewebsitetool", "edge/ar/tools/web-scraping/oxylabsscraperstool", - "edge/ar/tools/web-scraping/brightdata-tools" + "edge/ar/tools/web-scraping/brightdata-tools", + "edge/ar/tools/web-scraping/mrscraper-tools" ] }, { diff --git a/docs/edge/ar/tools/web-scraping/mrscraper-tools.mdx b/docs/edge/ar/tools/web-scraping/mrscraper-tools.mdx new file mode 100644 index 0000000000..7cbf918aee --- /dev/null +++ b/docs/edge/ar/tools/web-scraping/mrscraper-tools.mdx @@ -0,0 +1,128 @@ +--- +title: أدوات MrScraper +description: استخدم MrScraper مع CrewAI لاستخراج بيانات المواقع والزحف إليها بموثوقية، وجمع بيانات الويب العامة وتحويلها إلى مخرجات منظمة ونظيفة لمسارات عمل AI. +icon: spider +mode: "wide" +--- + +# أدوات MrScraper + +يوفر [MrScraper](https://mrscraper.com) استكشاف الويب والاستخراج باستخدام AI وجلب الصفحات المعروضة ومسارات عمل scraper القابلة لإعادة الاستخدام. يتضمن تكامل CrewAI خمس عشرة أداة مستقلة وfactory لاختيار الإمكانات التي يحتاجها Agent فقط. + +## التثبيت + +ثبّت `crewai-tools`. يستخدم التكامل تبعيات HTTP الموجودة، لذلك لا يلزم تثبيت SDK منفصل لـ MrScraper. + +```shell +uv add crewai-tools +``` + +عيّن رمز API الخاص بـ MrScraper كمتغير بيئة: + +```shell +export MRSCRAPER_API_TOKEN="your-mrscraper-token" +``` + +احتفظ بالرمز في متغير بيئة أو مدير أسرار. لا تضعه في prompts أو أوصاف Task أو ملفات المصدر أو وسيطات الأدوات التي يمكن للنموذج رؤيتها. + +## البدء السريع + +تعيد factory الخاصة بـ toolkit جميع الأدوات الخمس عشرة افتراضيًا. يُفضّل اختيار أصغر مجموعة مناسبة حتى يمتلك Agent مجموعة أدوات مركزة. + +```python +from crewai import Agent, Crew, Task +from crewai_tools import create_mrscraper_toolkit + +researcher = Agent( + role="Web researcher", + goal="Find public product pages and extract structured product data", + backstory="You make focused, cost-aware web extraction calls.", + tools=create_mrscraper_toolkit(groups=["Discovery", "Extraction"]), +) + +task = Task( + description="Find the relevant product page and extract its name and price.", + expected_output="A concise summary containing the source URL, product name, and price.", + agent=researcher, +) + +result = Crew(agents=[researcher], tasks=[task]).kickoff() +``` + +## الأدوات المتاحة + +| المجموعة | الأداة | الاستخدام | المدخلات الرئيسية | +| --- | --- | --- | --- | +| Account | `MrScraperGetAccountInfoTool` | فحص تفاصيل الحساب وحدود الاستخدام | لا يوجد | +| Discovery | `MrScraperCrawlWebsiteUrlsTool` | اكتشاف عناوين URL بدءًا من موقع | `url`, `max_depth`, `max_pages`, `limit` | +| Discovery | `MrScraperSearchGoogleSerpTool` | تنفيذ بحث Google SERP متزامن | `query`, `region`, `language`, `page` | +| Extraction | `MrScraperExtractPageByPromptTool` | استخراج البيانات من صفحة واحدة باستخدام تعليمات AI | `url`, `prompt`, `output_schema`, `mode` | +| Extraction | `MrScraperExtractListingsTool` | استخراج قوائم متكررة أو مقسمة إلى صفحات | `url`, `prompt`, `output_schema`, `max_pages` | +| Extraction | `MrScraperExtractStructuredDataTool` | استخراج preset مدعوم للبيانات المنظمة | `url`, `category`, `mode` | +| Extraction | `MrScraperFetchRenderedHtmlTool` | جلب HTML معروض باستخدام JavaScript أو Markdown أو cookies أو screenshots | `url`, `html`, `markdown`, `screenshot` | +| Results | `MrScraperGetResultsTool` | عرض نتائج scraper مقسمة إلى صفحات | `scraper_id`, `page`, `page_size` | +| Results | `MrScraperGetLatestResultsTool` | استرجاع أحدث نتائج scraper | `scraper_id`, `count` | +| Results | `MrScraperGetResultDetailTool` | استرجاع نتيجة واحدة باستخدام المعرّف | `result_id` | +| Scraper Creation | `MrScraperCreatePromptScraperTool` | إنشاء scraper قابل لإعادة الاستخدام يعتمد على prompt | `url`, `prompt`, `output_schema`, `mode` | +| Scraper Creation | `MrScraperCreateListingScraperTool` | إنشاء listing scraper قابل لإعادة الاستخدام | `url`, `prompt`, `output_schema`, `max_pages` | +| Scraper Creation | `MrScraperCreateWebsiteCrawlScraperTool` | إنشاء website crawl scraper قابل لإعادة الاستخدام | `url`, `max_depth`, `max_pages`, `limit` | +| Scraper Runs | `MrScraperRunExistingScraperTool` | تشغيل AI scraper أو manual scraper حالي لعنوان URL واحد | `scraper_type`, `scraper_id`, `url` | +| Scraper Runs | `MrScraperRunExistingScraperBatchTool` | تشغيل scraper حالي لعدة عناوين URL | `scraper_type`, `scraper_id`, `urls` | + +فئات البيانات المنظمة المدعومة هي `article` و`forumThread` و`hotel` و`jobPosting` و`post` و`product` و`property` و`restaurant` و`socialMediaProfile` و`tourAttraction`. + +## استخدام أداة واحدة مباشرة + +استخدم أداة واحدة عندما يحتاج Agent أو التطبيق إلى إمكانية واحدة فقط: + +```python +from crewai_tools import MrScraperExtractPageByPromptTool + +tool = MrScraperExtractPageByPromptTool() +result = tool.run( + url="https://example.com/products/123", + prompt="Extract the product name and current price", + output_schema={"name": "string", "price": "number"}, +) +``` + +يمكنك أيضًا تمرير `api_token="..."` إلى constructor الأداة أو factory الخاصة بـ toolkit. يُوصى باستخدام بيانات الاعتماد المعتمدة على البيئة لأنه يسهل إبقاؤها خارج كود التطبيق. + +## اختيار إمكانات Toolkit + +اختر الأدوات حسب اسم المجموعة دون حساسية لحالة الأحرف أو حسب الاسم العام الدقيق للأداة. لا تمرر `groups` و`tool_names` معًا. + +```python +from crewai_tools import create_mrscraper_toolkit + +# All tools share one configured HTTP client. +all_tools = create_mrscraper_toolkit() + +read_tools = create_mrscraper_toolkit(groups=["Account", "Results"]) + +selected_tools = create_mrscraper_toolkit( + tool_names=[ + "mrscraper_search_google_serp", + "mrscraper_fetch_rendered_html", + ] +) +``` + +المجموعات المتاحة هي `Account` و`Discovery` و`Extraction` و`Results` و`Scraper Creation` و`Scraper Runs`. + +## القيم المعادة + +تُعاد كائنات JSON والمصفوفات والقيم scalar كنص JSON مضغوط حتى تظل مستقرة عند تمريرها عبر Agents وTasks وFlows. تُعاد استجابات HTML والنصوص العادية الأخرى كما يوفرها MrScraper. + +## إرشادات التشغيل + +- اجعل حدود صفحات الزحف ودفعات URL صغيرة بقدر ما تسمح به Task. يمكن أن تستهلك عمليات الزحف واستدعاءات المتصفح المعروض واستخراج القوائم والتشغيل بالدفعات قدرًا كبيرًا من حصة API. +- لا تُعاد محاولة طلبات POST تلقائيًا لأن ذلك قد يؤدي إلى إنشاء jobs مكررة. تُرسل الحقول المتعلقة بإعادة المحاولة فقط للعمليات التي تدعمها. +- استخرج فقط المحتوى المصرح لك بالوصول إليه. راجع شروط الموقع المستهدف ومتطلبات الخصوصية وسياسة robots والقانون المعمول به، خصوصًا للبيانات الشخصية أو المحمية بتسجيل الدخول. + +## استكشاف الأخطاء وإصلاحها + +- **الرمز مفقود:** عيّن قيمة غير فارغة لـ `MRSCRAPER_API_TOKEN` قبل إنشاء الأداة. +- **وسيطات غير صالحة:** ترفض أدوات MrScraper المدخلات غير الموثقة. تحقق من schema الأداة واستخدم أسماء الحقول المقبولة بدقة. +- **استدعاءات بطيئة:** قلّل `max_pages` أو عمق الزحف أو حجم الدفعة أو مخرجات المتصفح المطلوبة عند الحاجة. +- **تنسيق إخراج غير متوقع:** يعيد بحث SERP صيغة JSON افتراضيًا؛ عيّن `format="html"` فقط عند الحاجة إلى HTML خام. diff --git a/docs/edge/ar/tools/web-scraping/overview.mdx b/docs/edge/ar/tools/web-scraping/overview.mdx index 3ba3b500e9..a649a99264 100644 --- a/docs/edge/ar/tools/web-scraping/overview.mdx +++ b/docs/edge/ar/tools/web-scraping/overview.mdx @@ -65,6 +65,10 @@ mode: "wide" تكاملات بحث SERP و Web Unlocker و Dataset API. + + + استكشاف الويب والاستخراج باستخدام AI والصفحات المعروضة ومسارات عمل scraper القابلة لإعادة الاستخدام. + ## **حالات الاستخدام الشائعة** @@ -109,4 +113,4 @@ agent = Agent( - **المواقع كثيفة JavaScript**: استخدم `SeleniumScrapingTool` للمحتوى الديناميكي - **التوسع والأداء**: استخدم `FirecrawlScrapeWebsiteTool` للاستخراج بكميات كبيرة - **البنية التحتية السحابية**: استخدم `BrowserBaseLoadTool` لأتمتة المتصفح القابلة للتوسع -- **سير العمل المعقدة**: استخدم `StagehandTool` لتفاعلات المتصفح الذكية \ No newline at end of file +- **سير العمل المعقدة**: استخدم `StagehandTool` لتفاعلات المتصفح الذكية diff --git a/docs/edge/en/tools/web-scraping/mrscraper-tools.mdx b/docs/edge/en/tools/web-scraping/mrscraper-tools.mdx new file mode 100644 index 0000000000..633bb4b698 --- /dev/null +++ b/docs/edge/en/tools/web-scraping/mrscraper-tools.mdx @@ -0,0 +1,128 @@ +--- +title: MrScraper Tools +description: Use MrScraper with CrewAI to reliably scrape and crawl websites, extract public web data, and turn it into clean, structured output for AI workflows. +icon: spider +mode: "wide" +--- + +# MrScraper Tools + +[MrScraper](https://mrscraper.com) provides web discovery, AI extraction, rendered-page fetching, and reusable scraper workflows. The CrewAI integration includes 15 independent tools and a factory for selecting only the capabilities an agent needs. + +## Installation + +Install `crewai-tools`. The integration uses its existing HTTP dependencies, so no separate MrScraper SDK is required. + +```shell +uv add crewai-tools +``` + +Set your MrScraper API token as an environment variable: + +```shell +export MRSCRAPER_API_TOKEN="your-mrscraper-token" +``` + +Keep the token in an environment variable or secret manager. Do not place it in prompts, task descriptions, source files, or tool arguments that the model can see. + +## Quick start + +The toolkit factory returns all 15 tools by default. Prefer selecting the smallest relevant group so the agent has a focused toolset. + +```python +from crewai import Agent, Crew, Task +from crewai_tools import create_mrscraper_toolkit + +researcher = Agent( + role="Web researcher", + goal="Find public product pages and extract structured product data", + backstory="You make focused, cost-aware web extraction calls.", + tools=create_mrscraper_toolkit(groups=["Discovery", "Extraction"]), +) + +task = Task( + description="Find the relevant product page and extract its name and price.", + expected_output="A concise summary containing the source URL, product name, and price.", + agent=researcher, +) + +result = Crew(agents=[researcher], tasks=[task]).kickoff() +``` + +## Available tools + +| Group | Tool | Use it for | Main inputs | +| --- | --- | --- | --- | +| Account | `MrScraperGetAccountInfoTool` | Inspect account details and usage limits | None | +| Discovery | `MrScraperCrawlWebsiteUrlsTool` | Discover URLs starting from a website | `url`, `max_depth`, `max_pages`, `limit` | +| Discovery | `MrScraperSearchGoogleSerpTool` | Run a synchronous Google SERP search | `query`, `region`, `language`, `page` | +| Extraction | `MrScraperExtractPageByPromptTool` | Extract data from one page with AI instructions | `url`, `prompt`, `output_schema`, `mode` | +| Extraction | `MrScraperExtractListingsTool` | Extract repeated or paginated listings | `url`, `prompt`, `output_schema`, `max_pages` | +| Extraction | `MrScraperExtractStructuredDataTool` | Extract a supported structured-data preset | `url`, `category`, `mode` | +| Extraction | `MrScraperFetchRenderedHtmlTool` | Fetch JavaScript-rendered HTML, Markdown, cookies, or screenshots | `url`, `html`, `markdown`, `screenshot` | +| Results | `MrScraperGetResultsTool` | List paginated results for a scraper | `scraper_id`, `page`, `page_size` | +| Results | `MrScraperGetLatestResultsTool` | Retrieve the newest results for a scraper | `scraper_id`, `count` | +| Results | `MrScraperGetResultDetailTool` | Retrieve one result by ID | `result_id` | +| Scraper Creation | `MrScraperCreatePromptScraperTool` | Create a reusable prompt-based scraper | `url`, `prompt`, `output_schema`, `mode` | +| Scraper Creation | `MrScraperCreateListingScraperTool` | Create a reusable listing scraper | `url`, `prompt`, `output_schema`, `max_pages` | +| Scraper Creation | `MrScraperCreateWebsiteCrawlScraperTool` | Create a reusable website crawl scraper | `url`, `max_depth`, `max_pages`, `limit` | +| Scraper Runs | `MrScraperRunExistingScraperTool` | Run an existing AI or manual scraper for one URL | `scraper_type`, `scraper_id`, `url` | +| Scraper Runs | `MrScraperRunExistingScraperBatchTool` | Run an existing scraper for multiple URLs | `scraper_type`, `scraper_id`, `urls` | + +The supported structured-data categories are `article`, `forumThread`, `hotel`, `jobPosting`, `post`, `product`, `property`, `restaurant`, `socialMediaProfile`, and `tourAttraction`. + +## Use one tool directly + +Use a single tool when the agent or application only needs one capability: + +```python +from crewai_tools import MrScraperExtractPageByPromptTool + +tool = MrScraperExtractPageByPromptTool() +result = tool.run( + url="https://example.com/products/123", + prompt="Extract the product name and current price", + output_schema={"name": "string", "price": "number"}, +) +``` + +You can also pass `api_token="..."` to a tool constructor or the toolkit factory. Environment-based credentials are recommended because they are easier to keep out of application code. + +## Select toolkit capabilities + +Select tools by case-insensitive group name or by exact public tool name. Do not pass `groups` and `tool_names` together. + +```python +from crewai_tools import create_mrscraper_toolkit + +# All tools share one configured HTTP client. +all_tools = create_mrscraper_toolkit() + +read_tools = create_mrscraper_toolkit(groups=["Account", "Results"]) + +selected_tools = create_mrscraper_toolkit( + tool_names=[ + "mrscraper_search_google_serp", + "mrscraper_fetch_rendered_html", + ] +) +``` + +Available groups are `Account`, `Discovery`, `Extraction`, `Results`, `Scraper Creation`, and `Scraper Runs`. + +## Return values + +JSON objects, arrays, and scalar values are returned as compact JSON text so they remain stable when passed through Agents, Tasks, and Flows. HTML and other plain-text responses are returned as provided by MrScraper. + +## Operational guidance + +- Keep crawl page limits and URL batches as small as the task permits. Crawls, rendered browser calls, listing extraction, and batch runs can consume significant API allowance. +- POST requests are not automatically retried because retrying can create duplicate jobs. Retry-related fields are sent only for operations that support them. +- Only scrape content you are authorized to access. Review the target site's terms, privacy requirements, robots policy, and applicable law, especially for login-protected or personal data. + +## Troubleshooting + +- **Missing token:** Set a nonblank `MRSCRAPER_API_TOKEN` before constructing a tool. +- **Invalid arguments:** MrScraper tools reject undocumented inputs. Check the tool schema and use the exact accepted field names. +- **Slow calls:** Reduce `max_pages`, crawl depth, batch size, or requested browser outputs where appropriate. +- **Unexpected output format:** SERP searches return JSON by default; set `format="html"` only when raw HTML is required. diff --git a/docs/edge/en/tools/web-scraping/overview.mdx b/docs/edge/en/tools/web-scraping/overview.mdx index 0031cf33e9..bbf52e0611 100644 --- a/docs/edge/en/tools/web-scraping/overview.mdx +++ b/docs/edge/en/tools/web-scraping/overview.mdx @@ -65,6 +65,10 @@ These tools enable your agents to interact with the web, extract data from websi SERP search, Web Unlocker, and Dataset API integrations. + + + Web discovery, AI extraction, rendered pages, and reusable scraper workflows. + ## **Common Use Cases** @@ -108,5 +112,6 @@ agent = Agent( - **Simple Tasks**: Use `ScrapeWebsiteTool` for basic content extraction - **JavaScript-Heavy Sites**: Use `SeleniumScrapingTool` for dynamic content - **Scale & Performance**: Use `FirecrawlScrapeWebsiteTool` for high-volume scraping +- **Discovery & Structured Extraction**: Use MrScraper tools for URL discovery, SERP search, rendered pages, and reusable extraction workflows - **Cloud Infrastructure**: Use `BrowserBaseLoadTool` for scalable browser automation - **Complex Workflows**: Use `StagehandTool` for intelligent browser interactions diff --git a/docs/edge/ko/tools/web-scraping/mrscraper-tools.mdx b/docs/edge/ko/tools/web-scraping/mrscraper-tools.mdx new file mode 100644 index 0000000000..f54110cb17 --- /dev/null +++ b/docs/edge/ko/tools/web-scraping/mrscraper-tools.mdx @@ -0,0 +1,128 @@ +--- +title: MrScraper 도구 +description: CrewAI에서 MrScraper를 사용하여 웹사이트를 안정적으로 scraping 및 crawling하고, 공개 웹 데이터를 추출하여 AI 워크플로를 위한 깔끔한 구조화 결과로 변환합니다. +icon: spider +mode: "wide" +--- + +# MrScraper 도구 + +[MrScraper](https://mrscraper.com)는 웹 탐색, AI 추출, 렌더링된 페이지 가져오기 및 재사용 가능한 스크래퍼 워크플로를 제공합니다. CrewAI 통합에는 15개의 독립적인 도구와 Agent에 필요한 기능만 선택할 수 있는 factory가 포함되어 있습니다. + +## 설치 + +`crewai-tools`를 설치합니다. 이 통합은 기존 HTTP 종속성을 사용하므로 별도의 MrScraper SDK가 필요하지 않습니다. + +```shell +uv add crewai-tools +``` + +MrScraper API 토큰을 환경 변수로 설정합니다. + +```shell +export MRSCRAPER_API_TOKEN="your-mrscraper-token" +``` + +토큰은 환경 변수나 비밀 관리자에 보관하세요. 모델이 볼 수 있는 prompt, Task 설명, 소스 파일 또는 도구 인수에 토큰을 넣지 마세요. + +## 빠른 시작 + +toolkit factory는 기본적으로 15개 도구를 모두 반환합니다. Agent가 집중된 도구 세트를 갖도록 가장 작은 관련 그룹을 선택하는 것이 좋습니다. + +```python +from crewai import Agent, Crew, Task +from crewai_tools import create_mrscraper_toolkit + +researcher = Agent( + role="Web researcher", + goal="Find public product pages and extract structured product data", + backstory="You make focused, cost-aware web extraction calls.", + tools=create_mrscraper_toolkit(groups=["Discovery", "Extraction"]), +) + +task = Task( + description="Find the relevant product page and extract its name and price.", + expected_output="A concise summary containing the source URL, product name, and price.", + agent=researcher, +) + +result = Crew(agents=[researcher], tasks=[task]).kickoff() +``` + +## 사용 가능한 도구 + +| 그룹 | 도구 | 용도 | 주요 입력 | +| --- | --- | --- | --- | +| Account | `MrScraperGetAccountInfoTool` | 계정 세부 정보 및 사용 한도 확인 | 없음 | +| Discovery | `MrScraperCrawlWebsiteUrlsTool` | 웹사이트에서 시작하여 URL 탐색 | `url`, `max_depth`, `max_pages`, `limit` | +| Discovery | `MrScraperSearchGoogleSerpTool` | 동기식 Google SERP 검색 실행 | `query`, `region`, `language`, `page` | +| Extraction | `MrScraperExtractPageByPromptTool` | AI 지침을 사용하여 단일 페이지에서 데이터 추출 | `url`, `prompt`, `output_schema`, `mode` | +| Extraction | `MrScraperExtractListingsTool` | 반복되거나 페이지가 나뉜 목록 추출 | `url`, `prompt`, `output_schema`, `max_pages` | +| Extraction | `MrScraperExtractStructuredDataTool` | 지원되는 구조화 데이터 preset 추출 | `url`, `category`, `mode` | +| Extraction | `MrScraperFetchRenderedHtmlTool` | JavaScript로 렌더링된 HTML, Markdown, cookie 또는 screenshot 가져오기 | `url`, `html`, `markdown`, `screenshot` | +| Results | `MrScraperGetResultsTool` | scraper의 페이지별 결과 목록 조회 | `scraper_id`, `page`, `page_size` | +| Results | `MrScraperGetLatestResultsTool` | scraper의 최신 결과 조회 | `scraper_id`, `count` | +| Results | `MrScraperGetResultDetailTool` | ID로 단일 결과 조회 | `result_id` | +| Scraper Creation | `MrScraperCreatePromptScraperTool` | 재사용 가능한 prompt 기반 scraper 생성 | `url`, `prompt`, `output_schema`, `mode` | +| Scraper Creation | `MrScraperCreateListingScraperTool` | 재사용 가능한 listing scraper 생성 | `url`, `prompt`, `output_schema`, `max_pages` | +| Scraper Creation | `MrScraperCreateWebsiteCrawlScraperTool` | 재사용 가능한 웹사이트 crawl scraper 생성 | `url`, `max_depth`, `max_pages`, `limit` | +| Scraper Runs | `MrScraperRunExistingScraperTool` | 하나의 URL에 기존 AI 또는 manual scraper 실행 | `scraper_type`, `scraper_id`, `url` | +| Scraper Runs | `MrScraperRunExistingScraperBatchTool` | 여러 URL에 기존 scraper 실행 | `scraper_type`, `scraper_id`, `urls` | + +지원되는 구조화 데이터 category는 `article`, `forumThread`, `hotel`, `jobPosting`, `post`, `product`, `property`, `restaurant`, `socialMediaProfile`, `tourAttraction`입니다. + +## 단일 도구 직접 사용 + +Agent나 애플리케이션에 하나의 기능만 필요할 때는 단일 도구를 사용하세요. + +```python +from crewai_tools import MrScraperExtractPageByPromptTool + +tool = MrScraperExtractPageByPromptTool() +result = tool.run( + url="https://example.com/products/123", + prompt="Extract the product name and current price", + output_schema={"name": "string", "price": "number"}, +) +``` + +도구 생성자나 toolkit factory에 `api_token="..."`을 전달할 수도 있습니다. 환경 기반 자격 증명은 애플리케이션 코드 외부에 보관하기 더 쉬우므로 권장됩니다. + +## Toolkit 기능 선택 + +대소문자를 구분하지 않는 그룹 이름이나 정확한 공개 도구 이름으로 도구를 선택합니다. `groups`와 `tool_names`를 함께 전달하지 마세요. + +```python +from crewai_tools import create_mrscraper_toolkit + +# All tools share one configured HTTP client. +all_tools = create_mrscraper_toolkit() + +read_tools = create_mrscraper_toolkit(groups=["Account", "Results"]) + +selected_tools = create_mrscraper_toolkit( + tool_names=[ + "mrscraper_search_google_serp", + "mrscraper_fetch_rendered_html", + ] +) +``` + +사용 가능한 그룹은 `Account`, `Discovery`, `Extraction`, `Results`, `Scraper Creation`, `Scraper Runs`입니다. + +## 반환값 + +JSON 객체, 배열 및 scalar 값은 Agent, Task 및 Flow를 통과할 때 안정적으로 유지되도록 압축된 JSON 텍스트로 반환됩니다. HTML 및 기타 일반 텍스트 응답은 MrScraper가 제공한 그대로 반환됩니다. + +## 운영 지침 + +- crawl 페이지 제한과 URL batch는 Task에 필요한 만큼만 작게 유지하세요. crawl, 렌더링된 browser 호출, listing 추출 및 batch 실행은 상당한 API 할당량을 소비할 수 있습니다. +- POST 요청은 중복 job을 생성할 수 있으므로 자동으로 재시도되지 않습니다. 재시도 관련 필드는 이를 지원하는 작업에만 전송됩니다. +- 접근 권한이 있는 콘텐츠만 scraping하세요. 특히 login으로 보호되거나 개인 데이터가 포함된 경우 대상 사이트의 약관, 개인정보 보호 요구 사항, robots 정책 및 관련 법률을 검토하세요. + +## 문제 해결 + +- **토큰 누락:** 도구를 생성하기 전에 비어 있지 않은 `MRSCRAPER_API_TOKEN`을 설정하세요. +- **잘못된 인수:** MrScraper 도구는 문서화되지 않은 입력을 거부합니다. 도구 schema를 확인하고 허용된 정확한 필드 이름을 사용하세요. +- **느린 호출:** 필요에 따라 `max_pages`, crawl 깊이, batch 크기 또는 요청된 browser 출력을 줄이세요. +- **예상치 못한 출력 형식:** SERP 검색은 기본적으로 JSON을 반환합니다. 원시 HTML이 필요한 경우에만 `format="html"`을 설정하세요. diff --git a/docs/edge/ko/tools/web-scraping/overview.mdx b/docs/edge/ko/tools/web-scraping/overview.mdx index 070f310bcf..3b93e38403 100644 --- a/docs/edge/ko/tools/web-scraping/overview.mdx +++ b/docs/edge/ko/tools/web-scraping/overview.mdx @@ -65,6 +65,10 @@ mode: "wide" SERP 검색, 웹 언락커, 데이터셋 API 통합 기능을 지원합니다. + + + 웹 탐색, AI 추출, 렌더링된 페이지 및 재사용 가능한 scraper 워크플로를 제공합니다. + ## **일반적인 사용 사례** @@ -109,4 +113,4 @@ agent = Agent( - **JavaScript 기반 사이트**: 동적 콘텐츠에는 `SeleniumScrapingTool`을 사용하세요 - **확장성 및 성능**: 대량 스크래핑에는 `FirecrawlScrapeWebsiteTool`을 사용하세요 - **클라우드 인프라**: 확장 가능한 브라우저 자동화에는 `BrowserBaseLoadTool`을 사용하세요 -- **복잡한 워크플로우**: 지능형 브라우저 상호작용에는 `StagehandTool`을 사용하세요 \ No newline at end of file +- **복잡한 워크플로우**: 지능형 브라우저 상호작용에는 `StagehandTool`을 사용하세요 diff --git a/docs/edge/pt-BR/tools/web-scraping/mrscraper-tools.mdx b/docs/edge/pt-BR/tools/web-scraping/mrscraper-tools.mdx new file mode 100644 index 0000000000..bfd35c7e1e --- /dev/null +++ b/docs/edge/pt-BR/tools/web-scraping/mrscraper-tools.mdx @@ -0,0 +1,128 @@ +--- +title: Ferramentas MrScraper +description: Use o MrScraper com CrewAI para extrair e rastrear sites com confiabilidade, coletar dados públicos da web e transformá-los em resultados limpos e estruturados para fluxos de AI. +icon: spider +mode: "wide" +--- + +# Ferramentas MrScraper + +O [MrScraper](https://mrscraper.com) oferece descoberta na web, extração com AI, obtenção de páginas renderizadas e fluxos de scrapers reutilizáveis. A integração com CrewAI inclui 15 ferramentas independentes e uma factory para selecionar apenas os recursos de que um Agent precisa. + +## Instalação + +Instale `crewai-tools`. A integração usa as dependências HTTP já existentes, portanto não é necessário instalar um SDK separado do MrScraper. + +```shell +uv add crewai-tools +``` + +Defina seu token de API do MrScraper como variável de ambiente: + +```shell +export MRSCRAPER_API_TOKEN="your-mrscraper-token" +``` + +Mantenha o token em uma variável de ambiente ou gerenciador de segredos. Não o coloque em prompts, descrições de Task, arquivos de código-fonte ou argumentos de ferramenta visíveis ao modelo. + +## Início rápido + +Por padrão, a factory do toolkit retorna todas as 15 ferramentas. Prefira selecionar o menor grupo relevante para que o Agent tenha um conjunto de ferramentas focado. + +```python +from crewai import Agent, Crew, Task +from crewai_tools import create_mrscraper_toolkit + +researcher = Agent( + role="Web researcher", + goal="Find public product pages and extract structured product data", + backstory="You make focused, cost-aware web extraction calls.", + tools=create_mrscraper_toolkit(groups=["Discovery", "Extraction"]), +) + +task = Task( + description="Find the relevant product page and extract its name and price.", + expected_output="A concise summary containing the source URL, product name, and price.", + agent=researcher, +) + +result = Crew(agents=[researcher], tasks=[task]).kickoff() +``` + +## Ferramentas disponíveis + +| Grupo | Ferramenta | Use para | Principais entradas | +| --- | --- | --- | --- | +| Account | `MrScraperGetAccountInfoTool` | Consultar detalhes da conta e limites de uso | Nenhuma | +| Discovery | `MrScraperCrawlWebsiteUrlsTool` | Descobrir URLs a partir de um site | `url`, `max_depth`, `max_pages`, `limit` | +| Discovery | `MrScraperSearchGoogleSerpTool` | Executar uma busca síncrona no Google SERP | `query`, `region`, `language`, `page` | +| Extraction | `MrScraperExtractPageByPromptTool` | Extrair dados de uma página com instruções de AI | `url`, `prompt`, `output_schema`, `mode` | +| Extraction | `MrScraperExtractListingsTool` | Extrair listagens repetidas ou paginadas | `url`, `prompt`, `output_schema`, `max_pages` | +| Extraction | `MrScraperExtractStructuredDataTool` | Extrair uma predefinição de dados estruturados | `url`, `category`, `mode` | +| Extraction | `MrScraperFetchRenderedHtmlTool` | Obter HTML renderizado por JavaScript, Markdown, cookies ou capturas de tela | `url`, `html`, `markdown`, `screenshot` | +| Results | `MrScraperGetResultsTool` | Listar resultados paginados de um scraper | `scraper_id`, `page`, `page_size` | +| Results | `MrScraperGetLatestResultsTool` | Obter os resultados mais recentes de um scraper | `scraper_id`, `count` | +| Results | `MrScraperGetResultDetailTool` | Obter um resultado pelo ID | `result_id` | +| Scraper Creation | `MrScraperCreatePromptScraperTool` | Criar um scraper reutilizável baseado em prompt | `url`, `prompt`, `output_schema`, `mode` | +| Scraper Creation | `MrScraperCreateListingScraperTool` | Criar um scraper reutilizável de listagens | `url`, `prompt`, `output_schema`, `max_pages` | +| Scraper Creation | `MrScraperCreateWebsiteCrawlScraperTool` | Criar um scraper reutilizável de rastreamento de site | `url`, `max_depth`, `max_pages`, `limit` | +| Scraper Runs | `MrScraperRunExistingScraperTool` | Executar um scraper AI ou manual existente para uma URL | `scraper_type`, `scraper_id`, `url` | +| Scraper Runs | `MrScraperRunExistingScraperBatchTool` | Executar um scraper existente para várias URLs | `scraper_type`, `scraper_id`, `urls` | + +As categorias de dados estruturados aceitas são `article`, `forumThread`, `hotel`, `jobPosting`, `post`, `product`, `property`, `restaurant`, `socialMediaProfile` e `tourAttraction`. + +## Uso direto de uma ferramenta + +Use uma única ferramenta quando o Agent ou a aplicação precisar apenas de um recurso: + +```python +from crewai_tools import MrScraperExtractPageByPromptTool + +tool = MrScraperExtractPageByPromptTool() +result = tool.run( + url="https://example.com/products/123", + prompt="Extract the product name and current price", + output_schema={"name": "string", "price": "number"}, +) +``` + +Também é possível passar `api_token="..."` para o construtor de uma ferramenta ou para a factory do toolkit. Credenciais baseadas no ambiente são recomendadas porque são mais fáceis de manter fora do código da aplicação. + +## Seleção de recursos do toolkit + +Selecione ferramentas pelo nome de grupo, sem distinção entre maiúsculas e minúsculas, ou pelo nome público exato da ferramenta. Não passe `groups` e `tool_names` juntos. + +```python +from crewai_tools import create_mrscraper_toolkit + +# All tools share one configured HTTP client. +all_tools = create_mrscraper_toolkit() + +read_tools = create_mrscraper_toolkit(groups=["Account", "Results"]) + +selected_tools = create_mrscraper_toolkit( + tool_names=[ + "mrscraper_search_google_serp", + "mrscraper_fetch_rendered_html", + ] +) +``` + +Os grupos disponíveis são `Account`, `Discovery`, `Extraction`, `Results`, `Scraper Creation` e `Scraper Runs`. + +## Valores retornados + +Objetos, arrays e valores escalares JSON são retornados como texto JSON compacto para permanecerem estáveis ao passar por Agents, Tasks e Flows. HTML e outras respostas de texto simples são retornados como fornecidos pelo MrScraper. + +## Orientações operacionais + +- Mantenha os limites de páginas e os lotes de URLs tão pequenos quanto a Task permitir. Rastreamentos, chamadas de navegador renderizado, extração de listagens e execuções em lote podem consumir uma parcela significativa da cota da API. +- Requisições POST não são repetidas automaticamente, pois a repetição pode criar jobs duplicados. Campos relacionados a novas tentativas são enviados apenas para operações compatíveis. +- Extraia apenas conteúdo que você está autorizado a acessar. Analise os termos, requisitos de privacidade, política de robots e legislação aplicável do site de destino, especialmente para dados pessoais ou protegidos por login. + +## Solução de problemas + +- **Token ausente:** defina um `MRSCRAPER_API_TOKEN` não vazio antes de criar uma ferramenta. +- **Argumentos inválidos:** as ferramentas MrScraper rejeitam entradas não documentadas. Verifique o schema da ferramenta e use os nomes exatos dos campos aceitos. +- **Chamadas lentas:** reduza `max_pages`, a profundidade do rastreamento, o tamanho do lote ou as saídas solicitadas do navegador, quando apropriado. +- **Formato de saída inesperado:** buscas SERP retornam JSON por padrão; use `format="html"` somente quando HTML bruto for necessário. diff --git a/docs/edge/pt-BR/tools/web-scraping/overview.mdx b/docs/edge/pt-BR/tools/web-scraping/overview.mdx index 0a493acaf1..8e6ef317db 100644 --- a/docs/edge/pt-BR/tools/web-scraping/overview.mdx +++ b/docs/edge/pt-BR/tools/web-scraping/overview.mdx @@ -61,6 +61,10 @@ Essas ferramentas permitem que seus agentes interajam com a web, extraiam dados Acesse dados web em escala com o Oxylabs. + + + Descoberta na web, extração com AI, páginas renderizadas e fluxos de scrapers reutilizáveis. + ## **Casos de Uso Comuns** From ebb2ae7d659e7d9fcdadc348c05c8a6e0dbb038e Mon Sep 17 00:00:00 2001 From: riandradiva Date: Thu, 3 Sep 2026 09:15:39 +0700 Subject: [PATCH 3/3] fix(tools): address MrScraper review feedback --- .../crewai_tools/tools/mrscraper/account.py | 1 + .../src/crewai_tools/tools/mrscraper/base.py | 1 + .../crewai_tools/tools/mrscraper/client.py | 3 + .../crewai_tools/tools/mrscraper/discovery.py | 2 + .../tools/mrscraper/extraction.py | 4 + .../crewai_tools/tools/mrscraper/payloads.py | 7 + .../crewai_tools/tools/mrscraper/results.py | 3 + .../tools/mrscraper/scraper_creation.py | 3 + .../tools/mrscraper/scraper_runs.py | 2 + .../tools/mrscraper/test_mrscraper_tools.py | 36 +- lib/crewai-tools/tool.specs.json | 329 ++++++++++++------ scripts/test_mrscraper_real.py | 59 +++- 12 files changed, 332 insertions(+), 118 deletions(-) diff --git a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/account.py b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/account.py index bec553d9ca..ace87354ad 100644 --- a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/account.py +++ b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/account.py @@ -17,6 +17,7 @@ class MrScraperGetAccountInfoTool(MrScraperBaseTool): args_schema: type[BaseModel] = GetAccountInfoInput def _run(self) -> str: + """Return subscription and token-usage details for the account.""" return self._client.request("GET", "primary", "/api/v1/subscription-accounts") diff --git a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/base.py b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/base.py index 8e6146c7f2..e1725276ed 100644 --- a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/base.py +++ b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/base.py @@ -41,6 +41,7 @@ def __init__( client: MrScraperClient | None = None, **kwargs: Any, ) -> None: + """Initialize the tool with an injected client or resolved API token.""" super().__init__(**kwargs) self._client = client or MrScraperClient(resolve_api_token(api_token)) diff --git a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/client.py b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/client.py index 6018d53683..ade4be1942 100644 --- a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/client.py +++ b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/client.py @@ -23,6 +23,7 @@ class MrScraperClient: """Make requests only to MrScraper's fixed API origins.""" def __init__(self, token: str, *, session: requests.Session | None = None) -> None: + """Initialize a client with a nonblank token and optional HTTP session.""" if not token.strip(): raise ValueError("MRSCRAPER_API_TOKEN must be a nonblank value") self._token = token @@ -82,6 +83,7 @@ def request( return self._sanitize(serialized) def _headers(self, origin: Origin) -> dict[str, str]: + """Build the authentication headers required by an API origin.""" headers = { "Accept": "application/json", "Content-Type": "application/json", @@ -93,6 +95,7 @@ def _headers(self, origin: Origin) -> dict[str, str]: return headers def _sanitize(self, value: str) -> str: + """Redact the configured token from response and error text.""" redacted = value.replace(self._token, "[REDACTED]") return _TOKEN_QUERY_RE.sub(r"\1[REDACTED]", redacted) diff --git a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/discovery.py b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/discovery.py index 3f29067631..c1dbf9d586 100644 --- a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/discovery.py +++ b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/discovery.py @@ -28,6 +28,7 @@ def _run( include_patterns: str | None = None, exclude_patterns: str | None = None, ) -> str: + """Discover URLs from a starting page with bounded crawl controls.""" return self._client.request( "POST", "primary", @@ -62,6 +63,7 @@ def _run( format: Literal["json", "html"] = "json", render_js: bool = False, ) -> str: + """Run a synchronous Google search and return JSON or HTML text.""" return self._client.request( "POST", "serp", diff --git a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/extraction.py b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/extraction.py index 229756f871..f40cb5a77a 100644 --- a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/extraction.py +++ b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/extraction.py @@ -49,6 +49,7 @@ def _run( mode: ScrapingMode = "Super", proxy_country: str | None = None, ) -> str: + """Extract prompted data from one page immediately.""" return self._client.request( "POST", "primary", @@ -81,6 +82,7 @@ def _run( max_pages: int = 1, proxy_country: str | None = None, ) -> str: + """Extract repeated listings across one or more pages immediately.""" return self._client.request( "POST", "primary", @@ -112,6 +114,7 @@ def _run( mode: ScrapingMode = "Super", proxy_country: str | None = None, ) -> str: + """Extract data using the selected bundled structured-data preset.""" payload: dict[str, Any] = { "graph": "general", "url": url, @@ -154,6 +157,7 @@ def _run( return_cookie: bool = False, super_mode: bool = False, ) -> str: + """Fetch a rendered page with optional browser outputs and controls.""" params, body = rendered_request( url=url, max_retries=max_retries, diff --git a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/payloads.py b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/payloads.py index 0c12cf05fc..c0f208b15c 100644 --- a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/payloads.py +++ b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/payloads.py @@ -7,6 +7,7 @@ def _include_if_present( payload: dict[str, Any], key: str, value: Any ) -> dict[str, Any]: + """Add a payload value unless it is absent.""" if value is not None: payload[key] = value return payload @@ -35,6 +36,7 @@ def general_payload( mode: str, proxy_country: str | None, ) -> dict[str, Any]: + """Build a General AI extraction payload.""" payload: dict[str, Any] = {"graph": "general", "url": url, "mode": mode} message = append_output_schema( prompt, output_schema, "Return the output as JSON matching this schema:" @@ -52,6 +54,7 @@ def listing_payload( max_pages: int, proxy_country: str | None, ) -> dict[str, Any]: + """Build a Listing AI extraction payload.""" payload: dict[str, Any] = { "graph": "listing", "url": url, @@ -74,6 +77,7 @@ def map_payload( include_patterns: str | None, exclude_patterns: str | None, ) -> dict[str, Any]: + """Build a Map AI crawl payload.""" payload: dict[str, Any] = { "graph": "map", "url": url, @@ -105,7 +109,10 @@ def rendered_request( return_cookie: bool, super_mode: bool, ) -> tuple[dict[str, Any], dict[str, Any]]: + """Split rendered-page options into query parameters and a JSON body.""" + def bool_text(value: bool) -> str: + """Serialize a boolean for rendered-page query parameters.""" return "true" if value else "false" params: dict[str, Any] = { diff --git a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/results.py b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/results.py index 1a74a44023..c92d05bec6 100644 --- a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/results.py +++ b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/results.py @@ -31,6 +31,7 @@ def _run( sort_by: Literal["createdAt"] = "createdAt", sort_order: Literal["ASC", "DESC"] = "DESC", ) -> str: + """Return a configured page of results for one scraper.""" return self._client.request( "GET", "primary", @@ -56,6 +57,7 @@ class MrScraperGetLatestResultsTool(MrScraperBaseTool): args_schema: type[BaseModel] = GetLatestResultsInput def _run(self, scraper_id: str, count: int = 10) -> str: + """Return the newest results for one scraper.""" return self._client.request( "GET", "primary", @@ -81,6 +83,7 @@ class MrScraperGetResultDetailTool(MrScraperBaseTool): args_schema: type[BaseModel] = GetResultDetailInput def _run(self, result_id: str) -> str: + """Return one result after safely encoding its identifier.""" encoded_id = quote(result_id, safe="") return self._client.request("GET", "primary", f"/api/v1/results/{encoded_id}") diff --git a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/scraper_creation.py b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/scraper_creation.py index 435ee39faf..b62ec29394 100644 --- a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/scraper_creation.py +++ b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/scraper_creation.py @@ -36,6 +36,7 @@ def _run( mode: ScrapingMode = "Super", proxy_country: str | None = None, ) -> str: + """Create a reusable General AI scraper.""" return self._client.request( "POST", "primary", @@ -68,6 +69,7 @@ def _run( max_pages: int = 1, proxy_country: str | None = None, ) -> str: + """Create a reusable Listing AI scraper.""" return self._client.request( "POST", "primary", @@ -101,6 +103,7 @@ def _run( include_patterns: str | None = None, exclude_patterns: str | None = None, ) -> str: + """Create a reusable Map AI scraper.""" return self._client.request( "POST", "primary", diff --git a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/scraper_runs.py b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/scraper_runs.py index 976f65874f..7269f17cf7 100644 --- a/lib/crewai-tools/src/crewai_tools/tools/mrscraper/scraper_runs.py +++ b/lib/crewai-tools/src/crewai_tools/tools/mrscraper/scraper_runs.py @@ -23,6 +23,7 @@ class MrScraperRunExistingScraperTool(MrScraperBaseTool): args_schema: type[BaseModel] = RunExistingScraperInput def _run(self, **values: Any) -> str: + """Run one validated URL through an existing AI or manual scraper.""" scraper_type = values["scraper_type"] endpoint = ( "/api/v1/scrapers-manual-rerun" @@ -53,6 +54,7 @@ def _run( scraper_id: str, urls: list[str], ) -> str: + """Run a URL batch through an existing AI or manual scraper.""" base = ( "/api/v1/scrapers-manual-rerun" if scraper_type == "manual" diff --git a/lib/crewai-tools/tests/tools/mrscraper/test_mrscraper_tools.py b/lib/crewai-tools/tests/tools/mrscraper/test_mrscraper_tools.py index cd9e738107..3ade6eceb5 100644 --- a/lib/crewai-tools/tests/tools/mrscraper/test_mrscraper_tools.py +++ b/lib/crewai-tools/tests/tools/mrscraper/test_mrscraper_tools.py @@ -43,7 +43,7 @@ RunExistingScraperInput, SearchGoogleSerpInput, ) -from pydantic import ValidationError +from pydantic import BaseModel, ValidationError import pytest import requests @@ -96,12 +96,14 @@ def __init__( status_code: int = 200, content_type: str = "application/json", ) -> None: + """Initialize a deterministic response double.""" self.value = value self.text = text if text is not None else json.dumps(value) self.status_code = status_code self.headers = {"Content-Type": content_type} def json(self) -> Any: + """Return the configured JSON value.""" return self.value @@ -111,11 +113,13 @@ def __init__( response: FakeResponse | None = None, error: requests.RequestException | None = None, ) -> None: + """Initialize a session double with a response or transport error.""" self.response = response or FakeResponse({"ok": True}) self.error = error self.calls: list[dict[str, Any]] = [] def request(self, method: str, url: str, **kwargs: Any) -> FakeResponse: + """Record a request before returning or raising the configured result.""" self.calls.append({"method": method, "url": url, **kwargs}) if self.error is not None: raise self.error @@ -126,11 +130,13 @@ def make_client( response: FakeResponse | None = None, error: requests.RequestException | None = None, ) -> tuple[MrScraperClient, FakeSession]: + """Build a client and expose its deterministic session double.""" session = FakeSession(response=response, error=error) return MrScraperClient(FAKE_TOKEN, session=session), session # type: ignore[arg-type] def test_all_tools_are_public_independent_base_tools() -> None: + """Expose every integration operation as a distinct BaseTool.""" client, _ = make_client() tools = [tool_class(client=client) for tool_class in TOOL_CLASSES] @@ -143,6 +149,7 @@ def test_all_tools_are_public_independent_base_tools() -> None: def test_toolkit_returns_all_groups_names_and_fresh_state(monkeypatch: pytest.MonkeyPatch) -> None: + """Select fresh tool instances by group or public tool name.""" monkeypatch.setenv("MRSCRAPER_API_TOKEN", FAKE_TOKEN) first = create_mrscraper_toolkit() second = create_mrscraper_toolkit() @@ -170,6 +177,7 @@ def test_toolkit_returns_all_groups_names_and_fresh_state(monkeypatch: pytest.Mo def test_agent_receives_15_independent_tools(monkeypatch: pytest.MonkeyPatch) -> None: + """Attach all independent MrScraper tools to an Agent.""" monkeypatch.setenv("MRSCRAPER_API_TOKEN", FAKE_TOKEN) tools = create_mrscraper_toolkit() agent = Agent( @@ -187,6 +195,7 @@ def test_agent_receives_15_independent_tools(monkeypatch: pytest.MonkeyPatch) -> def test_schema_required_defaults_enums_constraints_and_descriptions() -> None: + """Publish strict, documented schemas with the expected defaults.""" search_schema = SearchGoogleSerpInput.model_json_schema() assert search_schema["required"] == ["query"] assert search_schema["properties"]["page"]["default"] == 1 @@ -228,6 +237,7 @@ def test_schema_required_defaults_enums_constraints_and_descriptions() -> None: ], ) def test_strict_schema_validation(schema: Any, values: dict[str, Any]) -> None: + """Reject coercion, invalid bounds, and empty URL batches.""" with pytest.raises(ValidationError): schema.model_validate(values) @@ -396,8 +406,9 @@ def test_general_listing_map_payloads_and_schema_append_once() -> None: def test_structured_presets_are_exact_and_selected_without_category() -> None: - preset_path = Path( - "lib/crewai-tools/src/crewai_tools/tools/mrscraper/structured_data_prompts.json" + preset_path = ( + Path(__file__).resolve().parents[3] + / "src/crewai_tools/tools/mrscraper/structured_data_prompts.json" ) assert hashlib.sha256(preset_path.read_bytes()).hexdigest() == ( "3d9c15e8ebe7ad8cb04281251311200c1d3413452f14f252dc9ed3a8aae8533a" @@ -623,3 +634,22 @@ def test_generated_discovery_specs_include_all_tools_without_secrets() -> None: rendered = json.dumps(spec) assert FAKE_TOKEN not in rendered assert "api_token" not in spec["run_params_schema"].get("properties", {}) + + +@pytest.mark.parametrize( + ("tool_class", "input_schema"), + [ + (MrScraperFetchRenderedHtmlTool, FetchRenderedHtmlInput), + (MrScraperRunExistingScraperTool, RunExistingScraperInput), + ], +) +def test_generated_run_schemas_match_runtime_schemas( + tool_class: type[BaseTool], input_schema: type[BaseModel] +) -> None: + """Keep generated discovery defaults aligned with direct tool invocation.""" + specs = ToolSpecExtractor().extract_all_tools() + by_class = {spec["name"]: spec for spec in specs} + + assert by_class[tool_class.__name__]["run_params_schema"] == ( + input_schema.model_json_schema() + ) diff --git a/lib/crewai-tools/tool.specs.json b/lib/crewai-tools/tool.specs.json index 0435a8e8b5..2a5327679c 100644 --- a/lib/crewai-tools/tool.specs.json +++ b/lib/crewai-tools/tool.specs.json @@ -16217,8 +16217,8 @@ "description": "Inputs for the rendered-page API.", "properties": { "block_resources": { - "default": true, - "description": "Whether to block images, fonts, and stylesheets; defaults to true.", + "default": false, + "description": "Whether to block images, fonts, and stylesheets; defaults to false.", "title": "Block Resources", "type": "boolean" }, @@ -16262,8 +16262,8 @@ "type": "string" }, "return_cookie": { - "default": true, - "description": "Whether to include browser cookies; defaults to true.", + "default": false, + "description": "Whether to include browser cookies; defaults to false.", "title": "Return Cookie", "type": "boolean" }, @@ -16274,18 +16274,25 @@ "type": "boolean" }, "screenshot_mode": { - "default": "full", - "description": "Screenshot mode, 'full' or 'top'; used only when screenshot is true.", - "enum": [ - "full", - "top" + "anyOf": [ + { + "enum": [ + "full", + "top" + ], + "type": "string" + }, + { + "type": "null" + } ], - "title": "Screenshot Mode", - "type": "string" + "default": null, + "description": "Optional screenshot mode; used only when screenshot is true.", + "title": "Screenshot Mode" }, "super_mode": { - "default": true, - "description": "Whether to use stronger device mode; defaults to true.", + "default": false, + "description": "Whether to use stronger device mode; defaults to false.", "title": "Super Mode", "type": "boolean" }, @@ -16297,11 +16304,18 @@ "type": "integer" }, "token_cap": { - "default": 30, - "description": "Maximum processing token allowance; defaults to 30; minimum 1.", - "minimum": 1, - "title": "Token Cap", - "type": "integer" + "anyOf": [ + { + "minimum": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional maximum processing token allowance; minimum 1.", + "title": "Token Cap" }, "url": { "description": "Required target URL to render.", @@ -16324,15 +16338,22 @@ "title": "Wait For Selector" }, "wait_until": { - "default": "domcontentloaded", - "description": "Browser lifecycle event to await; defaults to 'domcontentloaded'.", - "enum": [ - "domcontentloaded", - "load", - "networkidle" + "anyOf": [ + { + "enum": [ + "domcontentloaded", + "load", + "networkidle" + ], + "type": "string" + }, + { + "type": "null" + } ], - "title": "Wait Until", - "type": "string" + "default": null, + "description": "Optional browser lifecycle event to await.", + "title": "Wait Until" } }, "required": [ @@ -16923,13 +16944,21 @@ "title": "Cookie Jar" }, "cookies": { - "description": "Manual browser-cookie objects; defaults to an empty array.", - "items": { - "additionalProperties": true, - "type": "object" - }, - "title": "Cookies", - "type": "array" + "anyOf": [ + { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional Manual browser-cookie objects.", + "title": "Cookies" }, "exclude_patterns": { "anyOf": [ @@ -16946,23 +16975,44 @@ "title": "Exclude Patterns" }, "home_page": { - "default": false, - "description": "Manual home-page visit flag; defaults to false.", - "title": "Home Page", - "type": "boolean" + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional Manual home-page visit flag.", + "title": "Home Page" }, "home_page_timeout": { - "default": 10, - "description": "Manual home-page timeout; defaults to 10; minimum 1.", - "minimum": 1, - "title": "Home Page Timeout", - "type": "integer" + "anyOf": [ + { + "minimum": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional Manual home-page timeout; minimum 1.", + "title": "Home Page Timeout" }, "html": { - "default": false, - "description": "General, Listing, or Manual HTML output flag; defaults to false.", - "title": "Html", - "type": "boolean" + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional General, Listing, or Manual HTML output flag.", + "title": "Html" }, "include_patterns": { "anyOf": [ @@ -16979,24 +17029,45 @@ "title": "Include Patterns" }, "limit": { - "default": 50, - "description": "Map result limit; defaults to 50; minimum 1.", - "minimum": 1, - "title": "Limit", - "type": "integer" + "anyOf": [ + { + "minimum": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional Map result limit; minimum 1.", + "title": "Limit" }, "markdown": { - "default": false, - "description": "General, Listing, or Manual Markdown output flag; defaults to false.", - "title": "Markdown", - "type": "boolean" + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional General, Listing, or Manual Markdown output flag.", + "title": "Markdown" }, "max_depth": { - "default": 2, - "description": "Map crawl depth; defaults to 2; minimum 0.", - "minimum": 0, - "title": "Max Depth", - "type": "integer" + "anyOf": [ + { + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional Map crawl depth; minimum 0.", + "title": "Max Depth" }, "max_pages": { "anyOf": [ @@ -17020,10 +17091,18 @@ "type": "integer" }, "paginator": { - "additionalProperties": true, - "description": "Manual paginator configuration; defaults to an empty object.", - "title": "Paginator", - "type": "object" + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional Manual paginator configuration.", + "title": "Paginator" }, "proxy": { "anyOf": [ @@ -17054,28 +17133,56 @@ "title": "Proxy Country" }, "record": { - "default": false, - "description": "Manual browser-session recording flag; defaults to false.", - "title": "Record", - "type": "boolean" + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional Manual browser-session recording flag.", + "title": "Record" }, "render_javascript": { - "default": false, - "description": "General/Listing JavaScript rendering flag; defaults to false.", - "title": "Render Javascript", - "type": "boolean" + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional General/Listing JavaScript rendering flag.", + "title": "Render Javascript" }, "return_cookie": { - "default": false, - "description": "Manual cookie-return flag; defaults to false.", - "title": "Return Cookie", - "type": "boolean" + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional Manual cookie-return flag.", + "title": "Return Cookie" }, "return_cookies": { - "default": false, - "description": "General/Listing cookie-return flag; defaults to false.", - "title": "Return Cookies", - "type": "boolean" + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional General/Listing cookie-return flag.", + "title": "Return Cookies" }, "scraper_id": { "description": "Required existing scraper ID.", @@ -17093,16 +17200,30 @@ "type": "string" }, "screenshot": { - "default": false, - "description": "General, Listing, or Manual screenshot flag; defaults to false.", - "title": "Screenshot", - "type": "boolean" + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional General, Listing, or Manual screenshot flag.", + "title": "Screenshot" }, "stream": { - "default": false, - "description": "Listing or Manual streaming flag; defaults to false.", - "title": "Stream", - "type": "boolean" + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional Listing or Manual streaming flag.", + "title": "Stream" }, "timeout": { "anyOf": [ @@ -17119,11 +17240,18 @@ "title": "Timeout" }, "token_cap": { - "default": 0, - "description": "Manual token cap; defaults to 0; minimum 0.", - "minimum": 0, - "title": "Token Cap", - "type": "integer" + "anyOf": [ + { + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional Manual token cap; minimum 0.", + "title": "Token Cap" }, "url": { "description": "Required URL to process in this run.", @@ -17132,10 +17260,17 @@ "type": "string" }, "use_home_page": { - "default": false, - "description": "General/Listing home-page visit flag; defaults to false.", - "title": "Use Home Page", - "type": "boolean" + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional General/Listing home-page visit flag.", + "title": "Use Home Page" }, "wait_for_selector": { "anyOf": [ diff --git a/scripts/test_mrscraper_real.py b/scripts/test_mrscraper_real.py index 62c77259db..f5059dfbbb 100644 --- a/scripts/test_mrscraper_real.py +++ b/scripts/test_mrscraper_real.py @@ -48,10 +48,10 @@ LISTING_URL = "https://www.cireba.com/cayman-islands-real-estate-listings/" SEARCH_QUERY = "CrewAI framework" -# Fill these IDs before running result/rerun tests. -AI_SCRAPER_ID = "0bc62b79-e314-4d70-a6c8-7f0bd58ae221" -MANUAL_SCRAPER_ID = "" -RESULT_ID = "" +# Configure these IDs in the environment before running result/rerun tests. +AI_SCRAPER_ID = os.getenv("MRSCRAPER_AI_SCRAPER_ID", "") +MANUAL_SCRAPER_ID = os.getenv("MRSCRAPER_MANUAL_SCRAPER_ID", "") +RESULT_ID = os.getenv("MRSCRAPER_RESULT_ID", "") MAX_OUTPUT_CHARS = 6_000 Test = Callable[[], Any] @@ -66,30 +66,31 @@ def tool(tool_class: type[Any]) -> Any: def require(value: str, name: str) -> str: """Require a configured value without printing secret contents.""" if not value.strip(): - raise RuntimeError( - f"{name} belum diisi. Set environment variable atau isi konstanta " - "di bagian atas file ini." - ) + raise RuntimeError(f"{name} is required; set the {name} environment variable.") return value def account() -> str: + """Smoke-test account information retrieval.""" return tool(MrScraperGetAccountInfoTool).run() def crawl_urls() -> str: + """Smoke-test bounded website URL discovery.""" return tool(MrScraperCrawlWebsiteUrlsTool).run( url=TARGET_URL, max_depth=1, max_pages=2, limit=5 ) def google_serp() -> str: + """Smoke-test a synchronous Google SERP request.""" return tool(MrScraperSearchGoogleSerpTool).run( query=SEARCH_QUERY, region="us", language="en", page=1, format="json" ) def extract_prompt() -> str: + """Smoke-test immediate prompt-based page extraction.""" return tool(MrScraperExtractPageByPromptTool).run( url=TARGET_URL, prompt="Extract the page title and main description.", @@ -99,21 +100,24 @@ def extract_prompt() -> str: def extract_listings() -> str: + """Smoke-test immediate real-estate listing extraction.""" return tool(MrScraperExtractListingsTool).run( url=LISTING_URL, - prompt="Extract book title and price from the first page.", + prompt="Extract the property title and price from the first page.", output_schema={"title": "string", "price": "string"}, max_pages=1, ) def extract_structured() -> str: + """Smoke-test extraction with a bundled structured-data preset.""" return tool(MrScraperExtractStructuredDataTool).run( url=TARGET_URL, category="article", mode="Cheap" ) def rendered_html() -> str: + """Smoke-test browser-rendered HTML retrieval.""" # Advanced options that remain False/None are intentionally not sent. return tool(MrScraperFetchRenderedHtmlTool).run( url=TARGET_URL, @@ -127,8 +131,9 @@ def rendered_html() -> str: def get_results() -> str: + """Smoke-test paginated result retrieval for a configured scraper.""" return tool(MrScraperGetResultsTool).run( - scraper_id=require(AI_SCRAPER_ID, "AI_SCRAPER_ID"), + scraper_id=require(AI_SCRAPER_ID, "MRSCRAPER_AI_SCRAPER_ID"), page=1, page_size=5, sort_order="DESC", @@ -136,62 +141,79 @@ def get_results() -> str: def get_latest_results() -> str: + """Smoke-test latest-result retrieval for a configured scraper.""" return tool(MrScraperGetLatestResultsTool).run( - scraper_id=require(AI_SCRAPER_ID, "AI_SCRAPER_ID"), count=5 + scraper_id=require(AI_SCRAPER_ID, "MRSCRAPER_AI_SCRAPER_ID"), count=5 ) def get_result_detail() -> str: + """Smoke-test retrieval of one configured result.""" return tool(MrScraperGetResultDetailTool).run( - result_id=require(RESULT_ID, "RESULT_ID") + result_id=require(RESULT_ID, "MRSCRAPER_RESULT_ID") ) def create_prompt_scraper() -> str: + """Smoke-test General AI scraper creation.""" return tool(MrScraperCreatePromptScraperTool).run( url=TARGET_URL, - prompt="Extract the property name and price, number of bedroom and bathroom, and mls ID.", - output_schema={"title": "string", "description": "string"}, + prompt=( + "Extract the property name, price, bedroom count, bathroom count, " + "and MLS ID." + ), + output_schema={ + "property_name": "string", + "price": "string", + "bedrooms": "number", + "bathrooms": "number", + "mls_id": "string", + }, mode="Cheap", ) def create_listing_scraper() -> str: + """Smoke-test real-estate Listing AI scraper creation.""" return tool(MrScraperCreateListingScraperTool).run( url=LISTING_URL, - prompt="Extract book title and price.", + prompt="Extract the property title and price.", output_schema={"title": "string", "price": "string"}, max_pages=1, ) def create_crawl_scraper() -> str: + """Smoke-test Map AI scraper creation.""" return tool(MrScraperCreateWebsiteCrawlScraperTool).run( url=TARGET_URL, max_depth=1, max_pages=2, limit=5 ) def run_ai_scraper() -> str: + """Smoke-test one run of a configured AI scraper.""" return tool(MrScraperRunExistingScraperTool).run( scraper_type="ai", - scraper_id=require(AI_SCRAPER_ID, "AI_SCRAPER_ID"), + scraper_id=require(AI_SCRAPER_ID, "MRSCRAPER_AI_SCRAPER_ID"), url=TARGET_URL, agent_type="general", ) def run_manual_scraper() -> str: + """Smoke-test one run of a configured manual scraper.""" return tool(MrScraperRunExistingScraperTool).run( scraper_type="manual", - scraper_id=require(MANUAL_SCRAPER_ID, "MANUAL_SCRAPER_ID"), + scraper_id=require(MANUAL_SCRAPER_ID, "MRSCRAPER_MANUAL_SCRAPER_ID"), url=TARGET_URL, ) def run_ai_batch() -> str: + """Smoke-test a small batch run of a configured AI scraper.""" return tool(MrScraperRunExistingScraperBatchTool).run( scraper_type="ai", - scraper_id=require(AI_SCRAPER_ID, "AI_SCRAPER_ID"), + scraper_id=require(AI_SCRAPER_ID, "MRSCRAPER_AI_SCRAPER_ID"), urls=[TARGET_URL, f"{TARGET_URL}/?second=1"], ) @@ -247,6 +269,7 @@ def crew_agent() -> Any: def main() -> int: + """Parse the CLI selection and execute one opt-in smoke test.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--list", action="store_true", help="List available tests") parser.add_argument("--test", choices=sorted(TESTS), help="Run one real test")