88
99from ..types import (
1010 web_screenshot_params ,
11+ web_web_crawl_md_params ,
1112 web_web_scrape_md_params ,
1213 web_web_scrape_html_params ,
1314 web_web_scrape_images_params ,
2526)
2627from .._base_client import make_request_options
2728from ..types .web_screenshot_response import WebScreenshotResponse
29+ from ..types .web_web_crawl_md_response import WebWebCrawlMdResponse
2830from ..types .web_web_scrape_md_response import WebWebScrapeMdResponse
2931from ..types .web_web_scrape_html_response import WebWebScrapeHTMLResponse
3032from ..types .web_web_scrape_images_response import WebWebScrapeImagesResponse
@@ -119,6 +121,82 @@ def screenshot(
119121 cast_to = WebScreenshotResponse ,
120122 )
121123
124+ def web_crawl_md (
125+ self ,
126+ * ,
127+ url : str ,
128+ follow_subdomains : bool | Omit = omit ,
129+ include_images : bool | Omit = omit ,
130+ include_links : bool | Omit = omit ,
131+ max_depth : int | Omit = omit ,
132+ max_pages : int | Omit = omit ,
133+ shorten_base64_images : bool | Omit = omit ,
134+ url_regex : str | Omit = omit ,
135+ use_main_content_only : bool | Omit = omit ,
136+ # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
137+ # The extra values given here take precedence over values defined on the client or passed to this method.
138+ extra_headers : Headers | None = None ,
139+ extra_query : Query | None = None ,
140+ extra_body : Body | None = None ,
141+ timeout : float | httpx .Timeout | None | NotGiven = not_given ,
142+ ) -> WebWebCrawlMdResponse :
143+ """
144+ Performs a crawl starting from a given URL, extracts page content as Markdown,
145+ and returns results for all crawled pages. Only follows links within the same
146+ domain as the starting URL. Costs 1 credit per successful page crawled.
147+
148+ Args:
149+ url: The starting URL for the crawl (must include http:// or https:// protocol)
150+
151+ follow_subdomains: When true, follow links on subdomains of the starting URL's domain (e.g.
152+ docs.example.com when starting from example.com). www and apex are always
153+ treated as equivalent.
154+
155+ include_images: Include image references in the Markdown output
156+
157+ include_links: Preserve hyperlinks in the Markdown output
158+
159+ max_depth: Maximum link depth from the starting URL (0 = only the starting page)
160+
161+ max_pages: Maximum number of pages to crawl. Hard cap: 500.
162+
163+ shorten_base64_images: Truncate base64-encoded image data in the Markdown output
164+
165+ url_regex: Regex pattern. Only URLs matching this pattern will be followed and scraped.
166+
167+ use_main_content_only: Extract only the main content, stripping headers, footers, sidebars, and
168+ navigation
169+
170+ extra_headers: Send extra headers
171+
172+ extra_query: Add additional query parameters to the request
173+
174+ extra_body: Add additional JSON properties to the request
175+
176+ timeout: Override the client-level default timeout for this request, in seconds
177+ """
178+ return self ._post (
179+ "/web/crawl" ,
180+ body = maybe_transform (
181+ {
182+ "url" : url ,
183+ "follow_subdomains" : follow_subdomains ,
184+ "include_images" : include_images ,
185+ "include_links" : include_links ,
186+ "max_depth" : max_depth ,
187+ "max_pages" : max_pages ,
188+ "shorten_base64_images" : shorten_base64_images ,
189+ "url_regex" : url_regex ,
190+ "use_main_content_only" : use_main_content_only ,
191+ },
192+ web_web_crawl_md_params .WebWebCrawlMdParams ,
193+ ),
194+ options = make_request_options (
195+ extra_headers = extra_headers , extra_query = extra_query , extra_body = extra_body , timeout = timeout
196+ ),
197+ cast_to = WebWebCrawlMdResponse ,
198+ )
199+
122200 def web_scrape_html (
123201 self ,
124202 * ,
@@ -394,6 +472,82 @@ async def screenshot(
394472 cast_to = WebScreenshotResponse ,
395473 )
396474
475+ async def web_crawl_md (
476+ self ,
477+ * ,
478+ url : str ,
479+ follow_subdomains : bool | Omit = omit ,
480+ include_images : bool | Omit = omit ,
481+ include_links : bool | Omit = omit ,
482+ max_depth : int | Omit = omit ,
483+ max_pages : int | Omit = omit ,
484+ shorten_base64_images : bool | Omit = omit ,
485+ url_regex : str | Omit = omit ,
486+ use_main_content_only : bool | Omit = omit ,
487+ # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
488+ # The extra values given here take precedence over values defined on the client or passed to this method.
489+ extra_headers : Headers | None = None ,
490+ extra_query : Query | None = None ,
491+ extra_body : Body | None = None ,
492+ timeout : float | httpx .Timeout | None | NotGiven = not_given ,
493+ ) -> WebWebCrawlMdResponse :
494+ """
495+ Performs a crawl starting from a given URL, extracts page content as Markdown,
496+ and returns results for all crawled pages. Only follows links within the same
497+ domain as the starting URL. Costs 1 credit per successful page crawled.
498+
499+ Args:
500+ url: The starting URL for the crawl (must include http:// or https:// protocol)
501+
502+ follow_subdomains: When true, follow links on subdomains of the starting URL's domain (e.g.
503+ docs.example.com when starting from example.com). www and apex are always
504+ treated as equivalent.
505+
506+ include_images: Include image references in the Markdown output
507+
508+ include_links: Preserve hyperlinks in the Markdown output
509+
510+ max_depth: Maximum link depth from the starting URL (0 = only the starting page)
511+
512+ max_pages: Maximum number of pages to crawl. Hard cap: 500.
513+
514+ shorten_base64_images: Truncate base64-encoded image data in the Markdown output
515+
516+ url_regex: Regex pattern. Only URLs matching this pattern will be followed and scraped.
517+
518+ use_main_content_only: Extract only the main content, stripping headers, footers, sidebars, and
519+ navigation
520+
521+ extra_headers: Send extra headers
522+
523+ extra_query: Add additional query parameters to the request
524+
525+ extra_body: Add additional JSON properties to the request
526+
527+ timeout: Override the client-level default timeout for this request, in seconds
528+ """
529+ return await self ._post (
530+ "/web/crawl" ,
531+ body = await async_maybe_transform (
532+ {
533+ "url" : url ,
534+ "follow_subdomains" : follow_subdomains ,
535+ "include_images" : include_images ,
536+ "include_links" : include_links ,
537+ "max_depth" : max_depth ,
538+ "max_pages" : max_pages ,
539+ "shorten_base64_images" : shorten_base64_images ,
540+ "url_regex" : url_regex ,
541+ "use_main_content_only" : use_main_content_only ,
542+ },
543+ web_web_crawl_md_params .WebWebCrawlMdParams ,
544+ ),
545+ options = make_request_options (
546+ extra_headers = extra_headers , extra_query = extra_query , extra_body = extra_body , timeout = timeout
547+ ),
548+ cast_to = WebWebCrawlMdResponse ,
549+ )
550+
397551 async def web_scrape_html (
398552 self ,
399553 * ,
@@ -590,6 +744,9 @@ def __init__(self, web: WebResource) -> None:
590744 self .screenshot = to_raw_response_wrapper (
591745 web .screenshot ,
592746 )
747+ self .web_crawl_md = to_raw_response_wrapper (
748+ web .web_crawl_md ,
749+ )
593750 self .web_scrape_html = to_raw_response_wrapper (
594751 web .web_scrape_html ,
595752 )
@@ -611,6 +768,9 @@ def __init__(self, web: AsyncWebResource) -> None:
611768 self .screenshot = async_to_raw_response_wrapper (
612769 web .screenshot ,
613770 )
771+ self .web_crawl_md = async_to_raw_response_wrapper (
772+ web .web_crawl_md ,
773+ )
614774 self .web_scrape_html = async_to_raw_response_wrapper (
615775 web .web_scrape_html ,
616776 )
@@ -632,6 +792,9 @@ def __init__(self, web: WebResource) -> None:
632792 self .screenshot = to_streamed_response_wrapper (
633793 web .screenshot ,
634794 )
795+ self .web_crawl_md = to_streamed_response_wrapper (
796+ web .web_crawl_md ,
797+ )
635798 self .web_scrape_html = to_streamed_response_wrapper (
636799 web .web_scrape_html ,
637800 )
@@ -653,6 +816,9 @@ def __init__(self, web: AsyncWebResource) -> None:
653816 self .screenshot = async_to_streamed_response_wrapper (
654817 web .screenshot ,
655818 )
819+ self .web_crawl_md = async_to_streamed_response_wrapper (
820+ web .web_crawl_md ,
821+ )
656822 self .web_scrape_html = async_to_streamed_response_wrapper (
657823 web .web_scrape_html ,
658824 )
0 commit comments