@@ -251,6 +251,7 @@ def web_crawl_md(
251251 follow_subdomains : bool | Omit = omit ,
252252 include_images : bool | Omit = omit ,
253253 include_links : bool | Omit = omit ,
254+ max_age_ms : int | Omit = omit ,
254255 max_depth : int | Omit = omit ,
255256 max_pages : int | Omit = omit ,
256257 shorten_base64_images : bool | Omit = omit ,
@@ -278,6 +279,10 @@ def web_crawl_md(
278279
279280 include_links: Preserve hyperlinks in the Markdown output
280281
282+ max_age_ms: Return a cached result if a prior scrape for the same parameters exists and is
283+ younger than this many milliseconds. Defaults to 1 day (86400000 ms) when
284+ omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh.
285+
281286 max_depth: Maximum link depth from the starting URL (0 = only the starting page)
282287
283288 max_pages: Maximum number of pages to crawl. Hard cap: 500.
@@ -305,6 +310,7 @@ def web_crawl_md(
305310 "follow_subdomains" : follow_subdomains ,
306311 "include_images" : include_images ,
307312 "include_links" : include_links ,
313+ "max_age_ms" : max_age_ms ,
308314 "max_depth" : max_depth ,
309315 "max_pages" : max_pages ,
310316 "shorten_base64_images" : shorten_base64_images ,
@@ -478,6 +484,7 @@ def web_scrape_sitemap(
478484 * ,
479485 domain : str ,
480486 max_links : int | Omit = omit ,
487+ url_regex : str | Omit = omit ,
481488 # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
482489 # The extra values given here take precedence over values defined on the client or passed to this method.
483490 extra_headers : Headers | None = None ,
@@ -494,6 +501,9 @@ def web_scrape_sitemap(
494501 max_links: Maximum number of links to return from the sitemap crawl. Defaults to 10,000.
495502 Minimum is 1, maximum is 100,000.
496503
504+ url_regex: Optional RE2-compatible regex pattern. Only URLs matching this pattern are
505+ returned and counted against maxLinks.
506+
497507 extra_headers: Send extra headers
498508
499509 extra_query: Add additional query parameters to the request
@@ -513,6 +523,7 @@ def web_scrape_sitemap(
513523 {
514524 "domain" : domain ,
515525 "max_links" : max_links ,
526+ "url_regex" : url_regex ,
516527 },
517528 web_web_scrape_sitemap_params .WebWebScrapeSitemapParams ,
518529 ),
@@ -733,6 +744,7 @@ async def web_crawl_md(
733744 follow_subdomains : bool | Omit = omit ,
734745 include_images : bool | Omit = omit ,
735746 include_links : bool | Omit = omit ,
747+ max_age_ms : int | Omit = omit ,
736748 max_depth : int | Omit = omit ,
737749 max_pages : int | Omit = omit ,
738750 shorten_base64_images : bool | Omit = omit ,
@@ -760,6 +772,10 @@ async def web_crawl_md(
760772
761773 include_links: Preserve hyperlinks in the Markdown output
762774
775+ max_age_ms: Return a cached result if a prior scrape for the same parameters exists and is
776+ younger than this many milliseconds. Defaults to 1 day (86400000 ms) when
777+ omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh.
778+
763779 max_depth: Maximum link depth from the starting URL (0 = only the starting page)
764780
765781 max_pages: Maximum number of pages to crawl. Hard cap: 500.
@@ -787,6 +803,7 @@ async def web_crawl_md(
787803 "follow_subdomains" : follow_subdomains ,
788804 "include_images" : include_images ,
789805 "include_links" : include_links ,
806+ "max_age_ms" : max_age_ms ,
790807 "max_depth" : max_depth ,
791808 "max_pages" : max_pages ,
792809 "shorten_base64_images" : shorten_base64_images ,
@@ -960,6 +977,7 @@ async def web_scrape_sitemap(
960977 * ,
961978 domain : str ,
962979 max_links : int | Omit = omit ,
980+ url_regex : str | Omit = omit ,
963981 # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
964982 # The extra values given here take precedence over values defined on the client or passed to this method.
965983 extra_headers : Headers | None = None ,
@@ -976,6 +994,9 @@ async def web_scrape_sitemap(
976994 max_links: Maximum number of links to return from the sitemap crawl. Defaults to 10,000.
977995 Minimum is 1, maximum is 100,000.
978996
997+ url_regex: Optional RE2-compatible regex pattern. Only URLs matching this pattern are
998+ returned and counted against maxLinks.
999+
9791000 extra_headers: Send extra headers
9801001
9811002 extra_query: Add additional query parameters to the request
@@ -995,6 +1016,7 @@ async def web_scrape_sitemap(
9951016 {
9961017 "domain" : domain ,
9971018 "max_links" : max_links ,
1019+ "url_regex" : url_regex ,
9981020 },
9991021 web_web_scrape_sitemap_params .WebWebScrapeSitemapParams ,
10001022 ),
0 commit comments