feat(crawl): add exclude URL regex to crawl and sitemap endpoints (#1982)

Stainless-Generated-From: 8f888227b73cd53ba7c029deb3a0076de22e956f

Author: stlc-bot

src/context/dev/resources/web.py

@@ -261,6 +261,7 @@ def map_urls(
         self,
         *,
         domain: str,
+        exclude_url_regex: str | Omit = omit,
         headers: Dict[str, str] | Omit = omit,
         include_subdomains: bool | Omit = omit,
         max_links: int | Omit = omit,
@@ -284,6 +285,9 @@ def map_urls(
         Args:
           domain: Domain to map, e.g. `stripe.com`.
 
+          exclude_url_regex: Optional RE2-compatible regex pattern. URLs matching this pattern are not
+              returned, even when they match urlRegex.
+
           headers: HTTP headers for the target origin. Non-empty headers bypass caching.
 
           include_subdomains: Include URLs on subdomains.
@@ -323,6 +327,7 @@ def map_urls(
                 query=maybe_transform(
                     {
                         "domain": domain,
+                        "exclude_url_regex": exclude_url_regex,
                         "headers": headers,
                         "include_subdomains": include_subdomains,
                         "max_links": max_links,
@@ -1412,6 +1417,7 @@ def web_crawl_md(
         ]
         | Omit = omit,
         exclude_selectors: Optional[SequenceNotStr[str]] | Omit = omit,
+        exclude_url_regex: str | Omit = omit,
         follow_subdomains: bool | Omit = omit,
         include_frames: bool | Omit = omit,
         include_images: bool | Omit = omit,
@@ -1449,6 +1455,9 @@ def web_crawl_md(
 
           exclude_selectors: Remove matching elements after inclusions. Exclusions take precedence.
 
+          exclude_url_regex: RE2 regex pattern. URLs matching this pattern are never followed or scraped,
+              even when they match urlRegex. The starting URL is always scraped.
+
           follow_subdomains: When true, follow links on subdomains of the starting URL's domain (e.g.
               docs.example.com when starting from example.com). www and apex are always
               treated as equivalent.
@@ -1510,6 +1519,7 @@ def web_crawl_md(
                     "url": url,
                     "country": country,
                     "exclude_selectors": exclude_selectors,
+                    "exclude_url_regex": exclude_url_regex,
                     "follow_subdomains": follow_subdomains,
                     "include_frames": include_frames,
                     "include_images": include_images,
@@ -1759,6 +1769,7 @@ async def map_urls(
         self,
         *,
         domain: str,
+        exclude_url_regex: str | Omit = omit,
         headers: Dict[str, str] | Omit = omit,
         include_subdomains: bool | Omit = omit,
         max_links: int | Omit = omit,
@@ -1782,6 +1793,9 @@ async def map_urls(
         Args:
           domain: Domain to map, e.g. `stripe.com`.
 
+          exclude_url_regex: Optional RE2-compatible regex pattern. URLs matching this pattern are not
+              returned, even when they match urlRegex.
+
           headers: HTTP headers for the target origin. Non-empty headers bypass caching.
 
           include_subdomains: Include URLs on subdomains.
@@ -1821,6 +1835,7 @@ async def map_urls(
                 query=await async_maybe_transform(
                     {
                         "domain": domain,
+                        "exclude_url_regex": exclude_url_regex,
                         "headers": headers,
                         "include_subdomains": include_subdomains,
                         "max_links": max_links,
@@ -2910,6 +2925,7 @@ async def web_crawl_md(
         ]
         | Omit = omit,
         exclude_selectors: Optional[SequenceNotStr[str]] | Omit = omit,
+        exclude_url_regex: str | Omit = omit,
         follow_subdomains: bool | Omit = omit,
         include_frames: bool | Omit = omit,
         include_images: bool | Omit = omit,
@@ -2947,6 +2963,9 @@ async def web_crawl_md(
 
           exclude_selectors: Remove matching elements after inclusions. Exclusions take precedence.
 
+          exclude_url_regex: RE2 regex pattern. URLs matching this pattern are never followed or scraped,
+              even when they match urlRegex. The starting URL is always scraped.
+
           follow_subdomains: When true, follow links on subdomains of the starting URL's domain (e.g.
               docs.example.com when starting from example.com). www and apex are always
               treated as equivalent.
@@ -3008,6 +3027,7 @@ async def web_crawl_md(
                     "url": url,
                     "country": country,
                     "exclude_selectors": exclude_selectors,
+                    "exclude_url_regex": exclude_url_regex,
                     "follow_subdomains": follow_subdomains,
                     "include_frames": include_frames,
                     "include_images": include_images,

src/context/dev/types/batch_submit_params.py

@@ -702,6 +702,12 @@ class InputCrawlDataFormats(TypedDict, total=False):
 class InputCrawlDataSourceStartURLControls(TypedDict, total=False):
     """Limits and filters for page discovery."""
 
+    exclude_regex: Annotated[str, PropertyInfo(alias="excludeRegex")]
+    """RE2 pattern for URLs to exclude, even when they match `regex`.
+
+    The `start_url` itself is always included.
+    """
+
     follow_subdomains: Annotated[bool, PropertyInfo(alias="followSubdomains")]
     """Follow links to subdomains."""
 
@@ -734,6 +740,12 @@ class InputCrawlDataSourceSitemapControls(TypedDict, total=False):
     A sitemap batch scrapes exactly those URLs and never follows links off them, so there is no crawl depth here.
     """
 
+    exclude_regex: Annotated[str, PropertyInfo(alias="excludeRegex")]
+    """
+    RE2 pattern; sitemap URLs matching it are not scraped, even when they match
+    `regex`.
+    """
+
     max_urls: Annotated[int, PropertyInfo(alias="maxUrls")]
     """Maximum pages to fetch. Unused reserved credits are refunded. Maximum 100000."""
 

src/context/dev/types/crawl_controls.py

@@ -32,6 +32,9 @@ class SourceSitemap(BaseModel):
 class CrawlControls(BaseModel):
     """Crawl settings as submitted."""
 
+    exclude_url_pattern: Optional[str] = None
+    """RE2 pattern URLs must not match to be crawled. Null when the crawl set none."""
+
     follow_subdomains: bool
     """Whether links to subdomains were followed. Always false for a sitemap crawl."""
 

src/context/dev/types/web_map_urls_params.py

@@ -15,6 +15,12 @@ class WebMapURLsParams(TypedDict, total=False):
     domain: Required[str]
     """Domain to map, e.g. `stripe.com`."""
 
+    exclude_url_regex: Annotated[str, PropertyInfo(alias="excludeUrlRegex")]
+    """Optional RE2-compatible regex pattern.
+
+    URLs matching this pattern are not returned, even when they match urlRegex.
+    """
+
     headers: Dict[str, str]
     """HTTP headers for the target origin. Non-empty headers bypass caching."""
 

src/context/dev/types/web_web_crawl_md_params.py

@@ -226,6 +226,13 @@ class WebWebCrawlMdParams(TypedDict, total=False):
     exclude_selectors: Annotated[Optional[SequenceNotStr[str]], PropertyInfo(alias="excludeSelectors")]
     """Remove matching elements after inclusions. Exclusions take precedence."""
 
+    exclude_url_regex: Annotated[str, PropertyInfo(alias="excludeUrlRegex")]
+    """RE2 regex pattern.
+
+    URLs matching this pattern are never followed or scraped, even when they match
+    urlRegex. The starting URL is always scraped.
+    """
+
     follow_subdomains: Annotated[bool, PropertyInfo(alias="followSubdomains")]
     """When true, follow links on subdomains of the starting URL's domain (e.g.
 

src/context/dev/types/web_web_crawl_md_response.py

@@ -41,8 +41,8 @@ class Metadata(BaseModel):
 
     num_skipped: int = FieldInfo(alias="numSkipped")
     """
-    Number of URLs skipped (PDFs when pdf.shouldParse=false, or URLs not matching
-    urlRegex)
+    Number of URLs skipped (PDFs when pdf.shouldParse=false, URLs not matching
+    urlRegex, or URLs matching excludeUrlRegex)
     """
 
     num_succeeded: int = FieldInfo(alias="numSucceeded")

tests/api_resources/test_web.py

@@ -186,6 +186,7 @@ def test_method_map_urls(self, client: ContextDev) -> None:
     def test_method_map_urls_with_all_params(self, client: ContextDev) -> None:
         web = client.web.map_urls(
             domain="stripe.com",
+            exclude_url_regex="/tag/|/author/",
             headers={"foo": "J!"},
             include_subdomains=True,
             max_links=1,
@@ -513,6 +514,7 @@ def test_method_web_crawl_md_with_all_params(self, client: ContextDev) -> None:
             url="https://example.com",
             country="de",
             exclude_selectors=["x"],
+            exclude_url_regex="/tag/|/author/",
             follow_subdomains=True,
             include_frames=True,
             include_images=True,
@@ -734,6 +736,7 @@ async def test_method_map_urls(self, async_client: AsyncContextDev) -> None:
     async def test_method_map_urls_with_all_params(self, async_client: AsyncContextDev) -> None:
         web = await async_client.web.map_urls(
             domain="stripe.com",
+            exclude_url_regex="/tag/|/author/",
             headers={"foo": "J!"},
             include_subdomains=True,
             max_links=1,
@@ -1061,6 +1064,7 @@ async def test_method_web_crawl_md_with_all_params(self, async_client: AsyncCont
             url="https://example.com",
             country="de",
             exclude_selectors=["x"],
+            exclude_url_regex="/tag/|/author/",
             follow_subdomains=True,
             include_frames=True,
             include_images=True,