feat(scrape): add product extraction format to POST /web/scrape (#1212)

Stainless-Generated-From: 0c94f72

Author: stlc-bot

src/context/dev/resources/web.py

@@ -397,6 +397,7 @@ def scrape(
         markdown_params: web_scrape_params.MarkdownParams | Omit = omit,
         max_age_ms: int | Omit = omit,
         parse_params: web_scrape_params.ParseParams | Omit = omit,
+        product_params: web_scrape_params.ProductParams | Omit = omit,
         screenshot_params: web_scrape_params.ScreenshotParams | Omit = omit,
         shared_params: web_scrape_params.SharedParams | Omit = omit,
         tags: SequenceNotStr[str] | Omit = omit,
@@ -413,16 +414,18 @@ def scrape(
         visit.
 
         Each cache key includes only the settings that affect that output. HTML
-        is shared with Markdown, parsed fields, highlights, and JSON extraction. Cached
-        outputs can come from different visits within maxAgeMs; use 0 for a fresh
-        capture. HTML-only requests use the existing fast acquisition path. Highlights
-        return the plain-text passages most relevant to highlightsParams.query. One
-        credit per request, including cache hits and missing pages, or two with browser
-        actions; highlights add 3 credits when passages are returned; JSON extraction
-        adds four credits and runs an LLM over the page Markdown on every request that
-        has text to extract; PDF OCR adds one credit per recovered page on fresh
-        extraction. Original response bytes and screenshots are limited to 20 MiB each,
-        screenshots to 40 megapixels, and the combined browser capture to 60 MiB.
+        is shared with Markdown, parsed fields, product data, highlights, and JSON
+        extraction. Cached outputs can come from different visits within maxAgeMs; use 0
+        for a fresh capture. HTML-only requests use the existing fast acquisition path.
+        Highlights return the plain-text passages most relevant to
+        highlightsParams.query. One credit per request, including cache hits and missing
+        pages, or two with browser actions; highlights add 3 credits when passages are
+        returned; JSON extraction adds four credits and runs an LLM over the page
+        Markdown on every request that has text to extract; PDF OCR adds one credit per
+        recovered page on fresh extraction; the product output adds one credit, plus six
+        more when the specialized model is used. Original response bytes and screenshots
+        are limited to 20 MiB each, screenshots to 40 megapixels, and the combined
+        browser capture to 60 MiB.
 
         Args:
           formats: Outputs to return. Enable at least one; omitted formats are false.
@@ -444,6 +447,8 @@ def scrape(
 
           parse_params: Required when formats.parse is true.
 
+          product_params: Product options. Requires formats.product: true.
+
           screenshot_params: Screenshot options. Requires formats.screenshot: true.
 
           shared_params: Shared browser and content settings. Content filters leave screenshots and
@@ -483,6 +488,7 @@ def scrape(
                     "markdown_params": markdown_params,
                     "max_age_ms": max_age_ms,
                     "parse_params": parse_params,
+                    "product_params": product_params,
                     "screenshot_params": screenshot_params,
                     "shared_params": shared_params,
                     "tags": tags,
@@ -1887,6 +1893,7 @@ async def scrape(
         markdown_params: web_scrape_params.MarkdownParams | Omit = omit,
         max_age_ms: int | Omit = omit,
         parse_params: web_scrape_params.ParseParams | Omit = omit,
+        product_params: web_scrape_params.ProductParams | Omit = omit,
         screenshot_params: web_scrape_params.ScreenshotParams | Omit = omit,
         shared_params: web_scrape_params.SharedParams | Omit = omit,
         tags: SequenceNotStr[str] | Omit = omit,
@@ -1903,16 +1910,18 @@ async def scrape(
         visit.
 
         Each cache key includes only the settings that affect that output. HTML
-        is shared with Markdown, parsed fields, highlights, and JSON extraction. Cached
-        outputs can come from different visits within maxAgeMs; use 0 for a fresh
-        capture. HTML-only requests use the existing fast acquisition path. Highlights
-        return the plain-text passages most relevant to highlightsParams.query. One
-        credit per request, including cache hits and missing pages, or two with browser
-        actions; highlights add 3 credits when passages are returned; JSON extraction
-        adds four credits and runs an LLM over the page Markdown on every request that
-        has text to extract; PDF OCR adds one credit per recovered page on fresh
-        extraction. Original response bytes and screenshots are limited to 20 MiB each,
-        screenshots to 40 megapixels, and the combined browser capture to 60 MiB.
+        is shared with Markdown, parsed fields, product data, highlights, and JSON
+        extraction. Cached outputs can come from different visits within maxAgeMs; use 0
+        for a fresh capture. HTML-only requests use the existing fast acquisition path.
+        Highlights return the plain-text passages most relevant to
+        highlightsParams.query. One credit per request, including cache hits and missing
+        pages, or two with browser actions; highlights add 3 credits when passages are
+        returned; JSON extraction adds four credits and runs an LLM over the page
+        Markdown on every request that has text to extract; PDF OCR adds one credit per
+        recovered page on fresh extraction; the product output adds one credit, plus six
+        more when the specialized model is used. Original response bytes and screenshots
+        are limited to 20 MiB each, screenshots to 40 megapixels, and the combined
+        browser capture to 60 MiB.
 
         Args:
           formats: Outputs to return. Enable at least one; omitted formats are false.
@@ -1934,6 +1943,8 @@ async def scrape(
 
           parse_params: Required when formats.parse is true.
 
+          product_params: Product options. Requires formats.product: true.
+
           screenshot_params: Screenshot options. Requires formats.screenshot: true.
 
           shared_params: Shared browser and content settings. Content filters leave screenshots and
@@ -1973,6 +1984,7 @@ async def scrape(
                     "markdown_params": markdown_params,
                     "max_age_ms": max_age_ms,
                     "parse_params": parse_params,
+                    "product_params": product_params,
                     "screenshot_params": screenshot_params,
                     "shared_params": shared_params,
                     "tags": tags,

src/context/dev/types/web_scrape_params.py

@@ -18,6 +18,7 @@
     "ParseParams",
     "ParseParamsRules",
     "ParseParamsRulesUnionMember1",
+    "ProductParams",
     "ScreenshotParams",
     "ScreenshotParamsArea",
     "ScreenshotParamsAreaElement",
@@ -65,6 +66,9 @@ class WebScrapeParams(TypedDict, total=False):
     parse_params: Annotated[ParseParams, PropertyInfo(alias="parseParams")]
     """Required when formats.parse is true."""
 
+    product_params: Annotated[ProductParams, PropertyInfo(alias="productParams")]
+    """Product options. Requires formats.product: true."""
+
     screenshot_params: Annotated[ScreenshotParams, PropertyInfo(alias="screenshotParams")]
     """Screenshot options. Requires formats.screenshot: true."""
 
@@ -130,6 +134,9 @@ class Formats(TypedDict, total=False):
     parse: bool
     """Fields selected by parseParams.rules."""
 
+    product: bool
+    """Structured product data for product detail pages. Adds one credit."""
+
     screenshot: bool
     """An inline image of the page."""
 
@@ -210,6 +217,18 @@ class ParseParams(TypedDict, total=False):
     """
 
 
+class ProductParams(TypedDict, total=False):
+    """Product options. Requires formats.product: true."""
+
+    use_ai_fallback: Annotated[bool, PropertyInfo(alias="useAIFallback")]
+    """
+    Extract the product with a specialized model when the page has no structured
+    product data. Adds six credits when the model returns a verdict. If the fallback
+    fails, returns a partial response with the deterministic result and no fallback
+    charge. Request deadlines and client disconnects still apply.
+    """
+
+
 class ScreenshotParamsAreaElement(TypedDict, total=False):
     selector: Required[str]
     """Must match one visible element."""

src/context/dev/types/web_scrape_response.py

@@ -22,6 +22,10 @@
     "MetadataAlternate",
     "MetadataHeading",
     "Parsed",
+    "Product",
+    "ProductData",
+    "ProductDataProduct",
+    "ProductDataProductVariant",
     "Screenshot",
     "KeyMetadata",
 ]
@@ -222,6 +226,98 @@ class Parsed(BaseModel):
     requested: bool
 
 
+class ProductDataProductVariant(BaseModel):
+    attributes: Dict[str, str]
+    """
+    Explicit variant attributes such as color, size, material, pattern and
+    properties declared by page.
+    """
+
+    images: List[str]
+    """Original source image URLs explicitly attached to this variant."""
+
+    sku: Optional[str] = None
+
+    url: Optional[str] = None
+    """Variant or offer URL when provided by the source. May be shared by variants."""
+
+
+class ProductDataProduct(BaseModel):
+    """The extracted product, or null when the page is not a product detail page."""
+
+    availability: Optional[
+        Literal[
+            "in_stock", "out_of_stock", "limited_availability", "preorder", "backorder", "made_to_order", "discontinued"
+        ]
+    ] = None
+    """Stock or ordering availability."""
+
+    brand: Optional[str] = None
+    """Brand or vendor."""
+
+    category: Optional[str] = None
+    """Product category."""
+
+    currency: Optional[str] = None
+    """ISO 4217 currency code."""
+
+    description: Optional[str] = None
+    """Product description."""
+
+    dimensions: List[str]
+    """Product dimensions as shown on the page."""
+
+    features: List[str]
+    """Key features and specifications."""
+
+    images: List[str]
+    """Product image URLs, main image first."""
+
+    image_url: Optional[str] = FieldInfo(alias="imageUrl", default=None)
+    """Main product image URL."""
+
+    name: str
+    """Product name."""
+
+    price: Optional[float] = None
+    """Current price."""
+
+    regular_price: Optional[float] = FieldInfo(alias="regularPrice", default=None)
+    """List price before any discount."""
+
+    sku: Optional[str] = None
+    """Product identifier such as a SKU or model number."""
+
+    tags: List[str]
+    """Product tags."""
+
+    target_audience: List[str] = FieldInfo(alias="targetAudience")
+    """Intended audience."""
+
+    variants: List[ProductDataProductVariant]
+    """
+    Product variations, such as different colors or sizes, with their attributes and
+    images. Empty if none are found. May not include every variation offered by the
+    store.
+    """
+
+
+class ProductData(BaseModel):
+    is_product_page: bool = FieldInfo(alias="isProductPage")
+    """Whether the page is a product detail page."""
+
+    product: Optional[ProductDataProduct] = None
+    """The extracted product, or null when the page is not a product detail page."""
+
+
+class Product(BaseModel):
+    """Product detail page classification and the extracted product."""
+
+    data: Optional[ProductData] = None
+
+    requested: bool
+
+
 class Screenshot(BaseModel):
     """An image data URL. Use directly as an image src."""
 
@@ -284,6 +380,9 @@ class WebScrapeResponse(BaseModel):
     parsed: Parsed
     """Fields produced by parseParams.rules, after shared content filters."""
 
+    product: Product
+    """Product detail page classification and the extracted product."""
+
     request_id: str
     """Unique id of this API call, also sent in the X-Request-Id response header.
 
@@ -298,8 +397,10 @@ class WebScrapeResponse(BaseModel):
 
     is_partial: Optional[Literal[True]] = FieldInfo(alias="isPartial", default=None)
     """
-    Present when return-partial captures a page that is still loading or returns
-    images before image processing finishes. Partial responses are not cached.
+    Present when return-partial captures a page that is still loading, returns
+    images before image processing finishes, or cuts product AI extraction short.
+    Also present if the optional product AI fallback fails. Partial responses are
+    not cached.
     """
 
     key_metadata: Optional[KeyMetadata] = None

tests/api_resources/test_web.py

@@ -248,6 +248,7 @@ def test_method_scrape_with_all_params(self, client: ContextDev) -> None:
                 "json": True,
                 "markdown": True,
                 "parse": True,
+                "product": True,
                 "screenshot": True,
             },
             url="https://example.com",
@@ -284,6 +285,7 @@ def test_method_scrape_with_all_params(self, client: ContextDev) -> None:
                     },
                 }
             },
+            product_params={"use_ai_fallback": True},
             screenshot_params={
                 "area": "viewport",
                 "format": "png",
@@ -783,6 +785,7 @@ async def test_method_scrape_with_all_params(self, async_client: AsyncContextDev
                 "json": True,
                 "markdown": True,
                 "parse": True,
+                "product": True,
                 "screenshot": True,
             },
             url="https://example.com",
@@ -819,6 +822,7 @@ async def test_method_scrape_with_all_params(self, async_client: AsyncContextDev
                     },
                 }
             },
+            product_params={"use_ai_fallback": True},
             screenshot_params={
                 "area": "viewport",
                 "format": "png",