diff --git a/.release-please-manifest.json b/.release-please-manifest.json
index 7deae338..cce9d1c6 100644
--- a/.release-please-manifest.json
+++ b/.release-please-manifest.json
@@ -1,3 +1,3 @@
{
- ".": "1.6.0"
+ ".": "1.7.0"
}
\ No newline at end of file
diff --git a/.stats.yml b/.stats.yml
index 98e8c8a2..592a4b2a 100644
--- a/.stats.yml
+++ b/.stats.yml
@@ -1,4 +1,4 @@
configured_endpoints: 21
-openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev%2Fcontext.dev-ca8e38b0f28a9967dab631b8868f1d59bc035fd994a717e0125c77ca592bd105.yml
-openapi_spec_hash: ef0a5df01201a032dcc41f9a25b733e6
+openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev%2Fcontext.dev-ee23a181bfa364090a90254a433704adf4d3b413730fe75e5ad9a26273ec517a.yml
+openapi_spec_hash: bc99d89cb1e7cdacc997214305bffee7
config_hash: 7d13dca2b2c6f71fc463cb6062efa5ea
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5da5a4a7..ce5d0565 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,13 @@
# Changelog
+## 1.7.0 (2026-04-24)
+
+Full Changelog: [v1.6.0...v1.7.0](https://github.com/context-dot-dev/context-ruby-sdk/compare/v1.6.0...v1.7.0)
+
+### Features
+
+* **api:** api update ([9c8b4d9](https://github.com/context-dot-dev/context-ruby-sdk/commit/9c8b4d9b1d813c5b7992998a3fa23cde63fe2f4c))
+
## 1.6.0 (2026-04-23)
Full Changelog: [v1.5.0...v1.6.0](https://github.com/context-dot-dev/context-ruby-sdk/compare/v1.5.0...v1.6.0)
diff --git a/Gemfile.lock b/Gemfile.lock
index 423db933..52851479 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -11,7 +11,7 @@ GIT
PATH
remote: .
specs:
- context.dev (1.6.0)
+ context.dev (1.7.0)
cgi
connection_pool
diff --git a/README.md b/README.md
index a875fb76..87c40c61 100644
--- a/README.md
+++ b/README.md
@@ -26,7 +26,7 @@ To use this gem, install via Bundler by adding the following to your application
```ruby
-gem "context.dev", "~> 1.6.0"
+gem "context.dev", "~> 1.7.0"
```
diff --git a/lib/context_dev/models/web_web_crawl_md_params.rb b/lib/context_dev/models/web_web_crawl_md_params.rb
index f21f3fe2..a71676a5 100644
--- a/lib/context_dev/models/web_web_crawl_md_params.rb
+++ b/lib/context_dev/models/web_web_crawl_md_params.rb
@@ -53,6 +53,14 @@ class WebWebCrawlMdParams < ContextDev::Internal::Type::BaseModel
# @return [Integer, nil]
optional :max_pages, Integer, api_name: :maxPages
+ # @!attribute parse_pdf
+ # When true (default), PDF pages are fetched and their text layer is extracted and
+ # converted to Markdown alongside HTML pages. When false, PDF pages are skipped
+ # entirely (not included in results and not counted as failures).
+ #
+ # @return [Boolean, nil]
+ optional :parse_pdf, ContextDev::Internal::Type::Boolean, api_name: :parsePDF
+
# @!attribute shorten_base64_images
# Truncate base64-encoded image data in the Markdown output
#
@@ -72,7 +80,7 @@ class WebWebCrawlMdParams < ContextDev::Internal::Type::BaseModel
# @return [Boolean, nil]
optional :use_main_content_only, ContextDev::Internal::Type::Boolean, api_name: :useMainContentOnly
- # @!method initialize(url:, follow_subdomains: nil, include_images: nil, include_links: nil, max_age_ms: nil, max_depth: nil, max_pages: nil, shorten_base64_images: nil, url_regex: nil, use_main_content_only: nil, request_options: {})
+ # @!method initialize(url:, follow_subdomains: nil, include_images: nil, include_links: nil, max_age_ms: nil, max_depth: nil, max_pages: nil, parse_pdf: nil, shorten_base64_images: nil, url_regex: nil, use_main_content_only: nil, request_options: {})
# Some parameter documentations has been truncated, see
# {ContextDev::Models::WebWebCrawlMdParams} for more details.
#
@@ -90,6 +98,8 @@ class WebWebCrawlMdParams < ContextDev::Internal::Type::BaseModel
#
# @param max_pages [Integer] Maximum number of pages to crawl. Hard cap: 500.
#
+ # @param parse_pdf [Boolean] When true (default), PDF pages are fetched and their text layer is extracted and
+ #
# @param shorten_base64_images [Boolean] Truncate base64-encoded image data in the Markdown output
#
# @param url_regex [String] Regex pattern. Only URLs matching this pattern will be followed and scraped.
diff --git a/lib/context_dev/models/web_web_crawl_md_response.rb b/lib/context_dev/models/web_web_crawl_md_response.rb
index 45f09d72..a2527ccb 100644
--- a/lib/context_dev/models/web_web_crawl_md_response.rb
+++ b/lib/context_dev/models/web_web_crawl_md_response.rb
@@ -33,6 +33,12 @@ class Metadata < ContextDev::Internal::Type::BaseModel
# @return [Integer]
required :num_failed, Integer, api_name: :numFailed
+ # @!attribute num_skipped
+ # Number of URLs skipped (PDFs when parsePDF=false, or URLs not matching urlRegex)
+ #
+ # @return [Integer]
+ required :num_skipped, Integer, api_name: :numSkipped
+
# @!attribute num_succeeded
# Number of pages successfully crawled
#
@@ -45,11 +51,16 @@ class Metadata < ContextDev::Internal::Type::BaseModel
# @return [Integer]
required :num_urls, Integer, api_name: :numUrls
- # @!method initialize(max_crawl_depth:, num_failed:, num_succeeded:, num_urls:)
+ # @!method initialize(max_crawl_depth:, num_failed:, num_skipped:, num_succeeded:, num_urls:)
+ # Some parameter documentations has been truncated, see
+ # {ContextDev::Models::WebWebCrawlMdResponse::Metadata} for more details.
+ #
# @param max_crawl_depth [Integer] Maximum crawl depth reached during the crawl
#
# @param num_failed [Integer] Number of pages that failed to crawl
#
+ # @param num_skipped [Integer] Number of URLs skipped (PDFs when parsePDF=false, or URLs not matching urlRegex)
+ #
# @param num_succeeded [Integer] Number of pages successfully crawled
#
# @param num_urls [Integer] Total number of URLs crawled
diff --git a/lib/context_dev/models/web_web_scrape_html_params.rb b/lib/context_dev/models/web_web_scrape_html_params.rb
index de82101e..e399d482 100644
--- a/lib/context_dev/models/web_web_scrape_html_params.rb
+++ b/lib/context_dev/models/web_web_scrape_html_params.rb
@@ -21,7 +21,15 @@ class WebWebScrapeHTMLParams < ContextDev::Internal::Type::BaseModel
# @return [Integer, nil]
optional :max_age_ms, Integer
- # @!method initialize(url:, max_age_ms: nil, request_options: {})
+ # @!attribute parse_pdf
+ # When true (default), PDF URLs are fetched and their text layer is extracted and
+ # returned wrapped in …. When false, PDF URLs are skipped
+ # and a 400 WEBSITE_ACCESS_ERROR is returned.
+ #
+ # @return [Boolean, nil]
+ optional :parse_pdf, ContextDev::Internal::Type::Boolean
+
+ # @!method initialize(url:, max_age_ms: nil, parse_pdf: nil, request_options: {})
# Some parameter documentations has been truncated, see
# {ContextDev::Models::WebWebScrapeHTMLParams} for more details.
#
@@ -29,6 +37,8 @@ class WebWebScrapeHTMLParams < ContextDev::Internal::Type::BaseModel
#
# @param max_age_ms [Integer] Return a cached result if a prior scrape for the same parameters exists and is y
#
+ # @param parse_pdf [Boolean] When true (default), PDF URLs are fetched and their text layer is extracted and
+ #
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}]
end
end
diff --git a/lib/context_dev/models/web_web_scrape_md_params.rb b/lib/context_dev/models/web_web_scrape_md_params.rb
index 3303725d..8f2f7c99 100644
--- a/lib/context_dev/models/web_web_scrape_md_params.rb
+++ b/lib/context_dev/models/web_web_scrape_md_params.rb
@@ -34,6 +34,14 @@ class WebWebScrapeMdParams < ContextDev::Internal::Type::BaseModel
# @return [Integer, nil]
optional :max_age_ms, Integer
+ # @!attribute parse_pdf
+ # When true (default), PDF URLs are fetched and their text layer is extracted and
+ # converted to Markdown. When false, PDF URLs are skipped and a 400
+ # WEBSITE_ACCESS_ERROR is returned.
+ #
+ # @return [Boolean, nil]
+ optional :parse_pdf, ContextDev::Internal::Type::Boolean
+
# @!attribute shorten_base64_images
# Shorten base64-encoded image data in the Markdown output
#
@@ -47,7 +55,7 @@ class WebWebScrapeMdParams < ContextDev::Internal::Type::BaseModel
# @return [Boolean, nil]
optional :use_main_content_only, ContextDev::Internal::Type::Boolean
- # @!method initialize(url:, include_images: nil, include_links: nil, max_age_ms: nil, shorten_base64_images: nil, use_main_content_only: nil, request_options: {})
+ # @!method initialize(url:, include_images: nil, include_links: nil, max_age_ms: nil, parse_pdf: nil, shorten_base64_images: nil, use_main_content_only: nil, request_options: {})
# Some parameter documentations has been truncated, see
# {ContextDev::Models::WebWebScrapeMdParams} for more details.
#
@@ -59,6 +67,8 @@ class WebWebScrapeMdParams < ContextDev::Internal::Type::BaseModel
#
# @param max_age_ms [Integer] Return a cached result if a prior scrape for the same parameters exists and is y
#
+ # @param parse_pdf [Boolean] When true (default), PDF URLs are fetched and their text layer is extracted and
+ #
# @param shorten_base64_images [Boolean] Shorten base64-encoded image data in the Markdown output
#
# @param use_main_content_only [Boolean] Extract only the main content of the page, excluding headers, footers, sidebars,
diff --git a/lib/context_dev/resources/web.rb b/lib/context_dev/resources/web.rb
index 213403b7..b67b32ea 100644
--- a/lib/context_dev/resources/web.rb
+++ b/lib/context_dev/resources/web.rb
@@ -105,7 +105,7 @@ def screenshot(params = {})
# Performs a crawl starting from a given URL, extracts page content as Markdown,
# and returns results for all crawled pages.
#
- # @overload web_crawl_md(url:, follow_subdomains: nil, include_images: nil, include_links: nil, max_age_ms: nil, max_depth: nil, max_pages: nil, shorten_base64_images: nil, url_regex: nil, use_main_content_only: nil, request_options: {})
+ # @overload web_crawl_md(url:, follow_subdomains: nil, include_images: nil, include_links: nil, max_age_ms: nil, max_depth: nil, max_pages: nil, parse_pdf: nil, shorten_base64_images: nil, url_regex: nil, use_main_content_only: nil, request_options: {})
#
# @param url [String] The starting URL for the crawl (must include http:// or https:// protocol)
#
@@ -121,6 +121,8 @@ def screenshot(params = {})
#
# @param max_pages [Integer] Maximum number of pages to crawl. Hard cap: 500.
#
+ # @param parse_pdf [Boolean] When true (default), PDF pages are fetched and their text layer is extracted and
+ #
# @param shorten_base64_images [Boolean] Truncate base64-encoded image data in the Markdown output
#
# @param url_regex [String] Regex pattern. Only URLs matching this pattern will be followed and scraped.
@@ -148,12 +150,14 @@ def web_crawl_md(params)
#
# Scrapes the given URL and returns the raw HTML content of the page.
#
- # @overload web_scrape_html(url:, max_age_ms: nil, request_options: {})
+ # @overload web_scrape_html(url:, max_age_ms: nil, parse_pdf: nil, request_options: {})
#
# @param url [String] Full URL to scrape (must include http:// or https:// protocol)
#
# @param max_age_ms [Integer] Return a cached result if a prior scrape for the same parameters exists and is y
#
+ # @param parse_pdf [Boolean] When true (default), PDF URLs are fetched and their text layer is extracted and
+ #
# @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}, nil]
#
# @return [ContextDev::Models::WebWebScrapeHTMLResponse]
@@ -165,7 +169,7 @@ def web_scrape_html(params)
@client.request(
method: :get,
path: "web/scrape/html",
- query: query.transform_keys(max_age_ms: "maxAgeMs"),
+ query: query.transform_keys(max_age_ms: "maxAgeMs", parse_pdf: "parsePDF"),
model: ContextDev::Models::WebWebScrapeHTMLResponse,
options: options
)
@@ -201,7 +205,7 @@ def web_scrape_images(params)
#
# Scrapes the given URL into LLM usable Markdown.
#
- # @overload web_scrape_md(url:, include_images: nil, include_links: nil, max_age_ms: nil, shorten_base64_images: nil, use_main_content_only: nil, request_options: {})
+ # @overload web_scrape_md(url:, include_images: nil, include_links: nil, max_age_ms: nil, parse_pdf: nil, shorten_base64_images: nil, use_main_content_only: nil, request_options: {})
#
# @param url [String] Full URL to scrape into LLM usable Markdown (must include http:// or https:// pr
#
@@ -211,6 +215,8 @@ def web_scrape_images(params)
#
# @param max_age_ms [Integer] Return a cached result if a prior scrape for the same parameters exists and is y
#
+ # @param parse_pdf [Boolean] When true (default), PDF URLs are fetched and their text layer is extracted and
+ #
# @param shorten_base64_images [Boolean] Shorten base64-encoded image data in the Markdown output
#
# @param use_main_content_only [Boolean] Extract only the main content of the page, excluding headers, footers, sidebars,
@@ -230,6 +236,7 @@ def web_scrape_md(params)
include_images: "includeImages",
include_links: "includeLinks",
max_age_ms: "maxAgeMs",
+ parse_pdf: "parsePDF",
shorten_base64_images: "shortenBase64Images",
use_main_content_only: "useMainContentOnly"
),
diff --git a/lib/context_dev/version.rb b/lib/context_dev/version.rb
index cf5ea5c9..4d8de878 100644
--- a/lib/context_dev/version.rb
+++ b/lib/context_dev/version.rb
@@ -1,5 +1,5 @@
# frozen_string_literal: true
module ContextDev
- VERSION = "1.6.0"
+ VERSION = "1.7.0"
end
diff --git a/rbi/context_dev/models/web_web_crawl_md_params.rbi b/rbi/context_dev/models/web_web_crawl_md_params.rbi
index dd4489d1..40bbbea1 100644
--- a/rbi/context_dev/models/web_web_crawl_md_params.rbi
+++ b/rbi/context_dev/models/web_web_crawl_md_params.rbi
@@ -61,6 +61,15 @@ module ContextDev
sig { params(max_pages: Integer).void }
attr_writer :max_pages
+ # When true (default), PDF pages are fetched and their text layer is extracted and
+ # converted to Markdown alongside HTML pages. When false, PDF pages are skipped
+ # entirely (not included in results and not counted as failures).
+ sig { returns(T.nilable(T::Boolean)) }
+ attr_reader :parse_pdf
+
+ sig { params(parse_pdf: T::Boolean).void }
+ attr_writer :parse_pdf
+
# Truncate base64-encoded image data in the Markdown output
sig { returns(T.nilable(T::Boolean)) }
attr_reader :shorten_base64_images
@@ -92,6 +101,7 @@ module ContextDev
max_age_ms: Integer,
max_depth: Integer,
max_pages: Integer,
+ parse_pdf: T::Boolean,
shorten_base64_images: T::Boolean,
url_regex: String,
use_main_content_only: T::Boolean,
@@ -117,6 +127,10 @@ module ContextDev
max_depth: nil,
# Maximum number of pages to crawl. Hard cap: 500.
max_pages: nil,
+ # When true (default), PDF pages are fetched and their text layer is extracted and
+ # converted to Markdown alongside HTML pages. When false, PDF pages are skipped
+ # entirely (not included in results and not counted as failures).
+ parse_pdf: nil,
# Truncate base64-encoded image data in the Markdown output
shorten_base64_images: nil,
# Regex pattern. Only URLs matching this pattern will be followed and scraped.
@@ -138,6 +152,7 @@ module ContextDev
max_age_ms: Integer,
max_depth: Integer,
max_pages: Integer,
+ parse_pdf: T::Boolean,
shorten_base64_images: T::Boolean,
url_regex: String,
use_main_content_only: T::Boolean,
diff --git a/rbi/context_dev/models/web_web_crawl_md_response.rbi b/rbi/context_dev/models/web_web_crawl_md_response.rbi
index 59c19a19..c7254fcd 100644
--- a/rbi/context_dev/models/web_web_crawl_md_response.rbi
+++ b/rbi/context_dev/models/web_web_crawl_md_response.rbi
@@ -64,6 +64,10 @@ module ContextDev
sig { returns(Integer) }
attr_accessor :num_failed
+ # Number of URLs skipped (PDFs when parsePDF=false, or URLs not matching urlRegex)
+ sig { returns(Integer) }
+ attr_accessor :num_skipped
+
# Number of pages successfully crawled
sig { returns(Integer) }
attr_accessor :num_succeeded
@@ -76,6 +80,7 @@ module ContextDev
params(
max_crawl_depth: Integer,
num_failed: Integer,
+ num_skipped: Integer,
num_succeeded: Integer,
num_urls: Integer
).returns(T.attached_class)
@@ -85,6 +90,8 @@ module ContextDev
max_crawl_depth:,
# Number of pages that failed to crawl
num_failed:,
+ # Number of URLs skipped (PDFs when parsePDF=false, or URLs not matching urlRegex)
+ num_skipped:,
# Number of pages successfully crawled
num_succeeded:,
# Total number of URLs crawled
@@ -97,6 +104,7 @@ module ContextDev
{
max_crawl_depth: Integer,
num_failed: Integer,
+ num_skipped: Integer,
num_succeeded: Integer,
num_urls: Integer
}
diff --git a/rbi/context_dev/models/web_web_scrape_html_params.rbi b/rbi/context_dev/models/web_web_scrape_html_params.rbi
index 781570fd..758b673d 100644
--- a/rbi/context_dev/models/web_web_scrape_html_params.rbi
+++ b/rbi/context_dev/models/web_web_scrape_html_params.rbi
@@ -27,10 +27,20 @@ module ContextDev
sig { params(max_age_ms: Integer).void }
attr_writer :max_age_ms
+ # When true (default), PDF URLs are fetched and their text layer is extracted and
+ # returned wrapped in …. When false, PDF URLs are skipped
+ # and a 400 WEBSITE_ACCESS_ERROR is returned.
+ sig { returns(T.nilable(T::Boolean)) }
+ attr_reader :parse_pdf
+
+ sig { params(parse_pdf: T::Boolean).void }
+ attr_writer :parse_pdf
+
sig do
params(
url: String,
max_age_ms: Integer,
+ parse_pdf: T::Boolean,
request_options: ContextDev::RequestOptions::OrHash
).returns(T.attached_class)
end
@@ -41,6 +51,10 @@ module ContextDev
# younger than this many milliseconds. Defaults to 1 day (86400000 ms) when
# omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh.
max_age_ms: nil,
+ # When true (default), PDF URLs are fetched and their text layer is extracted and
+ # returned wrapped in …. When false, PDF URLs are skipped
+ # and a 400 WEBSITE_ACCESS_ERROR is returned.
+ parse_pdf: nil,
request_options: {}
)
end
@@ -50,6 +64,7 @@ module ContextDev
{
url: String,
max_age_ms: Integer,
+ parse_pdf: T::Boolean,
request_options: ContextDev::RequestOptions
}
)
diff --git a/rbi/context_dev/models/web_web_scrape_md_params.rbi b/rbi/context_dev/models/web_web_scrape_md_params.rbi
index 3439d5c2..51bf3885 100644
--- a/rbi/context_dev/models/web_web_scrape_md_params.rbi
+++ b/rbi/context_dev/models/web_web_scrape_md_params.rbi
@@ -39,6 +39,15 @@ module ContextDev
sig { params(max_age_ms: Integer).void }
attr_writer :max_age_ms
+ # When true (default), PDF URLs are fetched and their text layer is extracted and
+ # converted to Markdown. When false, PDF URLs are skipped and a 400
+ # WEBSITE_ACCESS_ERROR is returned.
+ sig { returns(T.nilable(T::Boolean)) }
+ attr_reader :parse_pdf
+
+ sig { params(parse_pdf: T::Boolean).void }
+ attr_writer :parse_pdf
+
# Shorten base64-encoded image data in the Markdown output
sig { returns(T.nilable(T::Boolean)) }
attr_reader :shorten_base64_images
@@ -60,6 +69,7 @@ module ContextDev
include_images: T::Boolean,
include_links: T::Boolean,
max_age_ms: Integer,
+ parse_pdf: T::Boolean,
shorten_base64_images: T::Boolean,
use_main_content_only: T::Boolean,
request_options: ContextDev::RequestOptions::OrHash
@@ -77,6 +87,10 @@ module ContextDev
# younger than this many milliseconds. Defaults to 1 day (86400000 ms) when
# omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh.
max_age_ms: nil,
+ # When true (default), PDF URLs are fetched and their text layer is extracted and
+ # converted to Markdown. When false, PDF URLs are skipped and a 400
+ # WEBSITE_ACCESS_ERROR is returned.
+ parse_pdf: nil,
# Shorten base64-encoded image data in the Markdown output
shorten_base64_images: nil,
# Extract only the main content of the page, excluding headers, footers, sidebars,
@@ -93,6 +107,7 @@ module ContextDev
include_images: T::Boolean,
include_links: T::Boolean,
max_age_ms: Integer,
+ parse_pdf: T::Boolean,
shorten_base64_images: T::Boolean,
use_main_content_only: T::Boolean,
request_options: ContextDev::RequestOptions
diff --git a/rbi/context_dev/resources/web.rbi b/rbi/context_dev/resources/web.rbi
index 243474bb..8cd4f90e 100644
--- a/rbi/context_dev/resources/web.rbi
+++ b/rbi/context_dev/resources/web.rbi
@@ -108,6 +108,7 @@ module ContextDev
max_age_ms: Integer,
max_depth: Integer,
max_pages: Integer,
+ parse_pdf: T::Boolean,
shorten_base64_images: T::Boolean,
url_regex: String,
use_main_content_only: T::Boolean,
@@ -133,6 +134,10 @@ module ContextDev
max_depth: nil,
# Maximum number of pages to crawl. Hard cap: 500.
max_pages: nil,
+ # When true (default), PDF pages are fetched and their text layer is extracted and
+ # converted to Markdown alongside HTML pages. When false, PDF pages are skipped
+ # entirely (not included in results and not counted as failures).
+ parse_pdf: nil,
# Truncate base64-encoded image data in the Markdown output
shorten_base64_images: nil,
# Regex pattern. Only URLs matching this pattern will be followed and scraped.
@@ -149,6 +154,7 @@ module ContextDev
params(
url: String,
max_age_ms: Integer,
+ parse_pdf: T::Boolean,
request_options: ContextDev::RequestOptions::OrHash
).returns(ContextDev::Models::WebWebScrapeHTMLResponse)
end
@@ -159,6 +165,10 @@ module ContextDev
# younger than this many milliseconds. Defaults to 1 day (86400000 ms) when
# omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh.
max_age_ms: nil,
+ # When true (default), PDF URLs are fetched and their text layer is extracted and
+ # returned wrapped in …. When false, PDF URLs are skipped
+ # and a 400 WEBSITE_ACCESS_ERROR is returned.
+ parse_pdf: nil,
request_options: {}
)
end
@@ -186,6 +196,7 @@ module ContextDev
include_images: T::Boolean,
include_links: T::Boolean,
max_age_ms: Integer,
+ parse_pdf: T::Boolean,
shorten_base64_images: T::Boolean,
use_main_content_only: T::Boolean,
request_options: ContextDev::RequestOptions::OrHash
@@ -203,6 +214,10 @@ module ContextDev
# younger than this many milliseconds. Defaults to 1 day (86400000 ms) when
# omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh.
max_age_ms: nil,
+ # When true (default), PDF URLs are fetched and their text layer is extracted and
+ # converted to Markdown. When false, PDF URLs are skipped and a 400
+ # WEBSITE_ACCESS_ERROR is returned.
+ parse_pdf: nil,
# Shorten base64-encoded image data in the Markdown output
shorten_base64_images: nil,
# Extract only the main content of the page, excluding headers, footers, sidebars,
diff --git a/sig/context_dev/models/web_web_crawl_md_params.rbs b/sig/context_dev/models/web_web_crawl_md_params.rbs
index 258feea1..bceb32b2 100644
--- a/sig/context_dev/models/web_web_crawl_md_params.rbs
+++ b/sig/context_dev/models/web_web_crawl_md_params.rbs
@@ -9,6 +9,7 @@ module ContextDev
max_age_ms: Integer,
max_depth: Integer,
max_pages: Integer,
+ parse_pdf: bool,
:shorten_base64_images => bool,
url_regex: String,
use_main_content_only: bool
@@ -45,6 +46,10 @@ module ContextDev
def max_pages=: (Integer) -> Integer
+ attr_reader parse_pdf: bool?
+
+ def parse_pdf=: (bool) -> bool
+
attr_reader shorten_base64_images: bool?
def shorten_base64_images=: (bool) -> bool
@@ -65,6 +70,7 @@ module ContextDev
?max_age_ms: Integer,
?max_depth: Integer,
?max_pages: Integer,
+ ?parse_pdf: bool,
?shorten_base64_images: bool,
?url_regex: String,
?use_main_content_only: bool,
@@ -79,6 +85,7 @@ module ContextDev
max_age_ms: Integer,
max_depth: Integer,
max_pages: Integer,
+ parse_pdf: bool,
:shorten_base64_images => bool,
url_regex: String,
use_main_content_only: bool,
diff --git a/sig/context_dev/models/web_web_crawl_md_response.rbs b/sig/context_dev/models/web_web_crawl_md_response.rbs
index a290b1e3..e689febc 100644
--- a/sig/context_dev/models/web_web_crawl_md_response.rbs
+++ b/sig/context_dev/models/web_web_crawl_md_response.rbs
@@ -25,6 +25,7 @@ module ContextDev
{
max_crawl_depth: Integer,
num_failed: Integer,
+ num_skipped: Integer,
num_succeeded: Integer,
num_urls: Integer
}
@@ -34,6 +35,8 @@ module ContextDev
attr_accessor num_failed: Integer
+ attr_accessor num_skipped: Integer
+
attr_accessor num_succeeded: Integer
attr_accessor num_urls: Integer
@@ -41,6 +44,7 @@ module ContextDev
def initialize: (
max_crawl_depth: Integer,
num_failed: Integer,
+ num_skipped: Integer,
num_succeeded: Integer,
num_urls: Integer
) -> void
@@ -48,6 +52,7 @@ module ContextDev
def to_hash: -> {
max_crawl_depth: Integer,
num_failed: Integer,
+ num_skipped: Integer,
num_succeeded: Integer,
num_urls: Integer
}
diff --git a/sig/context_dev/models/web_web_scrape_html_params.rbs b/sig/context_dev/models/web_web_scrape_html_params.rbs
index 6140c451..648c5dac 100644
--- a/sig/context_dev/models/web_web_scrape_html_params.rbs
+++ b/sig/context_dev/models/web_web_scrape_html_params.rbs
@@ -1,7 +1,7 @@
module ContextDev
module Models
type web_web_scrape_html_params =
- { url: String, max_age_ms: Integer }
+ { url: String, max_age_ms: Integer, parse_pdf: bool }
& ContextDev::Internal::Type::request_parameters
class WebWebScrapeHTMLParams < ContextDev::Internal::Type::BaseModel
@@ -14,15 +14,21 @@ module ContextDev
def max_age_ms=: (Integer) -> Integer
+ attr_reader parse_pdf: bool?
+
+ def parse_pdf=: (bool) -> bool
+
def initialize: (
url: String,
?max_age_ms: Integer,
+ ?parse_pdf: bool,
?request_options: ContextDev::request_opts
) -> void
def to_hash: -> {
url: String,
max_age_ms: Integer,
+ parse_pdf: bool,
request_options: ContextDev::RequestOptions
}
end
diff --git a/sig/context_dev/models/web_web_scrape_md_params.rbs b/sig/context_dev/models/web_web_scrape_md_params.rbs
index c9fe083f..8ae2b1b9 100644
--- a/sig/context_dev/models/web_web_scrape_md_params.rbs
+++ b/sig/context_dev/models/web_web_scrape_md_params.rbs
@@ -6,6 +6,7 @@ module ContextDev
include_images: bool,
include_links: bool,
max_age_ms: Integer,
+ parse_pdf: bool,
:shorten_base64_images => bool,
use_main_content_only: bool
}
@@ -29,6 +30,10 @@ module ContextDev
def max_age_ms=: (Integer) -> Integer
+ attr_reader parse_pdf: bool?
+
+ def parse_pdf=: (bool) -> bool
+
attr_reader shorten_base64_images: bool?
def shorten_base64_images=: (bool) -> bool
@@ -42,6 +47,7 @@ module ContextDev
?include_images: bool,
?include_links: bool,
?max_age_ms: Integer,
+ ?parse_pdf: bool,
?shorten_base64_images: bool,
?use_main_content_only: bool,
?request_options: ContextDev::request_opts
@@ -52,6 +58,7 @@ module ContextDev
include_images: bool,
include_links: bool,
max_age_ms: Integer,
+ parse_pdf: bool,
:shorten_base64_images => bool,
use_main_content_only: bool,
request_options: ContextDev::RequestOptions
diff --git a/sig/context_dev/resources/web.rbs b/sig/context_dev/resources/web.rbs
index 404f8c78..c8c9a34c 100644
--- a/sig/context_dev/resources/web.rbs
+++ b/sig/context_dev/resources/web.rbs
@@ -32,6 +32,7 @@ module ContextDev
?max_age_ms: Integer,
?max_depth: Integer,
?max_pages: Integer,
+ ?parse_pdf: bool,
?shorten_base64_images: bool,
?url_regex: String,
?use_main_content_only: bool,
@@ -41,6 +42,7 @@ module ContextDev
def web_scrape_html: (
url: String,
?max_age_ms: Integer,
+ ?parse_pdf: bool,
?request_options: ContextDev::request_opts
) -> ContextDev::Models::WebWebScrapeHTMLResponse
@@ -54,6 +56,7 @@ module ContextDev
?include_images: bool,
?include_links: bool,
?max_age_ms: Integer,
+ ?parse_pdf: bool,
?shorten_base64_images: bool,
?use_main_content_only: bool,
?request_options: ContextDev::request_opts