@@ -18,21 +18,23 @@ pip install context.dev
1818
1919## Usage
2020
21+ Set ` CONTEXT_DEV_API_KEY ` to your API key; the client reads it automatically.
22+
23+ ### Scrape markdown and HTML
24+
2125The full API of this library can be found in [ api.md] ( api.md ) .
2226
2327``` python
24- import os
2528from context.dev import ContextDev
2629
27- client = ContextDev(
28- api_key = os.environ.get(" CONTEXT_DEV_API_KEY" ), # This is the default and can be omitted
29- )
30+ client = ContextDev()
3031
31- brand = client.brand.retrieve (
32- domain = " REPLACE_ME " ,
33- type = " by_domain " ,
32+ page = client.web.scrape (
33+ url = " https://example.com " ,
34+ formats = { " markdown " : True , " html " : True } ,
3435)
35- print (brand.request_id)
36+ print (page.markdown.data)
37+ print (page.html.data)
3638```
3739
3840### Extract structured JSON
@@ -60,11 +62,44 @@ page = client.web.scrape(
6062print (page.json_.data)
6163```
6264
65+ ### Extract relevant highlights
66+
67+ Return the passages that answer a question about the page.
68+
69+ ``` python
70+ from context.dev import ContextDev
71+
72+ client = ContextDev()
73+
74+ page = client.web.scrape(
75+ url = " https://example.com" ,
76+ formats = {" highlights" : True },
77+ highlights_params = {" query" : " What is this domain used for?" },
78+ )
79+ print (page.highlights.data)
80+ ```
81+
82+ ### Take a screenshot
83+
84+ The screenshot is returned as a base64 image data URL.
85+
86+ ``` python
87+ from context.dev import ContextDev
88+
89+ client = ContextDev()
90+
91+ page = client.web.scrape(
92+ url = " https://example.com" ,
93+ formats = {" screenshot" : True },
94+ )
95+ print (page.screenshot.data)
96+ ```
97+
6398## What you can do
6499
65100| Task | Method |
66101| --- | --- |
67- | Scrape a URL to markdown, HTML, JSON or a screenshot | ` client.web.scrape ` |
102+ | Scrape a URL to markdown, HTML, JSON, highlights or a screenshot | ` client.web.scrape ` |
68103| Crawl a site and get every page as markdown | ` client.web.web_crawl_md ` |
69104| Map every URL on a domain | ` client.web.map_urls ` |
70105| Search the web | ` client.web.search ` |
@@ -98,11 +133,11 @@ client = AsyncContextDev(
98133
99134
100135async def main () -> None :
101- brand = await client.brand.retrieve (
102- domain = " REPLACE_ME " ,
103- type = " by_domain " ,
136+ page = await client.web.scrape (
137+ url = " https://example.com " ,
138+ formats = { " markdown " : True } ,
104139 )
105- print (brand.request_id )
140+ print (page.markdown.data )
106141
107142
108143asyncio.run(main())
@@ -135,11 +170,11 @@ async def main() -> None:
135170 api_key = os.environ.get(" CONTEXT_DEV_API_KEY" ), # This is the default and can be omitted
136171 http_client = DefaultAioHttpClient(),
137172 ) as client:
138- brand = await client.brand.retrieve (
139- domain = " REPLACE_ME " ,
140- type = " by_domain " ,
173+ page = await client.web.scrape (
174+ url = " https://example.com " ,
175+ formats = { " markdown " : True } ,
141176 )
142- print (brand.request_id )
177+ print (page.markdown.data )
143178
144179
145180asyncio.run(main())
@@ -163,12 +198,12 @@ from context.dev import ContextDev
163198
164199client = ContextDev()
165200
166- brand = client.brand.retrieve (
167- domain = " xxx " ,
168- type = " by_domain " ,
201+ page = client.web.scrape (
202+ url = " https://example.com " ,
203+ formats = { " markdown " : True } ,
169204 timeout_opts = {" milliseconds" : 1000 },
170205)
171- print (brand.timeout_opts )
206+ print (page.markdown.data )
172207```
173208
174209## Handling errors
@@ -187,9 +222,9 @@ from context.dev import ContextDev
187222client = ContextDev()
188223
189224try :
190- client.brand.retrieve (
191- domain = " REPLACE_ME " ,
192- type = " by_domain " ,
225+ client.web.scrape (
226+ url = " https://example.com " ,
227+ formats = { " markdown " : True } ,
193228 )
194229except context.dev.APIConnectionError as e:
195230 print (" The server could not be reached" )
@@ -233,9 +268,9 @@ client = ContextDev(
233268)
234269
235270# Or, configure per-request:
236- client.with_options(max_retries = 5 ).brand.retrieve (
237- domain = " REPLACE_ME " ,
238- type = " by_domain " ,
271+ client.with_options(max_retries = 5 ).web.scrape (
272+ url = " https://example.com " ,
273+ formats = { " markdown " : True } ,
239274)
240275```
241276
@@ -245,6 +280,7 @@ By default requests time out after 1 minute. You can configure this with a `time
245280which accepts a float or an [ ` httpx.Timeout ` ] ( https://www.python-httpx.org/advanced/timeouts/#fine-tuning-the-configuration ) object:
246281
247282``` python
283+ import httpx
248284from context.dev import ContextDev
249285
250286# Configure the default for all requests:
@@ -259,9 +295,9 @@ client = ContextDev(
259295)
260296
261297# Override per-request:
262- client.with_options(timeout = 5.0 ).brand.retrieve (
263- domain = " REPLACE_ME " ,
264- type = " by_domain " ,
298+ client.with_options(timeout = 5.0 ).web.scrape (
299+ url = " https://example.com " ,
300+ formats = { " markdown " : True } ,
265301)
266302```
267303
@@ -303,14 +339,14 @@ The "raw" Response object can be accessed by prefixing `.with_raw_response.` to
303339from context.dev import ContextDev
304340
305341client = ContextDev()
306- response = client.brand .with_raw_response.retrieve (
307- domain = " REPLACE_ME " ,
308- type = " by_domain " ,
342+ response = client.web .with_raw_response.scrape (
343+ url = " https://example.com " ,
344+ formats = { " markdown " : True } ,
309345)
310346print (response.headers.get(' X-My-Header' ))
311347
312- brand = response.parse() # get the object that `brand.retrieve()` would have returned
313- print (brand.request_id )
348+ page = response.parse()
349+ print (page.markdown.data )
314350```
315351
316352These methods return an [ ` APIResponse ` ] ( https://github.com/context-dot-dev/context-python-sdk/tree/main/src/context/dev/_response.py ) object.
@@ -324,9 +360,9 @@ The above interface eagerly reads the full response body when you make the reque
324360To stream the response body, use ` .with_streaming_response ` instead, which requires a context manager and only reads the response body once you call ` .read() ` , ` .text() ` , ` .json() ` , ` .iter_bytes() ` , ` .iter_text() ` , ` .iter_lines() ` or ` .parse() ` . In the async client, these are async methods.
325361
326362``` python
327- with client.brand .with_streaming_response.retrieve (
328- domain = " REPLACE_ME " ,
329- type = " by_domain " ,
363+ with client.web .with_streaming_response.scrape (
364+ url = " https://example.com " ,
365+ formats = { " markdown " : True } ,
330366) as response:
331367 print (response.headers.get(" X-My-Header" ))
332368
0 commit comments