Skip to content

Commit 78920ab

Browse files
committed
docs: demonstrate scraping formats throughout README
1 parent a11a64a commit 78920ab

1 file changed

Lines changed: 74 additions & 38 deletions

File tree

‎README.md‎

Lines changed: 74 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -18,21 +18,23 @@ pip install context.dev
1818

1919
## Usage
2020

21+
Set `CONTEXT_DEV_API_KEY` to your API key; the client reads it automatically.
22+
23+
### Scrape markdown and HTML
24+
2125
The full API of this library can be found in [api.md](api.md).
2226

2327
```python
24-
import os
2528
from context.dev import ContextDev
2629

27-
client = ContextDev(
28-
api_key=os.environ.get("CONTEXT_DEV_API_KEY"), # This is the default and can be omitted
29-
)
30+
client = ContextDev()
3031

31-
brand = client.brand.retrieve(
32-
domain="REPLACE_ME",
33-
type="by_domain",
32+
page = client.web.scrape(
33+
url="https://example.com",
34+
formats={"markdown": True, "html": True},
3435
)
35-
print(brand.request_id)
36+
print(page.markdown.data)
37+
print(page.html.data)
3638
```
3739

3840
### Extract structured JSON
@@ -60,11 +62,44 @@ page = client.web.scrape(
6062
print(page.json_.data)
6163
```
6264

65+
### Extract relevant highlights
66+
67+
Return the passages that answer a question about the page.
68+
69+
```python
70+
from context.dev import ContextDev
71+
72+
client = ContextDev()
73+
74+
page = client.web.scrape(
75+
url="https://example.com",
76+
formats={"highlights": True},
77+
highlights_params={"query": "What is this domain used for?"},
78+
)
79+
print(page.highlights.data)
80+
```
81+
82+
### Take a screenshot
83+
84+
The screenshot is returned as a base64 image data URL.
85+
86+
```python
87+
from context.dev import ContextDev
88+
89+
client = ContextDev()
90+
91+
page = client.web.scrape(
92+
url="https://example.com",
93+
formats={"screenshot": True},
94+
)
95+
print(page.screenshot.data)
96+
```
97+
6398
## What you can do
6499

65100
| Task | Method |
66101
| --- | --- |
67-
| Scrape a URL to markdown, HTML, JSON or a screenshot | `client.web.scrape` |
102+
| Scrape a URL to markdown, HTML, JSON, highlights or a screenshot | `client.web.scrape` |
68103
| Crawl a site and get every page as markdown | `client.web.web_crawl_md` |
69104
| Map every URL on a domain | `client.web.map_urls` |
70105
| Search the web | `client.web.search` |
@@ -98,11 +133,11 @@ client = AsyncContextDev(
98133

99134

100135
async def main() -> None:
101-
brand = await client.brand.retrieve(
102-
domain="REPLACE_ME",
103-
type="by_domain",
136+
page = await client.web.scrape(
137+
url="https://example.com",
138+
formats={"markdown": True},
104139
)
105-
print(brand.request_id)
140+
print(page.markdown.data)
106141

107142

108143
asyncio.run(main())
@@ -135,11 +170,11 @@ async def main() -> None:
135170
api_key=os.environ.get("CONTEXT_DEV_API_KEY"), # This is the default and can be omitted
136171
http_client=DefaultAioHttpClient(),
137172
) as client:
138-
brand = await client.brand.retrieve(
139-
domain="REPLACE_ME",
140-
type="by_domain",
173+
page = await client.web.scrape(
174+
url="https://example.com",
175+
formats={"markdown": True},
141176
)
142-
print(brand.request_id)
177+
print(page.markdown.data)
143178

144179

145180
asyncio.run(main())
@@ -163,12 +198,12 @@ from context.dev import ContextDev
163198

164199
client = ContextDev()
165200

166-
brand = client.brand.retrieve(
167-
domain="xxx",
168-
type="by_domain",
201+
page = client.web.scrape(
202+
url="https://example.com",
203+
formats={"markdown": True},
169204
timeout_opts={"milliseconds": 1000},
170205
)
171-
print(brand.timeout_opts)
206+
print(page.markdown.data)
172207
```
173208

174209
## Handling errors
@@ -187,9 +222,9 @@ from context.dev import ContextDev
187222
client = ContextDev()
188223

189224
try:
190-
client.brand.retrieve(
191-
domain="REPLACE_ME",
192-
type="by_domain",
225+
client.web.scrape(
226+
url="https://example.com",
227+
formats={"markdown": True},
193228
)
194229
except context.dev.APIConnectionError as e:
195230
print("The server could not be reached")
@@ -233,9 +268,9 @@ client = ContextDev(
233268
)
234269

235270
# Or, configure per-request:
236-
client.with_options(max_retries=5).brand.retrieve(
237-
domain="REPLACE_ME",
238-
type="by_domain",
271+
client.with_options(max_retries=5).web.scrape(
272+
url="https://example.com",
273+
formats={"markdown": True},
239274
)
240275
```
241276

@@ -245,6 +280,7 @@ By default requests time out after 1 minute. You can configure this with a `time
245280
which accepts a float or an [`httpx.Timeout`](https://www.python-httpx.org/advanced/timeouts/#fine-tuning-the-configuration) object:
246281

247282
```python
283+
import httpx
248284
from context.dev import ContextDev
249285

250286
# Configure the default for all requests:
@@ -259,9 +295,9 @@ client = ContextDev(
259295
)
260296

261297
# Override per-request:
262-
client.with_options(timeout=5.0).brand.retrieve(
263-
domain="REPLACE_ME",
264-
type="by_domain",
298+
client.with_options(timeout=5.0).web.scrape(
299+
url="https://example.com",
300+
formats={"markdown": True},
265301
)
266302
```
267303

@@ -303,14 +339,14 @@ The "raw" Response object can be accessed by prefixing `.with_raw_response.` to
303339
from context.dev import ContextDev
304340

305341
client = ContextDev()
306-
response = client.brand.with_raw_response.retrieve(
307-
domain="REPLACE_ME",
308-
type="by_domain",
342+
response = client.web.with_raw_response.scrape(
343+
url="https://example.com",
344+
formats={"markdown": True},
309345
)
310346
print(response.headers.get('X-My-Header'))
311347

312-
brand = response.parse() # get the object that `brand.retrieve()` would have returned
313-
print(brand.request_id)
348+
page = response.parse()
349+
print(page.markdown.data)
314350
```
315351

316352
These methods return an [`APIResponse`](https://github.com/context-dot-dev/context-python-sdk/tree/main/src/context/dev/_response.py) object.
@@ -324,9 +360,9 @@ The above interface eagerly reads the full response body when you make the reque
324360
To stream the response body, use `.with_streaming_response` instead, which requires a context manager and only reads the response body once you call `.read()`, `.text()`, `.json()`, `.iter_bytes()`, `.iter_text()`, `.iter_lines()` or `.parse()`. In the async client, these are async methods.
325361

326362
```python
327-
with client.brand.with_streaming_response.retrieve(
328-
domain="REPLACE_ME",
329-
type="by_domain",
363+
with client.web.with_streaming_response.scrape(
364+
url="https://example.com",
365+
formats={"markdown": True},
330366
) as response:
331367
print(response.headers.get("X-My-Header"))
332368

0 commit comments

Comments
 (0)