support api key

Author: ChinchillaLover9000

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "decodo-sdk"
-version = "2.2.0"
+version = "2.3.0"
 description = "Official Python SDK for the Decodo APIs"
 readme = "README.md"
 license = { file = "LICENSE" }

src/decodo/__init__.py

@@ -1,4 +1,4 @@
-from .api.web_scraping_api import WebScrapingApi
+from .api.web_scraping_api import WebScrapingApi, WebScrapingApiRoutes
 from .client import DecodoClient, DecodoConfig, WebScrapingApiConfig
 from .errors import (
     AuthenticationError,
@@ -268,6 +268,7 @@ def __getattr__(name: str) -> object:
     "DecodoConfig",
     "WebScrapingApiConfig",
     "WebScrapingApi",
+    "WebScrapingApiRoutes",
     "BundledSchema",
     "RemoteSchema",
     "DecodoSchema",

src/decodo/api/web_scraping_api.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from collections.abc import Mapping
+from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, cast
 
 import jsonschema
@@ -22,16 +23,32 @@
 )
 
 
+@dataclass(frozen=True)
+class WebScrapingApiRoutes:
+    scrape: str
+    task: str
+
+
+SCRAPER_API_ROUTES = WebScrapingApiRoutes(scrape="/v2/scrape", task="/v3/task")
+DATA_API_ROUTES = WebScrapingApiRoutes(scrape="/v1/scrape", task="/v1/task")
+
+
 def _to_payload(params: ScrapeRequest | BatchRequest | Mapping[str, Any]) -> dict[str, Any]:
     if isinstance(params, BaseModel):
         return params.model_dump(by_alias=True, exclude_none=True, mode="json")
     return dict(params)
 
 
 class WebScrapingApi:
-    def __init__(self, http: HttpClient, schema: DecodoSchema = BundledSchema.shared) -> None:
+    def __init__(
+        self,
+        http: HttpClient,
+        schema: DecodoSchema = BundledSchema.shared,
+        routes: WebScrapingApiRoutes = SCRAPER_API_ROUTES,
+    ) -> None:
         self._http = http
         self._schema = schema
+        self._routes = routes
 
     def _validate(self, payload: dict[str, Any]) -> None:
         if self._schema is None:
@@ -55,20 +72,20 @@ def _validate(self, payload: dict[str, Any]) -> None:
     def scrape(self, params: ScrapeRequest | Mapping[str, Any]) -> SyncResponse:
         payload = _to_payload(params)
         self._validate(payload)
-        return cast(SyncResponse, self._http.post("/v2/scrape", payload))
+        return cast(SyncResponse, self._http.post(self._routes.scrape, payload))
 
     def scrape_async(self, params: ScrapeRequest | Mapping[str, Any]) -> AsyncTaskResponse:
         payload = _to_payload(params)
         self._validate(payload)
-        return cast(AsyncTaskResponse, self._http.post("/v3/task", payload))
+        return cast(AsyncTaskResponse, self._http.post(self._routes.task, payload))
 
     def scrape_batch(self, params: BatchRequest | Mapping[str, Any]) -> BatchResponse:
         payload = _to_payload(params)
-        return cast(BatchResponse, self._http.post("/v3/task/batch", payload))
+        return cast(BatchResponse, self._http.post(f"{self._routes.task}/batch", payload))
 
     def get_status(self, task_id: str) -> TaskMetadata:
-        return cast(TaskMetadata, self._http.get(f"/v3/task/{task_id}"))
+        return cast(TaskMetadata, self._http.get(f"{self._routes.task}/{task_id}"))
 
     def get_results(self, task_id: str) -> TaskResultsResponse | None:
-        result = self._http.get(f"/v3/task/{task_id}/results")
+        result = self._http.get(f"{self._routes.task}/{task_id}/results")
         return cast(TaskResultsResponse, result) if result is not None else None

src/decodo/client.py

@@ -3,18 +3,25 @@
 from dataclasses import dataclass
 from typing import Any
 
-from .api.web_scraping_api import WebScrapingApi
-from .http import BasicAuth, HttpClient, HttpClientConfig
+from .api.web_scraping_api import (
+    DATA_API_ROUTES,
+    SCRAPER_API_ROUTES,
+    WebScrapingApi,
+    WebScrapingApiRoutes,
+)
+from .http import ApiKeyAuth, BasicAuth, HttpClient, HttpClientConfig
 from .schema.bundled_schema import BundledSchema
 from .schema.types import DecodoSchema
 
 WEB_API_BASE_URL = "https://scraper-api.decodo.com"
+DATA_API_BASE_URL = "https://data.decodo.com"
 DEFAULT_TIMEOUT_MS = 180_000
 
 
 @dataclass
 class WebScrapingApiConfig:
-    token: str
+    token: str | None = None
+    api_key: str | None = None
     integration_header: str | None = None
 
 
@@ -25,6 +32,43 @@ class DecodoConfig:
     schema: DecodoSchema | None = None
 
 
+@dataclass(frozen=True)
+class _Transport:
+    base_url: str
+    auth: BasicAuth | ApiKeyAuth
+    routes: WebScrapingApiRoutes
+
+
+def _present(value: str | None) -> str | None:
+    return value if value is not None and value.strip() else None
+
+
+def _resolve_transport(config: WebScrapingApiConfig) -> _Transport:
+    token = _present(config.token)
+    api_key = _present(config.api_key)
+
+    if token is not None and api_key is not None:
+        raise ValueError(
+            "web_scraping_api accepts either token or api_key, not both. Provide only one."
+        )
+
+    if api_key is not None:
+        return _Transport(
+            base_url=DATA_API_BASE_URL,
+            auth=ApiKeyAuth(type="apiKey", api_key=api_key),
+            routes=DATA_API_ROUTES,
+        )
+
+    if token is not None:
+        return _Transport(
+            base_url=WEB_API_BASE_URL,
+            auth=BasicAuth(type="basic", token=token),
+            routes=SCRAPER_API_ROUTES,
+        )
+
+    raise ValueError("web_scraping_api requires token in DecodoConfig.")
+
+
 def _not_configured(namespace: str, hint: str) -> Any:
     raise RuntimeError(f"{namespace} is not configured. {hint}")
 
@@ -45,14 +89,15 @@ def __init__(self, config: DecodoConfig) -> None:
         schema: DecodoSchema = config.schema if config.schema is not None else BundledSchema.shared
 
         if config.web_scraping_api is not None:
+            transport = _resolve_transport(config.web_scraping_api)
             http = HttpClient(
                 HttpClientConfig(
-                    base_url=WEB_API_BASE_URL,
-                    auth=BasicAuth(type="basic", token=config.web_scraping_api.token),
+                    base_url=transport.base_url,
+                    auth=transport.auth,
                     timeout_ms=timeout_ms,
                     integration_header=config.web_scraping_api.integration_header,
                 )
             )
-            self.web_scraping_api = WebScrapingApi(http, schema)
+            self.web_scraping_api = WebScrapingApi(http, schema, transport.routes)
         else:
             self.web_scraping_api = _UnconfiguredWebScrapingApi()  # type: ignore[assignment]

src/decodo/http.py

@@ -46,7 +46,7 @@ def __init__(self, config: HttpClientConfig) -> None:
             self._auth_header = f"Basic {config.auth.token}"
         else:
             assert isinstance(config.auth, ApiKeyAuth)
-            self._auth_header = config.auth.api_key
+            self._auth_header = f"Bearer {config.auth.api_key}"
 
     def request(self, method: str, path: str, body: Any = None) -> Any:
         url = f"{self._base_url}{path}"

tests/test_client.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+from typing import Any
+from unittest.mock import MagicMock, patch
+
+import httpx
+import pytest
+
+from decodo.client import DecodoClient, DecodoConfig, WebScrapingApiConfig
+
+SCRAPE_PARAMS = {"target": "universal", "url": "https://example.com"}
+
+
+def _make_response(body: Any, status: int = 200) -> MagicMock:
+    mock = MagicMock(spec=httpx.Response)
+    mock.status_code = status
+    mock.is_success = 200 <= status < 300
+    mock.json.return_value = body
+    return mock
+
+
+class TestTransportSelection:
+    def _collect_urls(self, config: WebScrapingApiConfig) -> list[str]:
+        urls: list[str] = []
+
+        def fake_request(method: str, url: str, **kwargs: Any) -> MagicMock:
+            urls.append(url)
+            return _make_response({"results": []})
+
+        with patch("httpx.request", side_effect=fake_request):
+            client = DecodoClient(DecodoConfig(web_scraping_api=config))
+            client.web_scraping_api.scrape(SCRAPE_PARAMS)
+            client.web_scraping_api.get_status("task-1")
+
+        return urls
+
+    def test_routes_a_token_to_scraper_api_over_v2_v3(self) -> None:
+        assert self._collect_urls(WebScrapingApiConfig(token="test-token")) == [
+            "https://scraper-api.decodo.com/v2/scrape",
+            "https://scraper-api.decodo.com/v3/task/task-1",
+        ]
+
+    def test_routes_an_api_key_to_data_decodo_com_over_v1(self) -> None:
+        assert self._collect_urls(WebScrapingApiConfig(api_key="test-key")) == [
+            "https://data.decodo.com/v1/scrape",
+            "https://data.decodo.com/v1/task/task-1",
+        ]
+
+    def test_raises_when_both_token_and_api_key_are_provided(self) -> None:
+        with pytest.raises(ValueError, match="either token or api_key"):
+            DecodoClient(
+                DecodoConfig(
+                    web_scraping_api=WebScrapingApiConfig(
+                        token="test-token", api_key="test-key"
+                    )
+                )
+            )
+
+    def test_raises_when_neither_token_nor_api_key_is_provided(self) -> None:
+        with pytest.raises(ValueError, match="web_scraping_api requires"):
+            DecodoClient(DecodoConfig(web_scraping_api=WebScrapingApiConfig()))
+
+    @pytest.mark.parametrize("blank", ["", "   "])
+    def test_raises_when_the_only_credential_is_blank(self, blank: str) -> None:
+        with pytest.raises(ValueError, match="web_scraping_api requires"):
+            DecodoClient(DecodoConfig(web_scraping_api=WebScrapingApiConfig(token=blank)))
+        with pytest.raises(ValueError, match="web_scraping_api requires"):
+            DecodoClient(DecodoConfig(web_scraping_api=WebScrapingApiConfig(api_key=blank)))
+
+    @pytest.mark.parametrize("blank", ["", "   "])
+    def test_blank_credential_does_not_shadow_the_other_one(self, blank: str) -> None:
+        assert self._collect_urls(WebScrapingApiConfig(token=blank, api_key="test-key")) == [
+            "https://data.decodo.com/v1/scrape",
+            "https://data.decodo.com/v1/task/task-1",
+        ]
+        assert self._collect_urls(WebScrapingApiConfig(token="test-token", api_key=blank)) == [
+            "https://scraper-api.decodo.com/v2/scrape",
+            "https://scraper-api.decodo.com/v3/task/task-1",
+        ]

tests/test_http.py

@@ -5,7 +5,7 @@
 
 import httpx
 
-from decodo.http import BasicAuth, HttpClient, HttpClientConfig
+from decodo.http import ApiKeyAuth, BasicAuth, HttpClient, HttpClientConfig
 
 BASE_CONFIG = HttpClientConfig(
     base_url="https://api.test",
@@ -55,3 +55,33 @@ def fake_request(method: str, url: str, **kwargs: Any) -> MagicMock:
             client.get("/v3/task/1")
 
         assert captured["headers"]["x-integration"] == "cli"
+
+
+class TestHttpClientAuthHeader:
+    def _captured_headers(self, config: HttpClientConfig) -> dict[str, str]:
+        captured: dict[str, dict[str, str]] = {}
+
+        def fake_request(method: str, url: str, **kwargs: Any) -> MagicMock:
+            captured["headers"] = kwargs.get("headers", {})
+            return _make_response({"ok": True})
+
+        with patch("httpx.request", side_effect=fake_request):
+            HttpClient(config).get("/v3/task/1")
+
+        return captured["headers"]
+
+    def test_sends_basic_auth_for_a_token(self) -> None:
+        headers = self._captured_headers(BASE_CONFIG)
+
+        assert headers["Authorization"] == "Basic test-token"
+
+    def test_sends_bearer_auth_for_an_api_key(self) -> None:
+        headers = self._captured_headers(
+            HttpClientConfig(
+                base_url="https://api.test",
+                auth=ApiKeyAuth(type="apiKey", api_key="test-key"),
+                timeout_ms=5000,
+            )
+        )
+
+        assert headers["Authorization"] == "Bearer test-key"