diff --git a/scripts/_resolve_lib.py b/scripts/_resolve_lib.py index 059d2e0e1..17a013fb8 100644 --- a/scripts/_resolve_lib.py +++ b/scripts/_resolve_lib.py @@ -22,7 +22,14 @@ RepoInfo, RepoMetadata, ) -from ._utils import drop_falsy, flatten, pipe, unique_values_preserving_order, write_json +from ._utils import ( + drop_falsy, + flatten, + pipe, + unique_values_preserving_order, + write_json, + USER_AGENT, +) PYPI_BASE = "https://pypi.org/pypi/{}/json" @@ -619,7 +626,7 @@ async def _fetch_pypi_json( if fetched_at and now - fetched_at < ttl_seconds: return load_json(cache_path), "cache" - headers = {} + headers = {'User-Agent': USER_AGENT} if has_cache: if entry.get("etag"): headers["If-None-Match"] = entry["etag"] diff --git a/scripts/_utils.py b/scripts/_utils.py index 2ed1dd232..b727b8c4b 100644 --- a/scripts/_utils.py +++ b/scripts/_utils.py @@ -13,6 +13,9 @@ import inflect +USER_AGENT = "Mozilla/5.0 (thecrawl 1.0; +https://packages.sublimetext.io)" + + def err(*args, **kwargs): print(*args, **kwargs, file=sys.stderr) diff --git a/scripts/accumulate_stats.py b/scripts/accumulate_stats.py index f4cdaf83a..9b46827ec 100644 --- a/scripts/accumulate_stats.py +++ b/scripts/accumulate_stats.py @@ -8,7 +8,7 @@ from itertools import count, takewhile from urllib.request import Request, urlopen -from ._utils import write_json +from ._utils import write_json, USER_AGENT DEFAULT_OUTPUT_FILE = "stats.json" DEFAULT_URL = "https://stats.sublimetext.io/all-totals" @@ -121,7 +121,7 @@ def accumulate(value: int, container: dict, key: str, wanted_length: int, rollov def fetch_totals(url: str) -> dict: - request = Request(url, headers={"User-Agent": "Mozilla/5.0"}) + request = Request(url, headers={"User-Agent": USER_AGENT}) with urlopen(request) as resp: return json.load(resp) diff --git a/scripts/bitbucket.py b/scripts/bitbucket.py index ed96c597c..995d2c790 100644 --- a/scripts/bitbucket.py +++ b/scripts/bitbucket.py @@ -7,7 +7,7 @@ from typing import AsyncIterable, TypedDict, Literal, Iterable -from ._utils import drop_falsy, err, normalize_tz_aware_datetime +from ._utils import drop_falsy, err, normalize_tz_aware_datetime, USER_AGENT type QueryScope = Literal["METADATA", "TAGS", "BRANCHES"] @@ -62,7 +62,7 @@ class RepoInfo(TypedDict): async def fetch_json(session: aiohttp.ClientSession, url: str) -> dict: - headers = {} + headers = {'User-Agent': USER_AGENT} if token := os.getenv("BITBUCKET_TOKEN"): headers["Authorization"] = f"Bearer {token}" async with session.get(url, headers=headers) as resp: diff --git a/scripts/codeberg.py b/scripts/codeberg.py index f92d8cc28..9b26b4866 100644 --- a/scripts/codeberg.py +++ b/scripts/codeberg.py @@ -7,7 +7,7 @@ from typing import AsyncIterable, TypedDict, Literal, Iterable -from ._utils import drop_falsy, err, normalize_tz_aware_datetime +from ._utils import drop_falsy, err, normalize_tz_aware_datetime, USER_AGENT type QueryScope = Literal["METADATA", "TAGS", "BRANCHES"] @@ -71,7 +71,7 @@ def parse_owner_repo(url: str): def _auth_headers() -> dict[str, str]: - headers: dict[str, str] = {} + headers: dict[str, str] = {'User-Agent': USER_AGENT} # Codeberg (Forgejo/Gitea) supports Authorization: token if token := os.getenv("CODEBERG_TOKEN"): headers["Authorization"] = f"token {token}" diff --git a/scripts/compress_channel.py b/scripts/compress_channel.py index 7a4e03376..2d0149251 100644 --- a/scripts/compress_channel.py +++ b/scripts/compress_channel.py @@ -8,7 +8,7 @@ import os import sys -from ._utils import pl, write_json +from ._utils import pl, write_json, USER_AGENT NEW_CHANNEL = ( @@ -153,7 +153,7 @@ async def http_get_json(location: str, session: aiohttp.ClientSession) -> dict: async def http_get(location: str, session: aiohttp.ClientSession) -> str: headers = { - 'User-Agent': 'Mozilla/5.0', + 'User-Agent': USER_AGENT, 'Cache-Control': 'no-cache', 'Pragma': 'no-cache' } diff --git a/scripts/generate_registry.py b/scripts/generate_registry.py index 0918fc4d4..c2682ae19 100644 --- a/scripts/generate_registry.py +++ b/scripts/generate_registry.py @@ -12,7 +12,7 @@ from urllib.parse import urlparse from typing import Any, Callable, Iterable, Mapping, NotRequired, TypedDict, TypeGuard -from ._utils import flatten, pick, resolve_urls, update_url, write_json, pl +from ._utils import flatten, pick, resolve_urls, update_url, write_json, pl, USER_AGENT DEFAULT_OUTPUT_FILE = "./registry.json" @@ -337,7 +337,7 @@ async def http_get_json(location: str, session: aiohttp.ClientSession) -> dict: async def http_get(location: str, session: aiohttp.ClientSession) -> str: - headers = {'User-Agent': 'Mozilla/5.0'} + headers = {'User-Agent': USER_AGENT} async with session.get(location, headers=headers, raise_for_status=True) as resp: return await resp.text() diff --git a/scripts/github.py b/scripts/github.py index 90594feff..4ee3a52d1 100644 --- a/scripts/github.py +++ b/scripts/github.py @@ -11,7 +11,7 @@ from typing import AsyncIterable, Iterable, TypedDict -from ._utils import drop_falsy, normalize_tz_aware_datetime +from ._utils import drop_falsy, normalize_tz_aware_datetime, USER_AGENT # This module exposes a single entrypoint # fetch_repo_info(Url, Iterable[QueryScope]) -> RepoInfo @@ -292,6 +292,7 @@ def github_headers(accept: str) -> dict[str, str]: if not token: raise RuntimeError("GITHUB_TOKEN env var is not set") return { + "User-Agent": USER_AGENT, "Authorization": f"Bearer {token}", "Accept": accept, } diff --git a/scripts/gitlab.py b/scripts/gitlab.py index b9075f323..6dd3e2bc7 100644 --- a/scripts/gitlab.py +++ b/scripts/gitlab.py @@ -7,7 +7,7 @@ from urllib.parse import urlparse, quote from typing import AsyncIterable, TypedDict, Literal, Iterable -from ._utils import drop_falsy, err, normalize_tz_aware_datetime +from ._utils import drop_falsy, err, normalize_tz_aware_datetime, USER_AGENT QueryScope = Literal["METADATA", "TAGS", "BRANCHES"] Url = str @@ -70,7 +70,7 @@ def parse_owner_repo(url: str): def _auth_headers() -> dict[str, str]: - headers: dict[str, str] = {} + headers: dict[str, str] = {'User-Agent': USER_AGENT} if token := os.getenv("GITLAB_TOKEN"): headers["PRIVATE-TOKEN"] = token return headers