Add User-Agent header to all crawler requests

scripts/_resolve_lib.py

@@ -22,7 +22,14 @@
     RepoInfo,
     RepoMetadata,
 )
-from ._utils import drop_falsy, flatten, pipe, unique_values_preserving_order, write_json
+from ._utils import (
+    drop_falsy,
+    flatten,
+    pipe,
+    unique_values_preserving_order,
+    write_json,
+    USER_AGENT,
+)
 
 
 PYPI_BASE = "https://pypi.org/pypi/{}/json"
@@ -619,7 +626,7 @@ async def _fetch_pypi_json(
             if fetched_at and now - fetched_at < ttl_seconds:
                 return load_json(cache_path), "cache"
 
-    headers = {}
+    headers = {'User-Agent': USER_AGENT}
     if has_cache:
         if entry.get("etag"):
             headers["If-None-Match"] = entry["etag"]

scripts/_utils.py

@@ -13,6 +13,9 @@
 import inflect
 
 
+USER_AGENT = "Mozilla/5.0 (thecrawl 1.0; +https://packages.sublimetext.io)"
+
+
 def err(*args, **kwargs):
     print(*args, **kwargs, file=sys.stderr)
 

scripts/accumulate_stats.py

@@ -8,7 +8,7 @@
 from itertools import count, takewhile
 from urllib.request import Request, urlopen
 
-from ._utils import write_json
+from ._utils import write_json, USER_AGENT
 
 DEFAULT_OUTPUT_FILE = "stats.json"
 DEFAULT_URL = "https://stats.sublimetext.io/all-totals"
@@ -121,7 +121,7 @@ def accumulate(value: int, container: dict, key: str, wanted_length: int, rollov
 
 
 def fetch_totals(url: str) -> dict:
-    request = Request(url, headers={"User-Agent": "Mozilla/5.0"})
+    request = Request(url, headers={"User-Agent": USER_AGENT})
     with urlopen(request) as resp:
         return json.load(resp)
 

scripts/bitbucket.py

@@ -7,7 +7,7 @@
 
 from typing import AsyncIterable, TypedDict, Literal, Iterable
 
-from ._utils import drop_falsy, err, normalize_tz_aware_datetime
+from ._utils import drop_falsy, err, normalize_tz_aware_datetime, USER_AGENT
 
 
 type QueryScope = Literal["METADATA", "TAGS", "BRANCHES"]
@@ -62,7 +62,7 @@ class RepoInfo(TypedDict):
 
 
 async def fetch_json(session: aiohttp.ClientSession, url: str) -> dict:
-    headers = {}
+    headers = {'User-Agent': USER_AGENT}
     if token := os.getenv("BITBUCKET_TOKEN"):
         headers["Authorization"] = f"Bearer {token}"
     async with session.get(url, headers=headers) as resp:

scripts/codeberg.py

@@ -7,7 +7,7 @@
 
 from typing import AsyncIterable, TypedDict, Literal, Iterable
 
-from ._utils import drop_falsy, err, normalize_tz_aware_datetime
+from ._utils import drop_falsy, err, normalize_tz_aware_datetime, USER_AGENT
 
 
 type QueryScope = Literal["METADATA", "TAGS", "BRANCHES"]
@@ -71,7 +71,7 @@ def parse_owner_repo(url: str):
 
 
 def _auth_headers() -> dict[str, str]:
-    headers: dict[str, str] = {}
+    headers: dict[str, str] = {'User-Agent': USER_AGENT}
     # Codeberg (Forgejo/Gitea) supports Authorization: token <TOKEN>
     if token := os.getenv("CODEBERG_TOKEN"):
         headers["Authorization"] = f"token {token}"

scripts/compress_channel.py

@@ -8,7 +8,7 @@
 import os
 import sys
 
-from ._utils import pl, write_json
+from ._utils import pl, write_json, USER_AGENT
 
 
 NEW_CHANNEL = (
@@ -153,7 +153,7 @@ async def http_get_json(location: str, session: aiohttp.ClientSession) -> dict:
 
 async def http_get(location: str, session: aiohttp.ClientSession) -> str:
     headers = {
-        'User-Agent': 'Mozilla/5.0',
+        'User-Agent': USER_AGENT,
         'Cache-Control': 'no-cache',
         'Pragma': 'no-cache'
     }

scripts/generate_registry.py

@@ -12,7 +12,7 @@
 from urllib.parse import urlparse
 from typing import Any, Callable, Iterable, Mapping, NotRequired, TypedDict, TypeGuard
 
-from ._utils import flatten, pick, resolve_urls, update_url, write_json, pl
+from ._utils import flatten, pick, resolve_urls, update_url, write_json, pl, USER_AGENT
 
 
 DEFAULT_OUTPUT_FILE = "./registry.json"
@@ -337,7 +337,7 @@ async def http_get_json(location: str, session: aiohttp.ClientSession) -> dict:
 
 
 async def http_get(location: str, session: aiohttp.ClientSession) -> str:
-    headers = {'User-Agent': 'Mozilla/5.0'}
+    headers = {'User-Agent': USER_AGENT}
     async with session.get(location, headers=headers, raise_for_status=True) as resp:
         return await resp.text()
 

scripts/github.py

@@ -11,7 +11,7 @@
 
 from typing import AsyncIterable, Iterable, TypedDict
 
-from ._utils import drop_falsy, normalize_tz_aware_datetime
+from ._utils import drop_falsy, normalize_tz_aware_datetime, USER_AGENT
 
 # This module exposes a single entrypoint
 # fetch_repo_info(Url, Iterable[QueryScope]) -> RepoInfo
@@ -292,6 +292,7 @@ def github_headers(accept: str) -> dict[str, str]:
     if not token:
         raise RuntimeError("GITHUB_TOKEN env var is not set")
     return {
+        "User-Agent": USER_AGENT,
         "Authorization": f"Bearer {token}",
         "Accept": accept,
     }

scripts/gitlab.py

@@ -7,7 +7,7 @@
 from urllib.parse import urlparse, quote
 from typing import AsyncIterable, TypedDict, Literal, Iterable
 
-from ._utils import drop_falsy, err, normalize_tz_aware_datetime
+from ._utils import drop_falsy, err, normalize_tz_aware_datetime, USER_AGENT
 
 QueryScope = Literal["METADATA", "TAGS", "BRANCHES"]
 Url = str
@@ -70,7 +70,7 @@ def parse_owner_repo(url: str):
 
 
 def _auth_headers() -> dict[str, str]:
-    headers: dict[str, str] = {}
+    headers: dict[str, str] = {'User-Agent': USER_AGENT}
     if token := os.getenv("GITLAB_TOKEN"):
         headers["PRIVATE-TOKEN"] = token
     return headers