From 65cda37ad8f81e30d7614098435b09375d4a757f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B4=BE=E6=99=93=E6=BA=90?= Date: Sat, 15 Aug 2026 02:20:58 +0800 Subject: [PATCH 1/4] =?UTF-8?q?feat:=20add=20searxng-search=20=E2=80=94=20?= =?UTF-8?q?web=20search=20via=20self-hosted=20SearXNG=20instance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent-plugin contribution for the community registry: plugins/Fectivnfy112357/searxng-search/ --- .../Fectivnfy112357/searxng-search/LICENSE | 21 ++ .../Fectivnfy112357/searxng-search/README.md | 52 ++++ .../searxng-search/plugin.json | 12 + .../skills/searxng-search/SKILL.md | 79 +++++ .../references/configuration.md | 268 +++++++++++++++++ .../skills/searxng-search/scripts/search.py | 275 ++++++++++++++++++ 6 files changed, 707 insertions(+) create mode 100644 plugins/Fectivnfy112357/searxng-search/LICENSE create mode 100644 plugins/Fectivnfy112357/searxng-search/README.md create mode 100644 plugins/Fectivnfy112357/searxng-search/plugin.json create mode 100644 plugins/Fectivnfy112357/searxng-search/skills/searxng-search/SKILL.md create mode 100644 plugins/Fectivnfy112357/searxng-search/skills/searxng-search/references/configuration.md create mode 100644 plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/search.py diff --git a/plugins/Fectivnfy112357/searxng-search/LICENSE b/plugins/Fectivnfy112357/searxng-search/LICENSE new file mode 100644 index 0000000..3837a91 --- /dev/null +++ b/plugins/Fectivnfy112357/searxng-search/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Fectivnfy112357 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/plugins/Fectivnfy112357/searxng-search/README.md b/plugins/Fectivnfy112357/searxng-search/README.md new file mode 100644 index 0000000..9c84c00 --- /dev/null +++ b/plugins/Fectivnfy112357/searxng-search/README.md @@ -0,0 +1,52 @@ +# searxng-search + +Search the web through **your own self-hosted SearXNG instance** with a +zero-dependency Python script: categories, engines, time range, language and +safe-search filters, plus bearer/basic auth and TOML or JSON config. + +## The problem + +Web search inside an agent usually means a vendor API with credentials and +quotas. If you already run SearXNG (a privacy-respecting metasearch engine), +this skill gives the agent a first-class search tool against it — one config +file, no vendor SDK, no tracking. + +## Try it + +Install from `/plugins` → **Local**, configure your instance, then ask: + +```text +search the web for the latest SearXNG documentation +``` + +**Expected result**: a formatted list of results (title, URL, snippet) from +your SearXNG instance, filtered by the requested category / time range / +language. + +Direct usage: + +```text +python3 scripts/search.py -c news -t day "latest tech news" +python3 scripts/search.py -e google,duckduckgo -p 2 "rust programming" +python3 scripts/search.py -l zh-CN -n 10 "开源搜索引擎" +``` + +## Requirements + +- A **self-hosted SearXNG instance** reachable over HTTP(S) — you provide it; + no public instance is bundled. +- Python 3.11+ (TOML config; legacy JSON config works on older Python). +- A config file at `~/.config/agents/searxng.toml` (or `$XDG_CONFIG_HOME`); + see `skills/searxng-search/references/configuration.md` for all fields. + +## Data and network + +- The script talks **only to your configured SearXNG instance** (the + `base_url` in your config). No other network destinations. +- Credentials for your instance (bearer token / basic auth, if any) live in + your local config file; they are never sent anywhere else. +- No telemetry, no third-party services. + +## License + +MIT diff --git a/plugins/Fectivnfy112357/searxng-search/plugin.json b/plugins/Fectivnfy112357/searxng-search/plugin.json new file mode 100644 index 0000000..8eb7fe2 --- /dev/null +++ b/plugins/Fectivnfy112357/searxng-search/plugin.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", + "name": "searxng-search", + "version": "1.0.0", + "description": "Search the web through your own self-hosted SearXNG instance: categories, engines, time range, language and safe-search filters via a zero-dependency Python script, with bearer/basic auth and TOML or JSON config.", + "author": { + "name": "Fectivnfy112357", + "url": "https://github.com/Fectivnfy112357" + }, + "license": "MIT", + "keywords": ["searxng", "search", "self-hosted", "privacy", "metasearch"] +} diff --git a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/SKILL.md b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/SKILL.md new file mode 100644 index 0000000..c66292c --- /dev/null +++ b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/SKILL.md @@ -0,0 +1,79 @@ +--- +name: searxng-search +description: "Search the web using a self-hosted SearXNG instance. Use when users ask to search with SearXNG, or when web search is needed and a SearXNG instance is configured. Supports categories, engines, time range, and language filters." +license: MIT +allowed-tools: + - Bash(python3 scripts/*) +--- + +# SearXNG Search Skill + +Search the web using a SearXNG instance via its API. + +## Configuration issues + +This skill depends on a local SearXNG config file. Keep setup details out of this +file and load [references/configuration.md](references/configuration.md) only when needed. + +If `scripts/search.py` reports configuration, auth, or instance setup errors, +read [references/configuration.md](references/configuration.md) before retrying. Common examples include: + +- `ERROR: Config file not found` +- `ERROR: Invalid TOML ...` / `ERROR: Invalid JSON ...` +- `ERROR: base_url is required` +- `ERROR: Environment variable ... is not set` +- `ERROR: auth.token required for bearer auth` +- `ERROR: auth.user and auth.pass required for basic auth` +- `ERROR: Unknown auth.type ...` +- `ERROR: HTTP 401` / `ERROR: HTTP 403` + +## Usage + +Run the search script: + +```bash +python3 scripts/search.py [OPTIONS] +``` + +### Options + +| Flag | Description | +|---|---| +| `-c, --categories` | Comma-separated categories (`general`, `news`, `images`, `videos`, `music`, `files`, `it`, `science`, `social media`) | +| `-e, --engines` | Comma-separated engines (`google`, `duckduckgo`, `bing`, etc.) | +| `-l, --language` | Language code (`en`, `zh-CN`, `ja`, etc.) | +| `-p, --page` | Page number (default: 1) | +| `-t, --time-range` | Time range: `day`, `month`, `year` | +| `-n, --max-results` | Max results to show (overrides config default) | +| `-s, --safesearch` | Safe search: `0` (off), `1` (moderate), `2` (strict) | + +### Examples + +```bash +# Basic search +python3 scripts/search.py "SearXNG documentation" + +# Search news from the last day +python3 scripts/search.py -c news -t day "latest tech news" + +# Search with specific engines, page 2 +python3 scripts/search.py -e google,duckduckgo -p 2 "rust programming" + +# Search in Chinese with more results +python3 scripts/search.py -l zh-CN -n 10 "开源搜索引擎" +``` + +## Best Practices + +- **Technical topics** (programming, software, science, IT, etc.): Always use **English** as both the query language and search language (`-l en`), regardless of the user's input language. Translate the query to English if needed. English results are more comprehensive and up-to-date for technical content. +- **Chinese lifestyle topics** (food, travel, shopping, local services, social trends, etc.): In addition to the default search, run a **second search** with `-e baidu,sogou -l zh-CN` using a Chinese query to capture China-specific results. Merge and deduplicate results before presenting to the user. + +## Workflow + +1. User asks to search for something +2. Determine the topic type: + - **Technical**: translate query to English if needed, search with `-l en` + - **Chinese lifestyle**: run the default search first, then an additional search with `-e baidu,sogou -l zh-CN` +3. Run `scripts/search.py` with the query and any relevant filters +4. Present results to the user in a readable format +5. If user wants more results, use `-p` for pagination or `-n` for more per page diff --git a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/references/configuration.md b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/references/configuration.md new file mode 100644 index 0000000..5efc060 --- /dev/null +++ b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/references/configuration.md @@ -0,0 +1,268 @@ +# SearXNG configuration reference + +Use this document when you need to: + +- set up the skill for the first time +- change the SearXNG instance or auth settings +- debug configuration/auth/setup errors from `scripts/search.py` + +## Config file locations + +The script reads configuration from: + +1. `$XDG_CONFIG_HOME/agents/searxng.toml` +2. If `XDG_CONFIG_HOME` is not set, it defaults to `~/.config/agents/searxng.toml` +3. If the TOML file does not exist, the script falls back to the legacy JSON file at `$XDG_CONFIG_HOME/agents/searxng.json` + +In practice, the common paths are: + +- `~/.config/agents/searxng.toml` +- `~/.config/agents/searxng.json` (legacy fallback) + +## Supported fields + +| Field | Type | Required | Description | +|---|---|---|---| +| `base_url` | `string` | **Yes** | Base URL of the SearXNG instance, for example `https://searx.example.com` | +| `auth` | `object` | No | Authentication config | +| `auth.type` | `string` | If `auth` is set | Must be `"bearer"` or `"basic"` | +| `auth.token` | `string` | If `auth.type = "bearer"` | Bearer token value; supports `$ENV_VAR` or `${ENV_VAR}` | +| `auth.user` | `string` | If `auth.type = "basic"` | Basic auth username; supports `$ENV_VAR` or `${ENV_VAR}` | +| `auth.pass` | `string` | If `auth.type = "basic"` | Basic auth password; supports `$ENV_VAR` or `${ENV_VAR}` | +| `headers` | `object` | No | Extra HTTP headers to send with the request | +| `default_language` | `string` | No | Default language code, for example `en` or `zh-CN` | +| `default_categories` | `string[]` | No | Default categories, for example `["general", "news"]` | +| `default_engines` | `string[]` | No | Default engines, for example `["google", "duckduckgo"]` | +| `default_safesearch` | `number` | No | Default safe search level: `0`, `1`, or `2` | +| `default_time_range` | `string` | No | Default time range: `"day"`, `"month"`, or `"year"` | +| `default_max_results` | `number` | No | Default number of results to display; script default is `5` | +| `timeout` | `number` | No | Request timeout in seconds; script default is `30` | + +## Example TOML config: bearer auth + +```toml +base_url = "https://searx.example.com" +default_categories = ["general"] +default_engines = ["google", "duckduckgo", "brave"] +default_max_results = 10 + +[auth] +type = "bearer" +token = "your-token-here" + +[headers] +X-Custom-Header = "value" +``` + +## Example TOML config: bearer auth with environment variable + +```toml +base_url = "https://searx.example.com" +default_categories = ["general"] + +[auth] +type = "bearer" +token = "$SEARXNG_TOKEN" +``` + +Then export the variable before running the script: + +```bash +export SEARXNG_TOKEN='your-token-here' +``` + +## Example TOML config: basic auth + +```toml +base_url = "https://searx.example.com" +default_safesearch = 1 + +[auth] +type = "basic" +user = "admin" +pass = "password" +``` + +## Example TOML config: basic auth with environment variables + +```toml +base_url = "https://searx.example.com" + +[auth] +type = "basic" +user = "$SEARXNG_USER" +pass = "$SEARXNG_PASS" +``` + +```bash +export SEARXNG_USER='admin' +export SEARXNG_PASS='password' +``` + +## Legacy JSON fallback + +If `searxng.toml` does not exist, the script still supports the old JSON format. + +```json +{ + "base_url": "https://searx.example.com", + "default_categories": ["general"], + "default_engines": ["google", "duckduckgo"], + "default_max_results": 10, + "auth": { + "type": "bearer", + "token": "$SEARXNG_TOKEN" + }, + "headers": { + "X-Custom-Header": "value" + } +} +``` + +## Troubleshooting + +### `ERROR: Config file not found: ...` + +The config file is missing from the expected path. + +Checks: + +- confirm whether `XDG_CONFIG_HOME` is set +- create `~/.config/agents/searxng.toml` if you do not use a custom XDG path +- if you are still using the old JSON config, make sure the TOML file is absent and the JSON file exists at the fallback path + +Minimum working TOML: + +```toml +base_url = "https://your-searxng-instance.com" +``` + +### `ERROR: TOML config found at ... but this Python does not support TOML.` + +Your Python is too old for `tomllib`. + +Fix one of these: + +- use Python 3.11+ +- remove/rename `searxng.toml` and use the legacy JSON fallback instead + +### `ERROR: Invalid TOML in ...` / `ERROR: Invalid JSON in ...` + +The config file has syntax errors. + +Checks: + +- validate commas, quotes, and brackets in JSON +- validate table syntax (`[auth]`, `[headers]`) and string quoting in TOML +- make sure arrays look like `["general", "news"]` + +### `ERROR: base_url is required in ...` + +Add a `base_url` value. + +Example: + +```toml +base_url = "https://searx.example.com" +``` + +Use the SearXNG instance root URL. The script will normalize trailing slashes automatically. + +### `ERROR: Environment variable SOME_VAR is not set` + +You referenced an env var in `auth.token`, `auth.user`, or `auth.pass`, but it is not exported in the current shell. + +Fix: + +```bash +export SOME_VAR='value' +``` + +Then rerun the command from the same shell/session. + +### `ERROR: auth.token required for bearer auth` + +You set `auth.type = "bearer"` but did not provide `auth.token`. + +Example: + +```toml +[auth] +type = "bearer" +token = "$SEARXNG_TOKEN" +``` + +### `ERROR: auth.user and auth.pass required for basic auth` + +You set `auth.type = "basic"` but one or both credentials are missing. + +Example: + +```toml +[auth] +type = "basic" +user = "$SEARXNG_USER" +pass = "$SEARXNG_PASS" +``` + +### `ERROR: Unknown auth.type '...'` + +Only these auth modes are supported: + +- `bearer` +- `basic` + +Correct typos or remove `auth` entirely if the instance does not require authentication. + +### `ERROR: HTTP 401: ...` / `ERROR: HTTP 403: ...` + +The request reached the server, but auth or access control failed. + +Checks: + +- verify `auth.type` +- verify the bearer token or basic auth credentials +- verify any required custom headers under `headers` +- confirm your SearXNG instance allows API access from your environment + +### `ERROR: Request failed: ...` + +This is usually a network, DNS, TLS, or connectivity problem. + +Checks: + +- verify `base_url` +- make sure the instance is reachable from your machine +- check VPN, proxy, DNS, or certificate issues + +### `ERROR: Invalid JSON response from SearXNG` + +The endpoint responded, but not with valid JSON. This often means the URL points to the wrong place or a proxy/login page returned HTML. + +Checks: + +- verify `base_url` points to the SearXNG instance root +- confirm the instance supports `/search?format=json` +- check whether a reverse proxy or SSO page is intercepting the request + +### `ERROR: SearXNG returned error: ...` + +The server returned a structured API error. + +Checks: + +- inspect the error text +- verify query parameters and instance settings +- try the same instance in a browser or with `curl` to confirm the backend is healthy + +## Quick setup checklist + +1. Create `~/.config/agents/searxng.toml` +2. Set `base_url` +3. Add auth only if your instance requires it +4. Export any referenced environment variables +5. Run a smoke test: + +```bash +python3 scripts/search.py "SearXNG documentation" +``` diff --git a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/search.py b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/search.py new file mode 100644 index 0000000..8e8e17f --- /dev/null +++ b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/search.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python3 +"""SearXNG Search Script - Search the web via a SearXNG instance.""" + +import argparse +import base64 +import json +import os +import sys +import urllib.error +import urllib.parse +import urllib.request + +try: + import tomllib +except ModuleNotFoundError: # pragma: no cover - Python < 3.11 + tomllib = None + + +def load_config(): + config_dir = os.environ.get("XDG_CONFIG_HOME", os.path.expanduser("~/.config")) + config_dir = os.path.join(config_dir, "agents") + toml_file = os.path.join(config_dir, "searxng.toml") + json_file = os.path.join(config_dir, "searxng.json") + + if os.path.isfile(toml_file): + config_file = toml_file + if tomllib is None: + print( + f"ERROR: TOML config found at {config_file}, but this Python does not support TOML.", + file=sys.stderr, + ) + print("Upgrade to Python 3.11+ or remove the TOML config to use legacy JSON fallback.", file=sys.stderr) + sys.exit(1) + + try: + with open(config_file, "rb") as f: + config = tomllib.load(f) + except tomllib.TOMLDecodeError as e: + print(f"ERROR: Invalid TOML in {config_file}: {e}", file=sys.stderr) + sys.exit(1) + elif os.path.isfile(json_file): + config_file = json_file + try: + with open(config_file) as f: + config = json.load(f) + except json.JSONDecodeError as e: + print(f"ERROR: Invalid JSON in {config_file}: {e}", file=sys.stderr) + sys.exit(1) + else: + print(f"ERROR: Config file not found: {toml_file}", file=sys.stderr) + print( + f"Legacy JSON fallback path: {json_file}", + file=sys.stderr, + ) + print( + 'Create TOML config with at minimum: base_url = "https://your-searxng-instance.com"', + file=sys.stderr, + ) + sys.exit(1) + + if not config.get("base_url"): + print(f"ERROR: base_url is required in {config_file}", file=sys.stderr) + sys.exit(1) + + return config + + +def resolve_env(value: str) -> str: + """Resolve $ENV_VAR or ${ENV_VAR} references in a string.""" + if not value: + return value + if value.startswith("$"): + var_name = value.lstrip("$").strip("{}") + resolved = os.environ.get(var_name) + if not resolved: + print(f"ERROR: Environment variable {var_name} is not set", file=sys.stderr) + sys.exit(1) + return resolved + return value + + +def build_request(config: dict, args: argparse.Namespace) -> urllib.request.Request: + base_url = config["base_url"].rstrip("/") + + params = { + "q": args.query, + "format": "json", + "pageno": str(args.page), + } + + categories = args.categories or config.get("default_categories") + if categories: + params["categories"] = ",".join(categories) if isinstance(categories, list) else categories + + engines = args.engines or config.get("default_engines") + if engines: + params["engines"] = ",".join(engines) if isinstance(engines, list) else engines + + language = args.language or config.get("default_language") + if language: + params["language"] = language + + safesearch = args.safesearch if args.safesearch is not None else config.get("default_safesearch") + if safesearch is not None: + params["safesearch"] = str(safesearch) + + time_range = args.time_range or config.get("default_time_range") + if time_range: + params["time_range"] = time_range + + url = f"{base_url}/search?{urllib.parse.urlencode(params)}" + req = urllib.request.Request(url) + + # Auth + auth = config.get("auth", {}) + auth_type = auth.get("type", "") + if auth_type == "bearer": + token = resolve_env(auth.get("token", "")) + if not token: + print("ERROR: auth.token required for bearer auth", file=sys.stderr) + sys.exit(1) + req.add_header("Authorization", f"Bearer {token}") + elif auth_type == "basic": + user = resolve_env(auth.get("user", "")) + password = resolve_env(auth.get("pass", "")) + if not user or not password: + print("ERROR: auth.user and auth.pass required for basic auth", file=sys.stderr) + sys.exit(1) + credentials = base64.b64encode(f"{user}:{password}".encode()).decode() + req.add_header("Authorization", f"Basic {credentials}") + elif auth_type: + print(f"ERROR: Unknown auth.type '{auth_type}'. Use 'bearer' or 'basic'.", file=sys.stderr) + sys.exit(1) + + # Headers + headers = config.get("headers", {}) + for key, value in headers.items(): + req.add_header(key, value) + + # Set a reasonable User-Agent if not already set via headers + if "User-Agent" not in headers: + req.add_header("User-Agent", "searxng-search-skill (+https://github.com/XYenon/agents)") + + return req + + +def format_results(data: dict, max_results: int, page: int): + # Answers — direct answers from engines + answers = data.get("answers", []) + if answers: + print("# Answers\n") + for a in answers: + answer = a if isinstance(a, str) else a.get("answer", "") + if answer: + print(f"> {answer}\n") + + # Infoboxes — knowledge panel style info + for ib in data.get("infoboxes", []): + name = ib.get("infobox", "") + content = ib.get("content", "") + if name: + print(f"# Infobox: {name}\n") + if content: + print(f"{content}\n") + for attr in ib.get("attributes", []): + print(f"- **{attr['label']}:** {attr['value']}") + urls = ib.get("urls", []) + if urls: + print() + for u in urls: + official = " (official)" if u.get("official") else "" + print(f"- [{u['title']}{official}]({u['url']})") + print() + + # Search results + results = data.get("results", []) + if not results: + print("No results found.") + return + + print("# Results\n") + for r in results[:max_results]: + title = r.get("title", "Untitled") + url = r.get("url", "") + content = r.get("content", "") + engines = r.get("engines", []) + category = r.get("category", "") + score = r.get("score", 0) + date = r.get("publishedDate") + + print(f"## [{title}]({url})") + if content: + print(f"\n{content}") + meta = [] + if engines: + meta.append(f"engines: {','.join(engines)}") + if category: + meta.append(f"category: {category}") + if score: + meta.append(f"score: {score}") + if date: + meta.append(f"date: {date}") + if meta: + print(f"\n*{' | '.join(meta)}*") + print() + + # Suggestions + suggestions = data.get("suggestions", []) + if suggestions: + print(f"**Suggestions:** {', '.join(suggestions)}\n") + + # Corrections + corrections = data.get("corrections", []) + if corrections: + print(f"**Corrections:** {', '.join(corrections)}\n") + + total = len(results) + shown = min(max_results, total) + print("---") + print(f"Showing {shown} of {total} results (page {page})") + + +def main(): + parser = argparse.ArgumentParser(description="Search the web using SearXNG") + parser.add_argument("query", nargs="+", help="Search query") + parser.add_argument("-c", "--categories", help="Comma-separated categories (e.g. general,news,images)") + parser.add_argument("-e", "--engines", help="Comma-separated engines (e.g. google,duckduckgo)") + parser.add_argument("-l", "--language", help="Language code (e.g. en, zh-CN)") + parser.add_argument("-p", "--page", type=int, default=1, help="Page number (default: 1)") + parser.add_argument("-t", "--time-range", choices=["day", "month", "year"], help="Time range filter") + parser.add_argument("-n", "--max-results", type=int, help="Maximum results to display") + parser.add_argument("-s", "--safesearch", type=int, choices=[0, 1, 2], help="Safe search level: 0, 1, 2") + + args = parser.parse_args() + args.query = " ".join(args.query) + + config = load_config() + + if args.max_results is None: + args.max_results = config.get("default_max_results", 5) + + timeout = config.get("timeout", 30) + + req = build_request(config, args) + + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + body = resp.read().decode() + except urllib.error.HTTPError as e: + body = e.read().decode() if e.fp else "" + print(f"ERROR: HTTP {e.code}: {body[:500]}", file=sys.stderr) + sys.exit(1) + except urllib.error.URLError as e: + print(f"ERROR: Request failed: {e.reason}", file=sys.stderr) + sys.exit(1) + except TimeoutError: + print(f"ERROR: Request timed out after {timeout}s", file=sys.stderr) + sys.exit(1) + + try: + data = json.loads(body) + except json.JSONDecodeError: + print("ERROR: Invalid JSON response from SearXNG", file=sys.stderr) + print(f"Response: {body[:500]}", file=sys.stderr) + sys.exit(1) + + if "error" in data: + print(f"ERROR: SearXNG returned error: {data['error']}", file=sys.stderr) + sys.exit(1) + + format_results(data, args.max_results, args.page) + + +if __name__ == "__main__": + main() From a49ad611b1a3ab661a8f42c532e2079275dbe39e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B4=BE=E6=99=93=E6=BA=90?= Date: Sat, 15 Aug 2026 15:31:24 +0800 Subject: [PATCH 2/4] =?UTF-8?q?fix(searxng-search):=20address=20PR=20#9=20?= =?UTF-8?q?review=20=E2=80=94=20HTTPS,=20downstream=20disclosure,=20env-va?= =?UTF-8?q?r,=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 针对 #9 审阅的 4 条 credential / 披露 / 凭据 / 测试门禁要求做修复。 ## 1. HTTPS 强制 + URL 校验 - 新增 `validate_base_url(url, config)`: HTTPS 一律放行；HTTP 仅允许 loopback 主机（127.0.0.1 / ::1 / localhost / 0.0.0.0）; 非 loopback HTTP 需 config 中显式 `allow_insecure_http = true` 并发出强警告 （含凭据明文链路风险），否则 die。 - 在 `load_config()` 末尾调用 `validate_base_url`。 - 不再接受 ftp / file 等非 http(s) scheme。 ## 2. direct vs downstream 披露 - `README.md` "Data and network" 拆为 "Direct destination (the script itself)" 与 "Downstream destinations (the SearXNG instance, not the script)"，列出 引擎名（Google / Bing / DuckDuckGo / Brave / Baidu 等由实例运维者配置）。 - `SKILL.md` 顶部新增 "Network destinations" 节，同一区分。 - `plugin.json` description 同步更新。 ## 3. env-var 优先 + 文件权限 - `references/configuration.md` 顶部加 "Recommended: keep secrets out of the file"，env-var 示例前置；明文示例移到 "Plaintext secrets in the config" 节，并配 `chmod 600` 警告。 - 新增 `check_file_permissions(path)`：POSIX 上 `mode & 0o077` 时 warn； Windows / 缺文件不告警。 - `load_config()` 末尾调用 `check_file_permissions`。 ## 4. 可复现回归测试 - `scripts/tests/test_url_validation.py` (9): HTTPS / loopback / opt-in / non-loopback reject / 非法 scheme / 缺 host。 - `scripts/tests/test_auth_header.py` (8): bearer / basic 头构造，env-var 解析，User-Agent 引用真实仓库，token 不漏到 URL。 - `scripts/tests/test_redaction.py` (5): 5 种凭据形态 + die/warn 路径 不回显凭据。 - `scripts/tests/test_config.py` (11): 缺文件 / 坏 TOML / 缺 base_url / env-var 未设 / 文件权限告警。 - `scripts/tests/test_timeout.py` (2): 默认 30s + 配置覆盖。 - `scripts/tests/test_invalid_response.py` (5): 非 JSON / 结构化 error / HTTPError body 含凭据被 mask / URLError / TimeoutError。 - `scripts/run_tests.py` (unittest discover 入口) + `test/searxng-search.test.mjs` (node --test bridge)。 ## 顺带修 - User-Agent: `XYenon/agents` -> `Fectivnfy112357/MiniMax-Code-Plugins` - 重构 `print(..., file=sys.stderr); sys.exit(1)` 为 `die(msg)` / `warn(msg)` 助手（消除 10+ 处重复）。 - 新增 `__pycache__/` / `*.pyc` 到 plugin-local `.gitignore`。 测试结果: 40/40 OK; node --test pass 1 fail 0; validator OK --- .../Fectivnfy112357/searxng-search/README.md | 38 ++- .../searxng-search/plugin.json | 2 +- .../skills/searxng-search/.gitignore | 2 + .../skills/searxng-search/SKILL.md | 18 ++ .../references/configuration.md | 126 +++++++--- .../searxng-search/scripts/run_tests.py | 19 ++ .../skills/searxng-search/scripts/search.py | 226 ++++++++++++++---- .../searxng-search/scripts/tests/__init__.py | 1 + .../searxng-search/scripts/tests/_fixtures.py | 47 ++++ .../scripts/tests/test_auth_header.py | 91 +++++++ .../scripts/tests/test_config.py | 128 ++++++++++ .../scripts/tests/test_invalid_response.py | 96 ++++++++ .../scripts/tests/test_redaction.py | 63 +++++ .../scripts/tests/test_timeout.py | 42 ++++ .../scripts/tests/test_url_validation.py | 92 +++++++ test/searxng-search.test.mjs | 32 +++ 16 files changed, 933 insertions(+), 90 deletions(-) create mode 100644 plugins/Fectivnfy112357/searxng-search/skills/searxng-search/.gitignore create mode 100644 plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/run_tests.py create mode 100644 plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/__init__.py create mode 100644 plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/_fixtures.py create mode 100644 plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_auth_header.py create mode 100644 plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_config.py create mode 100644 plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_invalid_response.py create mode 100644 plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_redaction.py create mode 100644 plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_timeout.py create mode 100644 plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_url_validation.py create mode 100644 test/searxng-search.test.mjs diff --git a/plugins/Fectivnfy112357/searxng-search/README.md b/plugins/Fectivnfy112357/searxng-search/README.md index 9c84c00..216892f 100644 --- a/plugins/Fectivnfy112357/searxng-search/README.md +++ b/plugins/Fectivnfy112357/searxng-search/README.md @@ -41,11 +41,39 @@ python3 scripts/search.py -l zh-CN -n 10 "开源搜索引擎" ## Data and network -- The script talks **only to your configured SearXNG instance** (the - `base_url` in your config). No other network destinations. -- Credentials for your instance (bearer token / basic auth, if any) live in - your local config file; they are never sent anywhere else. -- No telemetry, no third-party services. +This skill has **two levels of network destinations**; the difference matters. + +### Direct destination (the script itself) + +The script makes a single HTTPS (or loopback HTTP) request to **your +configured SearXNG instance** — the `base_url` in your config. The +script does not contact any other host. + +### Downstream destinations (the SearXNG instance, not the script) + +Your SearXNG instance then forwards the **query string**, **language +code**, and **selected categories** to the upstream **engines** it has +been configured with (Google, Bing, DuckDuckGo, Brave, Baidu, etc., as +enabled by the instance operator). Those engines receive the request +content from your instance — the script does not see or control that +hop. + +**Practical implication:** anything you put in the search query +(personal context, project names, internal jargon) reaches the engines +your instance is configured to use. This is true of any SearXNG client, +not something this skill introduces. If that is a concern, configure +your instance to use only engines you trust, or self-host engines +locally. + +### Credentials and config + +- Credentials for your instance (bearer token / basic auth, if any) live + in your local config file. They are sent only to your instance — the + script never sends them anywhere else. +- Use `$ENV_VAR` references in the config instead of writing tokens in + plaintext, and `chmod 600` the file on POSIX systems. See + `skills/searxng-search/references/configuration.md`. +- No telemetry, no third-party services run by this script. ## License diff --git a/plugins/Fectivnfy112357/searxng-search/plugin.json b/plugins/Fectivnfy112357/searxng-search/plugin.json index 8eb7fe2..2131f1c 100644 --- a/plugins/Fectivnfy112357/searxng-search/plugin.json +++ b/plugins/Fectivnfy112357/searxng-search/plugin.json @@ -2,7 +2,7 @@ "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", "name": "searxng-search", "version": "1.0.0", - "description": "Search the web through your own self-hosted SearXNG instance: categories, engines, time range, language and safe-search filters via a zero-dependency Python script, with bearer/basic auth and TOML or JSON config.", + "description": "Search the web through your own self-hosted SearXNG instance: categories, engines, time range, language and safe-search filters via a zero-dependency Python script, with bearer/basic auth and TOML or JSON config. The script contacts only the configured SearXNG instance (HTTPS by default; HTTP only for loopback or explicit opt-in); the instance then forwards the query, language, and categories to its own configured upstream engines (Google, Bing, DuckDuckGo, etc., as enabled by the instance operator). No telemetry, no third-party services run by this script.", "author": { "name": "Fectivnfy112357", "url": "https://github.com/Fectivnfy112357" diff --git a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/.gitignore b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/.gitignore new file mode 100644 index 0000000..7a60b85 --- /dev/null +++ b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +*.pyc diff --git a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/SKILL.md b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/SKILL.md index c66292c..ca0a0db 100644 --- a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/SKILL.md +++ b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/SKILL.md @@ -10,6 +10,24 @@ allowed-tools: Search the web using a SearXNG instance via its API. +## Network destinations + +Two levels — both matter: + +- **Direct (this script):** the script makes a single request to your + configured `base_url`. HTTPS by default; plain HTTP is allowed only + for loopback hosts (127.0.0.1, ::1, localhost) or for non-loopback + hosts if the config contains `allow_insecure_http = true` (with a + warning). If `auth.type` is set, the Authorization header travels to + that single host and nowhere else. +- **Downstream (your SearXNG instance):** the instance then forwards + the **query**, **language**, and **categories** to the engines it + itself has been configured with (Google, Bing, DuckDuckGo, Brave, + Baidu, etc., as enabled by the instance operator). This skill does + not control that hop — it is a property of any SearXNG client. If + query contents reaching the upstream engines is a concern, configure + your instance to use engines you trust, or self-host engines locally. + ## Configuration issues This skill depends on a local SearXNG config file. Keep setup details out of this diff --git a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/references/configuration.md b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/references/configuration.md index 5efc060..94112f6 100644 --- a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/references/configuration.md +++ b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/references/configuration.md @@ -6,6 +6,18 @@ Use this document when you need to: - change the SearXNG instance or auth settings - debug configuration/auth/setup errors from `scripts/search.py` +## Recommended: keep secrets out of the file + +The configuration file is world-readable on most systems. **Prefer +`$ENV_VAR` references for any secret** (bearer token, basic auth user +and password) and `chmod 600` the file so only your user can read it. +The script will exit with a clear error if a referenced environment +variable is not set; it does not silently send a literal `$NAME` value. + +Plaintext secrets in the config file are supported for testing but are +**not recommended** — see the [Plaintext secrets in the config](#plaintext-secrets-in-the-config) +section for the trade-offs and the file-permissions warning. + ## Config file locations The script reads configuration from: @@ -19,16 +31,31 @@ In practice, the common paths are: - `~/.config/agents/searxng.toml` - `~/.config/agents/searxng.json` (legacy fallback) +## File permissions + +On POSIX systems, restrict the config file to the owner. The script +emits a warning if the file is readable by group or others. + +```bash +# read/write for owner only +chmod 600 ~/.config/agents/searxng.toml +``` + +On Windows, use the file Properties → Security tab to limit read access +to your user account. The script does not currently detect overly +permissive ACLs on Windows. + ## Supported fields | Field | Type | Required | Description | |---|---|---|---| -| `base_url` | `string` | **Yes** | Base URL of the SearXNG instance, for example `https://searx.example.com` | +| `base_url` | `string` | **Yes** | Base URL of the SearXNG instance, for example `https://searx.example.com`. Plain HTTP is allowed only for loopback hosts, or for non-loopback hosts if `allow_insecure_http = true` is set. | +| `allow_insecure_http` | `bool` | No | Opt-in gate that allows plain `http://` to a non-loopback host. Triggers a strong warning at load time because the Authorization header would travel in cleartext. | | `auth` | `object` | No | Authentication config | | `auth.type` | `string` | If `auth` is set | Must be `"bearer"` or `"basic"` | -| `auth.token` | `string` | If `auth.type = "bearer"` | Bearer token value; supports `$ENV_VAR` or `${ENV_VAR}` | -| `auth.user` | `string` | If `auth.type = "basic"` | Basic auth username; supports `$ENV_VAR` or `${ENV_VAR}` | -| `auth.pass` | `string` | If `auth.type = "basic"` | Basic auth password; supports `$ENV_VAR` or `${ENV_VAR}` | +| `auth.token` | `string` | If `auth.type = "bearer"` | Bearer token value; supports `$ENV_VAR` or `${ENV_VAR}` (recommended) | +| `auth.user` | `string` | If `auth.type = "basic"` | Basic auth username; supports `$ENV_VAR` or `${ENV_VAR}` (recommended) | +| `auth.pass` | `string` | If `auth.type = "basic"` | Basic auth password; supports `$ENV_VAR` or `${ENV_VAR}` (recommended) | | `headers` | `object` | No | Extra HTTP headers to send with the request | | `default_language` | `string` | No | Default language code, for example `en` or `zh-CN` | | `default_categories` | `string[]` | No | Default categories, for example `["general", "news"]` | @@ -38,7 +65,7 @@ In practice, the common paths are: | `default_max_results` | `number` | No | Default number of results to display; script default is `5` | | `timeout` | `number` | No | Request timeout in seconds; script default is `30` | -## Example TOML config: bearer auth +## Example TOML config: bearer auth with environment variable (recommended) ```toml base_url = "https://searx.example.com" @@ -48,56 +75,64 @@ default_max_results = 10 [auth] type = "bearer" -token = "your-token-here" +token = "$SEARXNG_TOKEN" [headers] X-Custom-Header = "value" ``` -## Example TOML config: bearer auth with environment variable +Export the variable before running the script: + +```bash +export SEARXNG_TOKEN='your-token-here' +``` + +The script exits with `ERROR: Environment variable SEARXNG_TOKEN is not set` +if the variable is missing; it does not send a malformed `Bearer $NAME` +header. + +## Example TOML config: basic auth with environment variables (recommended) ```toml base_url = "https://searx.example.com" -default_categories = ["general"] [auth] -type = "bearer" -token = "$SEARXNG_TOKEN" +type = "basic" +user = "$SEARXNG_USER" +pass = "$SEARXNG_PASS" ``` -Then export the variable before running the script: - ```bash -export SEARXNG_TOKEN='your-token-here' +export SEARXNG_USER='admin' +export SEARXNG_PASS='password' ``` -## Example TOML config: basic auth +## Plaintext secrets in the config -```toml -base_url = "https://searx.example.com" -default_safesearch = 1 +Plaintext values in the config file are supported. They are easier to +set up but live in a file that may be readable beyond the owner. -[auth] -type = "basic" -user = "admin" -pass = "password" +If you must use them, restrict the file: + +```bash +chmod 600 ~/.config/agents/searxng.toml ``` -## Example TOML config: basic auth with environment variables +The script will warn at load time if the file is readable by group or +others. ```toml base_url = "https://searx.example.com" +default_safesearch = 1 [auth] type = "basic" -user = "$SEARXNG_USER" -pass = "$SEARXNG_PASS" +user = "admin" +pass = "password" ``` -```bash -export SEARXNG_USER='admin' -export SEARXNG_PASS='password' -``` +Treat the file as sensitive: do not commit it, do not share it, and +rotate the credentials if the file is ever exposed. ## Legacy JSON fallback @@ -168,6 +203,32 @@ base_url = "https://searx.example.com" Use the SearXNG instance root URL. The script will normalize trailing slashes automatically. +### `ERROR: base_url uses plain HTTP for non-loopback host ...` / `WARN: ... cleartext` + +The script refuses to send an `Authorization` header (or query +parameters) over plain HTTP to a non-loopback host. Pick one of: + +- Switch the instance to HTTPS (recommended). +- Point `base_url` at a loopback address (e.g. `http://127.0.0.1:8888`) + if you run SearXNG behind a local reverse proxy. +- Set `allow_insecure_http = true` in the config to opt in. The script + emits a warning each run, and you should rotate any credentials that + have been sent over plain HTTP. + +### `WARN: Config file ... is readable beyond the owner ...` + +The config file's POSIX mode is wider than `0600`, so other users on the +system can read it. + +Fix: + +```bash +chmod 600 ~/.config/agents/searxng.toml +``` + +On Windows, restrict read access in the file Properties → Security tab. +The script does not currently detect overly permissive ACLs on Windows. + ### `ERROR: Environment variable SOME_VAR is not set` You referenced an env var in `auth.token`, `auth.user`, or `auth.pass`, but it is not exported in the current shell. @@ -258,10 +319,11 @@ Checks: ## Quick setup checklist 1. Create `~/.config/agents/searxng.toml` -2. Set `base_url` -3. Add auth only if your instance requires it -4. Export any referenced environment variables -5. Run a smoke test: +2. Set `base_url` (HTTPS preferred; loopback HTTP also accepted) +3. `chmod 600` the file so only your user can read it +4. Add auth only if your instance requires it; prefer `$ENV_VAR` references +5. Export any referenced environment variables +6. Run a smoke test: ```bash python3 scripts/search.py "SearXNG documentation" diff --git a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/run_tests.py b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/run_tests.py new file mode 100644 index 0000000..f987bef --- /dev/null +++ b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/run_tests.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +"""Run searxng-search regression tests (stdlib unittest; no network, no gh). + +Usage: python run_tests.py (from the scripts/ directory) + python -m unittest discover -s tests -p "test_*.py" +""" +import os +import sys +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) # scripts/ dir so `import search` works + +if __name__ == "__main__": + loader = unittest.TestLoader() + suite = loader.discover(os.path.join(HERE, "tests"), pattern="test_*.py") + runner = unittest.TextTestRunner(verbosity=2) + result = runner.run(suite) + sys.exit(0 if result.wasSuccessful() else 1) diff --git a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/search.py b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/search.py index 8e8e17f..f6b7a76 100644 --- a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/search.py +++ b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/search.py @@ -1,10 +1,27 @@ #!/usr/bin/env python3 -"""SearXNG Search Script - Search the web via a SearXNG instance.""" +"""SearXNG Search Script - Search the web via a SearXNG instance. + +Reads a local TOML (or legacy JSON) config, builds a request against the +user's configured SearXNG instance, and prints the results as markdown. + +Security model: + - Default scheme: HTTPS. Plain HTTP is allowed only for loopback hosts + (127.0.0.1, ::1, localhost) or for non-loopback hosts if the config has + an explicit `allow_insecure_http = true` (with a strong warning). + - Auth values can be referenced via `$ENV_VAR` / `${ENV_VAR}` so the + plaintext never lives in the config file. + - All error output is run through `redact_secrets` so a token that + leaks into a server response body is masked before reaching stderr. + - On POSIX, a config file readable beyond the owner (mode & 0o077) is + flagged with a warning. +""" +from __future__ import annotations import argparse import base64 import json import os +import re import sys import urllib.error import urllib.parse @@ -16,6 +33,73 @@ tomllib = None +# ---------- error / progress helpers ---------- + +# Credential shapes that must never reach the transcript. Same pattern set +# as the github-explore skill (PR #8) so the agent learns one redaction +# grammar across skills. Order matters: the most specific label-shaped +# patterns run first so the whole token is masked in one pass, not +# partially replaced and then bypassed. +_SECRET_PATTERNS: list[tuple[re.Pattern[str], str]] = [ + (re.compile(r"(?i)(bearer\s+)[A-Za-z0-9._\-]{20,}"), r"\1***"), + (re.compile(r"(?i)(authorization:\s*basic\s+)[A-Za-z0-9+/=]+"), r"\1***"), + (re.compile(r"github_pat_[A-Za-z0-9_]+"), "github_pat_***"), + (re.compile(r"gh[pousr]_[A-Za-z0-9]{20,}"), "ghx_***"), + (re.compile(r"(?i)\b(token|password|passwd|secret|api[_-]?key)\b(\s*[:=]\s*)\S+"), r"\1\2***"), + (re.compile(r"(?i)\b(GH_TOKEN|GITHUB_TOKEN|SEARXNG_TOKEN|SEARXNG_USER|SEARXNG_PASS)\s*=\s*\S+"), r"\1=***"), +] + + +def redact_secrets(text: str) -> str: + """Mask credential-shaped substrings in `text`. + + Best-effort: only obvious credential shapes are masked, so ordinary + error text (URLs without embedded tokens, JSON error bodies, rate-limit + messages) passes through intact. + """ + if not text: + return text + out = text + for pat, repl in _SECRET_PATTERNS: + out = pat.sub(repl, out) + return out + + +def die(msg: str, code: int = 1) -> None: + """Print a redacted error to stderr and exit.""" + print(f"ERROR: {redact_secrets(msg)}", file=sys.stderr) + sys.exit(code) + + +def warn(msg: str) -> None: + """Print a redacted warning to stderr.""" + print(f"WARN: {redact_secrets(msg)}", file=sys.stderr) + + +def info(msg: str) -> None: + """Progress message to stderr so stdout stays pipeable.""" + print(f"… {msg}", file=sys.stderr) + + +# ---------- config loading ---------- + +def check_file_permissions(path: str) -> None: + """Warn if `path` is readable beyond the owner (POSIX only). + + On platforms that don't expose POSIX mode bits (Windows), no warning + is emitted — the platform's ACLs are out of scope for this check. + """ + try: + mode = os.stat(path).st_mode + except OSError: + return + if mode & 0o077: + warn( + f"Config file {path} is readable beyond the owner (mode {oct(mode & 0o777)}). " + "Run `chmod 600` (or platform equivalent) to restrict access." + ) + + def load_config(): config_dir = os.environ.get("XDG_CONFIG_HOME", os.path.expanduser("~/.config")) config_dir = os.path.join(config_dir, "agents") @@ -25,60 +109,105 @@ def load_config(): if os.path.isfile(toml_file): config_file = toml_file if tomllib is None: - print( - f"ERROR: TOML config found at {config_file}, but this Python does not support TOML.", - file=sys.stderr, + die( + f"TOML config found at {config_file}, but this Python does not support TOML. " + "Upgrade to Python 3.11+ or remove the TOML config to use legacy JSON fallback." ) - print("Upgrade to Python 3.11+ or remove the TOML config to use legacy JSON fallback.", file=sys.stderr) - sys.exit(1) try: with open(config_file, "rb") as f: config = tomllib.load(f) except tomllib.TOMLDecodeError as e: - print(f"ERROR: Invalid TOML in {config_file}: {e}", file=sys.stderr) - sys.exit(1) + die(f"Invalid TOML in {config_file}: {e}") elif os.path.isfile(json_file): config_file = json_file try: with open(config_file) as f: config = json.load(f) except json.JSONDecodeError as e: - print(f"ERROR: Invalid JSON in {config_file}: {e}", file=sys.stderr) - sys.exit(1) + die(f"Invalid JSON in {config_file}: {e}") else: - print(f"ERROR: Config file not found: {toml_file}", file=sys.stderr) - print( - f"Legacy JSON fallback path: {json_file}", - file=sys.stderr, + die( + f"Config file not found: {toml_file} (legacy JSON fallback path: {json_file}). " + 'Create TOML config with at minimum: base_url = "https://your-searxng-instance.com"' ) - print( - 'Create TOML config with at minimum: base_url = "https://your-searxng-instance.com"', - file=sys.stderr, - ) - sys.exit(1) if not config.get("base_url"): - print(f"ERROR: base_url is required in {config_file}", file=sys.stderr) - sys.exit(1) + die(f"base_url is required in {config_file}") + validate_base_url(config["base_url"], config) + check_file_permissions(config_file) return config def resolve_env(value: str) -> str: - """Resolve $ENV_VAR or ${ENV_VAR} references in a string.""" + """Resolve `$ENV_VAR` or `${ENV_VAR}` references in a string. + + If the variable is unset, exit with a clear error. We do NOT fall back + to using the literal `$VAR` text — that would silently send a + malformed Authorization header. + """ if not value: return value if value.startswith("$"): var_name = value.lstrip("$").strip("{}") resolved = os.environ.get(var_name) if not resolved: - print(f"ERROR: Environment variable {var_name} is not set", file=sys.stderr) - sys.exit(1) + die(f"Environment variable {var_name} is not set") return resolved return value +# ---------- URL validation ---------- + +_LOOPBACK_HOSTS = frozenset({"127.0.0.1", "::1", "localhost", "0.0.0.0", "[::1]"}) + + +def validate_base_url(url: str, config: dict) -> None: + """Validate the configured base_url and exit on policy violation. + + Policy: + - HTTPS is always accepted. + - Plain HTTP to a loopback host is accepted silently (common dev + setup behind a local reverse proxy). + - Plain HTTP to a non-loopback host is rejected unless the config + contains `allow_insecure_http = true`. When the opt-in is set, a + strong warning is emitted because Authorization headers and query + strings will travel in cleartext. + - Schemes other than http/https are rejected. + """ + try: + parsed = urllib.parse.urlparse(url) + except ValueError as e: + die(f"base_url could not be parsed: {e}") + scheme = (parsed.scheme or "").lower() + host = (parsed.hostname or "").lower() + if scheme not in ("http", "https"): + die(f"base_url scheme must be http or https, got {scheme!r}") + if not host: + die("base_url is missing a host") + if scheme == "https": + return + # scheme == "http" + if host in _LOOPBACK_HOSTS: + return + if config.get("allow_insecure_http") is True: + warn( + f"base_url uses plain HTTP for non-loopback host {host!r}. " + "Authorization headers and search queries will travel in cleartext. " + "Switch to https:// unless you have a specific reason to keep HTTP." + ) + return + die( + f"base_url uses plain HTTP for non-loopback host {host!r}, which is " + "rejected by default (Authorization headers would travel in cleartext). " + "Use https://, point at a loopback address, or set `allow_insecure_http = true` " + "in the config (NOT recommended)." + ) + + +# ---------- request building ---------- + def build_request(config: dict, args: argparse.Namespace) -> urllib.request.Request: base_url = config["base_url"].rstrip("/") @@ -111,41 +240,39 @@ def build_request(config: dict, args: argparse.Namespace) -> urllib.request.Requ url = f"{base_url}/search?{urllib.parse.urlencode(params)}" req = urllib.request.Request(url) - # Auth auth = config.get("auth", {}) auth_type = auth.get("type", "") if auth_type == "bearer": token = resolve_env(auth.get("token", "")) if not token: - print("ERROR: auth.token required for bearer auth", file=sys.stderr) - sys.exit(1) + die("auth.token required for bearer auth") req.add_header("Authorization", f"Bearer {token}") elif auth_type == "basic": user = resolve_env(auth.get("user", "")) password = resolve_env(auth.get("pass", "")) if not user or not password: - print("ERROR: auth.user and auth.pass required for basic auth", file=sys.stderr) - sys.exit(1) + die("auth.user and auth.pass required for basic auth") credentials = base64.b64encode(f"{user}:{password}".encode()).decode() req.add_header("Authorization", f"Basic {credentials}") elif auth_type: - print(f"ERROR: Unknown auth.type '{auth_type}'. Use 'bearer' or 'basic'.", file=sys.stderr) - sys.exit(1) + die(f"Unknown auth.type '{auth_type}'. Use 'bearer' or 'basic'.") - # Headers headers = config.get("headers", {}) for key, value in headers.items(): req.add_header(key, value) - # Set a reasonable User-Agent if not already set via headers - if "User-Agent" not in headers: - req.add_header("User-Agent", "searxng-search-skill (+https://github.com/XYenon/agents)") + if "User-Agent" not in headers and "user-agent" not in {h.lower() for h in headers}: + req.add_header( + "User-Agent", + "searxng-search-skill/1.0 (+https://github.com/Fectivnfy112357/MiniMax-Code-Plugins)", + ) return req +# ---------- result formatting ---------- + def format_results(data: dict, max_results: int, page: int): - # Answers — direct answers from engines answers = data.get("answers", []) if answers: print("# Answers\n") @@ -154,7 +281,6 @@ def format_results(data: dict, max_results: int, page: int): if answer: print(f"> {answer}\n") - # Infoboxes — knowledge panel style info for ib in data.get("infoboxes", []): name = ib.get("infobox", "") content = ib.get("content", "") @@ -172,7 +298,6 @@ def format_results(data: dict, max_results: int, page: int): print(f"- [{u['title']}{official}]({u['url']})") print() - # Search results results = data.get("results", []) if not results: print("No results found.") @@ -204,12 +329,10 @@ def format_results(data: dict, max_results: int, page: int): print(f"\n*{' | '.join(meta)}*") print() - # Suggestions suggestions = data.get("suggestions", []) if suggestions: print(f"**Suggestions:** {', '.join(suggestions)}\n") - # Corrections corrections = data.get("corrections", []) if corrections: print(f"**Corrections:** {', '.join(corrections)}\n") @@ -220,6 +343,8 @@ def format_results(data: dict, max_results: int, page: int): print(f"Showing {shown} of {total} results (page {page})") +# ---------- main ---------- + def main(): parser = argparse.ArgumentParser(description="Search the web using SearXNG") parser.add_argument("query", nargs="+", help="Search query") @@ -247,26 +372,23 @@ def main(): with urllib.request.urlopen(req, timeout=timeout) as resp: body = resp.read().decode() except urllib.error.HTTPError as e: - body = e.read().decode() if e.fp else "" - print(f"ERROR: HTTP {e.code}: {body[:500]}", file=sys.stderr) - sys.exit(1) + try: + body = e.read().decode() if e.fp else "" + except Exception: + body = "" + die(f"HTTP {e.code}: {body[:500]}") except urllib.error.URLError as e: - print(f"ERROR: Request failed: {e.reason}", file=sys.stderr) - sys.exit(1) + die(f"Request failed: {e.reason}") except TimeoutError: - print(f"ERROR: Request timed out after {timeout}s", file=sys.stderr) - sys.exit(1) + die(f"Request timed out after {timeout}s") try: data = json.loads(body) except json.JSONDecodeError: - print("ERROR: Invalid JSON response from SearXNG", file=sys.stderr) - print(f"Response: {body[:500]}", file=sys.stderr) - sys.exit(1) + die(f"Invalid JSON response from SearXNG. Response: {body[:500]}") if "error" in data: - print(f"ERROR: SearXNG returned error: {data['error']}", file=sys.stderr) - sys.exit(1) + die(f"SearXNG returned error: {data['error']}") format_results(data, args.max_results, args.page) diff --git a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/__init__.py b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/__init__.py new file mode 100644 index 0000000..c0da114 --- /dev/null +++ b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/__init__.py @@ -0,0 +1 @@ +"""Marks `tests` as a Python package so unittest discovery works.""" diff --git a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/_fixtures.py b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/_fixtures.py new file mode 100644 index 0000000..ae8fb73 --- /dev/null +++ b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/_fixtures.py @@ -0,0 +1,47 @@ +"""Shared test fixtures for the searxng-search test suite.""" +from __future__ import annotations + +import urllib.error + + +class FakeResponse: + """Minimal stand-in for the context manager returned by `urlopen`.""" + + def __init__(self, body: bytes = b"", status: int = 200) -> None: + self._body = body + self.status = status + + def read(self) -> bytes: + return self._body + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +class FakeHTTPError(urllib.error.HTTPError): + """Fake HTTPError for testing the error path. Subclasses + `urllib.error.HTTPError` so the script's `except` clause matches. + """ + + class _FP: + def __init__(self, body: bytes): + self._body = body + + def read(self) -> bytes: + return self._body + + def __init__(self, code: int, body: bytes = b"", reason: str = "HTTP Error") -> None: + super().__init__( + url="http://dummy.local/", + code=code, + msg=reason, + hdrs=None, + fp=FakeHTTPError._FP(body), + ) + self._body = body + + def read(self) -> bytes: + return self._body diff --git a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_auth_header.py b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_auth_header.py new file mode 100644 index 0000000..f3e468d --- /dev/null +++ b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_auth_header.py @@ -0,0 +1,91 @@ +"""Review point 1+4: the Authorization header is constructed correctly for +bearer and basic auth, and the resolved values flow from the config into +the header without leaking into the URL. + +`build_request` is a pure function over `(config, args)`; we drive it +directly without touching the network. +""" +import argparse +import base64 +import os +import sys +import unittest +from unittest import mock + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import search + + +def _args(**over): + base = dict( + query="q", categories=None, engines=None, language=None, + page=1, time_range=None, max_results=None, safesearch=None, + ) + base.update(over) + return argparse.Namespace(**base) + + +class TestBuildRequestAuthHeader(unittest.TestCase): + def test_bearer_header_carries_token(self): + config = {"base_url": "https://searx.example.com", "auth": {"type": "bearer", "token": "abc123"}} + req = search.build_request(config, _args()) + self.assertEqual(req.get_header("Authorization"), "Bearer abc123") + + def test_bearer_resolves_env_var(self): + config = {"base_url": "https://searx.example.com", "auth": {"type": "bearer", "token": "$SEARXNG_TOKEN"}} + with mock.patch.dict(os.environ, {"SEARXNG_TOKEN": "secret-value"}): + req = search.build_request(config, _args()) + self.assertEqual(req.get_header("Authorization"), "Bearer secret-value") + + def test_basic_header_is_base64_user_colon_pass(self): + config = { + "base_url": "https://searx.example.com", + "auth": {"type": "basic", "user": "admin", "pass": "pw"}, + } + req = search.build_request(config, _args()) + creds = req.get_header("Authorization") + self.assertTrue(creds.startswith("Basic ")) + encoded = creds.split(" ", 1)[1] + self.assertEqual(base64.b64decode(encoded).decode(), "admin:pw") + + def test_basic_resolves_env_vars(self): + config = { + "base_url": "https://searx.example.com", + "auth": {"type": "basic", "user": "$SEARXNG_USER", "pass": "$SEARXNG_PASS"}, + } + with mock.patch.dict(os.environ, {"SEARXNG_USER": "u1", "SEARXNG_PASS": "p1"}): + req = search.build_request(config, _args()) + creds = req.get_header("Authorization") + self.assertEqual(base64.b64decode(creds.split(" ", 1)[1]).decode(), "u1:p1") + + def test_no_auth_section_sends_no_authorization_header(self): + config = {"base_url": "https://searx.example.com"} + req = search.build_request(config, _args()) + self.assertIsNone(req.get_header("Authorization")) + + def test_token_does_not_leak_into_url(self): + config = {"base_url": "https://searx.example.com", "auth": {"type": "bearer", "token": "abc123"}} + req = search.build_request(config, _args()) + self.assertNotIn("abc123", req.full_url) + self.assertIn("q=q", req.full_url) + + +class TestUserAgent(unittest.TestCase): + def test_user_agent_mentions_real_repo(self): + req = search.build_request({"base_url": "https://searx.example.com"}, _args()) + ua = req.get_header("User-agent") + self.assertIn("Fectivnfy112357", ua) + self.assertNotIn("XYenon", ua) + + def test_user_agent_overridden_by_config_headers(self): + config = { + "base_url": "https://searx.example.com", + "headers": {"User-Agent": "my-agent/9.9"}, + } + req = search.build_request(config, _args()) + self.assertEqual(req.get_header("User-agent"), "my-agent/9.9") + + +if __name__ == "__main__": + unittest.main() diff --git a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_config.py b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_config.py new file mode 100644 index 0000000..5ca9ecc --- /dev/null +++ b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_config.py @@ -0,0 +1,128 @@ +"""Review point 3: config loading is robust to missing files, malformed +TOML/JSON, missing env vars, and overly-permissive file modes (POSIX). + +We exercise the real config-loading path with `tempfile.TemporaryDirectory` +so the tests cover the actual TOML/JSON parse + env-var resolve + perm-check +pipeline rather than a mock that drifts from the real code. + +`die` and `warn` print to stderr and call `sys.exit`. We capture stderr +directly to verify the message; the exit code is incidental. +""" +import io +import os +import sys +import tempfile +import unittest +from unittest import mock + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import search + + +def _capture(callable_): + """Run `callable_()` with `sys.exit` raising and stderr captured. + Return (exit_code, stderr_text).""" + err = io.StringIO() + rc = {"v": None} + def _exit(code=0): + rc["v"] = code + raise SystemExit(code) + with mock.patch.object(sys, "stderr", err), \ + mock.patch.object(search.sys, "exit", side_effect=_exit): + try: + callable_() + except SystemExit: + pass + return rc["v"], err.getvalue() + + +class TestLoadConfigErrors(unittest.TestCase): + def test_no_toml_no_json_exits(self): + with tempfile.TemporaryDirectory() as d, \ + mock.patch.dict(os.environ, {"XDG_CONFIG_HOME": d}, clear=True): + rc, err = _capture(search.load_config) + self.assertEqual(rc, 1) + self.assertIn("Config file not found", err) + + def test_invalid_toml_exits(self): + with tempfile.TemporaryDirectory() as d, \ + mock.patch.dict(os.environ, {"XDG_CONFIG_HOME": d}, clear=True), \ + mock.patch.object(search, "validate_base_url"), \ + mock.patch.object(search, "check_file_permissions"): + os.makedirs(os.path.join(d, "agents"), exist_ok=True) + with open(os.path.join(d, "agents", "searxng.toml"), "w") as f: + f.write("[auth\ntype = broken") + rc, err = _capture(search.load_config) + self.assertEqual(rc, 1) + self.assertIn("Invalid TOML", err) + + def test_missing_base_url_exits(self): + with tempfile.TemporaryDirectory() as d, \ + mock.patch.dict(os.environ, {"XDG_CONFIG_HOME": d}, clear=True), \ + mock.patch.object(search, "validate_base_url"), \ + mock.patch.object(search, "check_file_permissions"): + os.makedirs(os.path.join(d, "agents"), exist_ok=True) + with open(os.path.join(d, "agents", "searxng.toml"), "w") as f: + f.write('default_categories = ["general"]\n') + rc, err = _capture(search.load_config) + self.assertEqual(rc, 1) + self.assertIn("base_url is required", err) + + def test_valid_config_loads_with_url_check(self): + with tempfile.TemporaryDirectory() as d, \ + mock.patch.dict(os.environ, {"XDG_CONFIG_HOME": d}, clear=True), \ + mock.patch.object(search, "check_file_permissions") as cp: + os.makedirs(os.path.join(d, "agents"), exist_ok=True) + with open(os.path.join(d, "agents", "searxng.toml"), "w") as f: + f.write('base_url = "https://searx.example.com"\n') + config = search.load_config() + self.assertEqual(config["base_url"], "https://searx.example.com") + cp.assert_called_once() + + +class TestResolveEnv(unittest.TestCase): + def test_unset_env_var_exits_with_message(self): + with mock.patch.dict(os.environ, {}, clear=True): + rc, err = _capture(lambda: search.resolve_env("$SEARXNG_TOKEN")) + self.assertEqual(rc, 1) + self.assertIn("SEARXNG_TOKEN is not set", err) + + def test_brace_syntax_is_stripped(self): + with mock.patch.dict(os.environ, {"SEARXNG_TOKEN": "abc"}): + self.assertEqual(search.resolve_env("${SEARXNG_TOKEN}"), "abc") + + def test_plain_string_passes_through(self): + self.assertEqual(search.resolve_env("plain-literal"), "plain-literal") + + def test_empty_string_passes_through(self): + self.assertEqual(search.resolve_env(""), "") + + +class TestFilePermissions(unittest.TestCase): + def test_world_readable_warns(self): + # Drive the check by patching os.stat so the assertion is portable + # across platforms (Windows doesn't preserve 0o600 reliably). + fake_stat = type("S", (), {"st_mode": 0o644})() + with mock.patch.object(search.os, "stat", return_value=fake_stat), \ + mock.patch.object(search, "warn", wraps=search.warn) as w: + search.check_file_permissions("/dummy/path") + self.assertTrue(w.called) + self.assertIn("readable beyond the owner", str(w.call_args)) + + def test_owner_only_does_not_warn(self): + fake_stat = type("S", (), {"st_mode": 0o600})() + with mock.patch.object(search.os, "stat", return_value=fake_stat), \ + mock.patch.object(search, "warn", wraps=search.warn) as w: + search.check_file_permissions("/dummy/path") + self.assertFalse(w.called) + + def test_missing_file_silently_skips(self): + with mock.patch.object(search.os, "stat", side_effect=OSError("missing")), \ + mock.patch.object(search, "warn", wraps=search.warn) as w: + search.check_file_permissions("/does/not/exist") + self.assertFalse(w.called) + + +if __name__ == "__main__": + unittest.main() diff --git a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_invalid_response.py b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_invalid_response.py new file mode 100644 index 0000000..c15070e --- /dev/null +++ b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_invalid_response.py @@ -0,0 +1,96 @@ +"""Review point 4: error paths for non-JSON responses, HTTPError body +echo, and SearXNG structured error response. The script must exit +non-zero and (in the body-echo case) not leak credentials that the +server reflected back. +""" +import argparse +import io +import os +import sys +import unittest +from unittest import mock + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import search +from _fixtures import FakeHTTPError, FakeResponse + + +def _args(**over): + base = dict( + query="q", categories=None, engines=None, language=None, + page=1, time_range=None, max_results=None, safesearch=None, + ) + base.update(over) + return argparse.Namespace(**base) + + +def _capture_main(argv, urlopen_mock, load_config_mock=None): + """Invoke `search.main()` with patched argv / urlopen / load_config, + return (exit_code, stderr_text).""" + if load_config_mock is None: + load_config_mock = mock.Mock(return_value={"base_url": "https://searx.example.com"}) + err = io.StringIO() + with mock.patch.object(sys, "argv", ["search.py"] + argv), \ + mock.patch.object(sys, "stderr", err), \ + mock.patch.object(search, "load_config", load_config_mock), \ + mock.patch.object(search, "build_request", + mock.Mock(return_value=mock.Mock(get_header=mock.Mock(return_value=None)))), \ + mock.patch.object(search.urllib.request, "urlopen", urlopen_mock), \ + mock.patch.object(search.sys, "exit", side_effect=SystemExit) as e: + try: + search.main() + except SystemExit: + pass + return e.call_args[0][0] if e.call_args else None, err.getvalue() + + +class TestInvalidResponse(unittest.TestCase): + def test_non_json_response_exits_with_diagnostic(self): + body = b"oops not json" + urlopen = mock.Mock(return_value=FakeResponse(body)) + rc, err = _capture_main(["q"], urlopen) + self.assertEqual(rc, 1) + self.assertIn("Invalid JSON", err) + self.assertIn("oops not json", err) + + def test_searxng_structured_error_exits(self): + body = b'{"error": "instance disabled"}' + urlopen = mock.Mock(return_value=FakeResponse(body)) + rc, err = _capture_main(["q"], urlopen) + self.assertEqual(rc, 1) + self.assertIn("SearXNG returned error", err) + self.assertIn("instance disabled", err) + + def test_http_error_echoes_body_redacted(self): + # The "server" reflects back an Authorization header in its error body. + body = b'{"detail": "got Authorization: Basic ' + (b"A" * 60) + b'"}' + urlopen = mock.Mock(side_effect=FakeHTTPError(401, body)) + rc, err = _capture_main(["q"], urlopen) + self.assertEqual(rc, 1) + self.assertIn("HTTP 401", err) + # The reflected credential must be masked, not echoed. + self.assertNotIn(b"A" * 60, err.encode()) + self.assertIn("Authorization: Basic ***", err) + + def test_url_error_reports_reason(self): + from urllib.error import URLError + urlopen = mock.Mock(side_effect=URLError("dns failure")) + rc, err = _capture_main(["q"], urlopen) + self.assertEqual(rc, 1) + self.assertIn("Request failed", err) + self.assertIn("dns failure", err) + + def test_timeout_error_reports_configured_timeout(self): + urlopen = mock.Mock(side_effect=TimeoutError("")) + # We can't easily test the actual timeout from urlopen's perspective + # without hitting the network, but we can verify the error message + # uses the configured value. + rc, err = _capture_main(["q"], urlopen) + self.assertEqual(rc, 1) + self.assertIn("Request timed out", err) + self.assertIn("30s", err) # default + + +if __name__ == "__main__": + unittest.main() diff --git a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_redaction.py b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_redaction.py new file mode 100644 index 0000000..094c108 --- /dev/null +++ b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_redaction.py @@ -0,0 +1,63 @@ +"""Review point 4: on error, sensitive info that leaked into stderr is masked. + +`redact_secrets` covers 5 credential shapes. `die` and `warn` route +through it before printing. We also assert the `Authorization: Basic` +header value is masked in error output. +""" +import contextlib +import io +import os +import sys +import unittest +from unittest import mock + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import search + + +TOKEN = "ghp_1234567890abcdefghijklmnopqrstuvwxyz" +FPAT = "github_pat_ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz" +BEARER_FULL = "Bearer " + TOKEN +BASIC_FULL = "Authorization: Basic " + ("A" * 60) + + +class TestRedactSecrets(unittest.TestCase): + def test_redacts_credential_shapes(self): + cases = { + TOKEN: "ghx_***", + FPAT: "github_pat_***", + BEARER_FULL: "Bearer ***", + "token=supersecretvalue": "token=***", + "SEARXNG_TOKEN=ghp_abc": "SEARXNG_TOKEN=***", + } + for raw, expected in cases.items(): + self.assertEqual(search.redact_secrets(raw), expected, raw) + + def test_redacts_authorization_basic_header_value(self): + self.assertEqual(search.redact_secrets(BASIC_FULL), "Authorization: Basic ***") + + def test_leaves_ordinary_text_intact(self): + msg = "HTTP 401: unauthorized (attempt 3/3)" + self.assertEqual(search.redact_secrets(msg), msg) + + +class TestStderrRedaction(unittest.TestCase): + def test_die_redacts_token_and_exits(self): + err = io.StringIO() + with mock.patch.object(sys, "stderr", err), self.assertRaises(SystemExit) as cm: + search.die("failed: " + TOKEN) + self.assertEqual(cm.exception.code, 1) + self.assertNotIn(TOKEN, err.getvalue()) + self.assertIn("***", err.getvalue()) + + def test_warn_redacts_authorization_basic(self): + err = io.StringIO() + with mock.patch.object(sys, "stderr", err): + search.warn("upstream sent: " + BASIC_FULL) + self.assertNotIn("A" * 60, err.getvalue()) + self.assertIn("Authorization: Basic ***", err.getvalue()) + + +if __name__ == "__main__": + unittest.main() diff --git a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_timeout.py b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_timeout.py new file mode 100644 index 0000000..207743b --- /dev/null +++ b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_timeout.py @@ -0,0 +1,42 @@ +"""Review point 4: the configured `timeout` is passed through to +`urllib.request.urlopen`. We mock urlopen and inspect the call. +""" +import os +import sys +import unittest +from unittest import mock + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import search +from _fixtures import FakeResponse + + +def _run_main(load_config_return, argv=("search.py", "q")): + """Run `search.main()` with all network/config side effects mocked, + and return the urlopen mock so the test can inspect its call args.""" + u = mock.Mock(return_value=FakeResponse(b'{"results":[]}')) + with mock.patch.object(sys, "argv", list(argv)), \ + mock.patch.object(search, "load_config", return_value=load_config_return), \ + mock.patch.object(search, "build_request", return_value=mock.Mock()), \ + mock.patch.object(search.urllib.request, "urlopen", u), \ + mock.patch.object(search.sys, "exit", side_effect=SystemExit): + try: + search.main() + except SystemExit: + pass + return u + + +class TestTimeoutConfig(unittest.TestCase): + def test_default_timeout_is_30(self): + u = _run_main({"base_url": "https://x"}) + self.assertEqual(u.call_args.kwargs.get("timeout"), 30) + + def test_custom_timeout_is_passed_through(self): + u = _run_main({"base_url": "https://x", "timeout": 7}) + self.assertEqual(u.call_args.kwargs.get("timeout"), 7) + + +if __name__ == "__main__": + unittest.main() diff --git a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_url_validation.py b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_url_validation.py new file mode 100644 index 0000000..1c31e37 --- /dev/null +++ b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_url_validation.py @@ -0,0 +1,92 @@ +"""Review point 1: base_url must default to HTTPS; plain HTTP is allowed +only for loopback hosts or for non-loopback hosts with explicit opt-in. + +`validate_base_url` exits on policy violation. We assert that: + - https to any host passes silently. + - http to loopback (127.0.0.1, ::1, localhost, 0.0.0.0) passes silently. + - http to a non-loopback host is rejected unless `allow_insecure_http = true`. + - when opted in, a warning is emitted and the call still passes. + - any other scheme (ftp, file, ...) is rejected. +""" +import contextlib +import io +import os +import sys +import unittest +from unittest import mock + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import search +from _fixtures import FakeResponse # noqa: F401 (imported for parity with other test files) + + +def _captured(callable_, *args, **kwargs): + """Run `callable_(args, kwargs)` and capture stderr/stdout. Returns + (returncode, stderr, stdout).""" + err, out = io.StringIO(), io.StringIO() + rc = {"v": None} + def _exit(code=0): + rc["v"] = code + raise SystemExit(code) + with mock.patch.object(search, "warn", wraps=search.warn) as _w, \ + mock.patch.object(search.sys, "stderr", err), \ + mock.patch.object(search.sys, "stdout", out), \ + mock.patch.object(search.sys, "exit", side_effect=_exit): + try: + callable_(*args, **kwargs) + except SystemExit: + pass + return rc["v"], err.getvalue(), out.getvalue() + + +class TestValidateBaseUrl(unittest.TestCase): + def test_https_to_any_host_passes(self): + rc, err, _ = _captured(search.validate_base_url, "https://searx.example.com", {}) + self.assertIsNone(rc, f"https should pass silently, got rc={rc} err={err!r}") + + def test_http_to_ipv4_loopback_passes(self): + rc, err, _ = _captured(search.validate_base_url, "http://127.0.0.1:8888", {}) + self.assertIsNone(rc, f"loopback ipv4 should pass, got rc={rc} err={err!r}") + + def test_http_to_localhost_passes(self): + rc, err, _ = _captured(search.validate_base_url, "http://localhost:8080", {}) + self.assertIsNone(rc, f"localhost should pass, got rc={rc} err={err!r}") + + def test_http_to_ipv6_loopback_passes(self): + rc, err, _ = _captured(search.validate_base_url, "http://[::1]:8080/", {}) + self.assertIsNone(rc, f"ipv6 loopback should pass, got rc={rc} err={err!r}") + + def test_http_to_non_loopback_is_rejected(self): + rc, err, _ = _captured(search.validate_base_url, "http://searx.example.com", {}) + self.assertEqual(rc, 1) + self.assertIn("rejected by default", err) + self.assertIn("searx.example.com", err) + self.assertIn("https://", err) + self.assertIn("loopback", err) + self.assertIn("allow_insecure_http", err) + + def test_http_to_non_loopback_with_opt_in_passes_with_warning(self): + rc, err, _ = _captured(search.validate_base_url, "http://searx.example.com", {"allow_insecure_http": True}) + self.assertIsNone(rc) + self.assertIn("cleartext", err) + self.assertIn("searx.example.com", err) + + def test_ftp_scheme_is_rejected(self): + rc, err, _ = _captured(search.validate_base_url, "ftp://searx.example.com/", {}) + self.assertEqual(rc, 1) + self.assertIn("scheme must be http or https", err) + + def test_empty_scheme_is_rejected(self): + rc, err, _ = _captured(search.validate_base_url, "searx.example.com", {}) + self.assertEqual(rc, 1) + self.assertIn("scheme", err) + + def test_missing_host_is_rejected(self): + rc, err, _ = _captured(search.validate_base_url, "https://", {}) + self.assertEqual(rc, 1) + self.assertIn("host", err) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/searxng-search.test.mjs b/test/searxng-search.test.mjs new file mode 100644 index 0000000..998ad0e --- /dev/null +++ b/test/searxng-search.test.mjs @@ -0,0 +1,32 @@ +// Bridge from the repo's `node --test` to the Python regression suite. +// Spawns `python run_tests.py` and asserts a zero exit code. No network, +// no gh binary required — the Python suite is self-contained. +import { test } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const SCRIPT_DIR = join( + __dirname, + '..', + 'plugins', + 'Fectivnfy112357', + 'searxng-search', + 'skills', + 'searxng-search', + 'scripts', +); + +test('searxng-search Python regression tests', () => { + const r = spawnSync('python', [join(SCRIPT_DIR, 'run_tests.py')], { + cwd: SCRIPT_DIR, + encoding: 'utf-8', + }); + if (r.status !== 0) { + console.error(r.stdout); + console.error(r.stderr); + } + assert.equal(r.status, 0, 'python run_tests.py must exit 0'); +}); From b31a1040c1b82b7f4f2f58bda3cd182ed2c4ca95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B4=BE=E6=99=93=E6=BA=90?= Date: Sat, 15 Aug 2026 15:40:47 +0800 Subject: [PATCH 3/4] fix(searxng-search): skip file-perm check on non-POSIX platforms The previous implementation always ran `os.stat().st_mode & 0o077`, but on Windows `st_mode` is a synthetic value (typically 0o666) and is not derived from the file's real ACL. The docstring already promised to skip the check on non-POSIX; the implementation now matches by guarding with `os.name != "posix"`. The two POSIX-only tests now self-skip on Windows, and a new `test_skipped_on_non_posix` locks in the no-op behavior. Total test count goes from 40 to 41 (2 skipped on Windows). Found by real test against `http://textvision.top:40001/` (a SearXNG 2026.8.3 instance) where the spurious permission warning fired on every run despite the fix path not being available on Windows. --- .../skills/searxng-search/scripts/search.py | 9 +++++++-- .../searxng-search/scripts/tests/test_config.py | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/search.py b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/search.py index f6b7a76..2a29b4d 100644 --- a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/search.py +++ b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/search.py @@ -86,9 +86,14 @@ def info(msg: str) -> None: def check_file_permissions(path: str) -> None: """Warn if `path` is readable beyond the owner (POSIX only). - On platforms that don't expose POSIX mode bits (Windows), no warning - is emitted — the platform's ACLs are out of scope for this check. + On platforms that don't expose POSIX mode bits (Windows, where + `os.stat().st_mode` is a synthetic mode not derived from real ACLs), + the check is skipped — restricting the file must be done via + Properties → Security. The docstring was previously aspirational; + the implementation now matches. """ + if os.name != "posix": + return try: mode = os.stat(path).st_mode except OSError: diff --git a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_config.py b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_config.py index 5ca9ecc..4975bd5 100644 --- a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_config.py +++ b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_config.py @@ -101,6 +101,8 @@ def test_empty_string_passes_through(self): class TestFilePermissions(unittest.TestCase): def test_world_readable_warns(self): + if os.name != "posix": + self.skipTest("POSIX-mode-bits check is skipped on non-POSIX platforms") # Drive the check by patching os.stat so the assertion is portable # across platforms (Windows doesn't preserve 0o600 reliably). fake_stat = type("S", (), {"st_mode": 0o644})() @@ -111,6 +113,8 @@ def test_world_readable_warns(self): self.assertIn("readable beyond the owner", str(w.call_args)) def test_owner_only_does_not_warn(self): + if os.name != "posix": + self.skipTest("POSIX-mode-bits check is skipped on non-POSIX platforms") fake_stat = type("S", (), {"st_mode": 0o600})() with mock.patch.object(search.os, "stat", return_value=fake_stat), \ mock.patch.object(search, "warn", wraps=search.warn) as w: @@ -123,6 +127,18 @@ def test_missing_file_silently_skips(self): search.check_file_permissions("/does/not/exist") self.assertFalse(w.called) + def test_skipped_on_non_posix(self): + # Even with a "world-readable" stat, the check is a no-op on + # non-POSIX because `st_mode` is a synthetic value on Windows + # and not derived from the file's real ACL. + if os.name == "posix": + self.skipTest("POSIX-only assertion") + fake_stat = type("S", (), {"st_mode": 0o644})() + with mock.patch.object(search.os, "stat", return_value=fake_stat), \ + mock.patch.object(search, "warn", wraps=search.warn) as w: + search.check_file_permissions("/dummy/path") + self.assertFalse(w.called) + if __name__ == "__main__": unittest.main() From c80de45af4e6b9d3a5394aeb492f2085fe939ee5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B4=BE=E6=99=93=E6=BA=90?= Date: Mon, 17 Aug 2026 12:28:21 +0800 Subject: [PATCH 4/4] =?UTF-8?q?fix(searxng-search):=20address=20PR=20#9=20?= =?UTF-8?q?round-2=20review=20=E2=80=94=20redirects,=20redaction,=20portab?= =?UTF-8?q?le=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-2 P1 fixes from reviewer hetaoBackend: - Block all HTTP redirects (_SafeRedirectHandler, replaces urllib's default handler): a 30x response is an error with a safe diagnostic, so the Authorization header can never be forwarded to a different origin or downgraded HTTPS->HTTP. Regression tests + live loopback cross-port 302 check. - Exact-secret redaction: register_secret()/reset_active_secrets() track the actual token/password/base64 used for the request; redact_secrets substitutes them by value before shape-based fallback, catching short or non-shape tokens. 401/403/407 response bodies are suppressed entirely so a server that reflects credentials cannot leak them. - Cross-platform interpreter discovery (py -3 / python / python3, each verified via --version) with an explicit failure when none is found, instead of a misleading 26/27 pass. The node bridge now lives inside the Plugin (plugins///test/) so the contribution stays self-contained; the repo-root bridge was removed. Additional hardening found in self-review: - Cap response/error bodies at 10 MiB (MAX_RESPONSE_BYTES). - Validate numeric config fields (timeout, default_max_results) so a string value exits cleanly instead of raising a raw TypeError. - Require a JSON object root from the instance; reject non-UTF-8 bodies with a clean diagnostic instead of a traceback. - Custom config headers now resolve $ENV_VAR references and register resolved values for exact redaction, matching auth.* handling. - Remove unused info() helper; add close() to the HTTPError fixture to silence urllib addbase GC cleanup noise. Docs: SKILL.md/README disclose that redirects are never followed; configuration.md documents $ENV_VAR in headers and a redirect-blocked troubleshooting entry. Test suite: 77 stdlib unittest cases (2 POSIX-only skips), wired into npm test via the in-plugin node bridge; validate.mjs passes. --- .../Fectivnfy112357/searxng-search/README.md | 5 +- .../skills/searxng-search/SKILL.md | 6 +- .../references/configuration.md | 16 +- .../skills/searxng-search/scripts/search.py | 242 +++++++++++++++++- .../searxng-search/scripts/tests/_fixtures.py | 24 +- .../scripts/tests/test_auth_header.py | 58 +++++ .../scripts/tests/test_config.py | 55 ++++ .../scripts/tests/test_invalid_response.py | 140 ++++++++-- .../scripts/tests/test_redaction.py | 111 ++++++++ .../scripts/tests/test_redirect.py | 198 ++++++++++++++ .../scripts/tests/test_timeout.py | 15 +- .../test/searxng-search.test.mjs | 72 ++++++ test/searxng-search.test.mjs | 32 --- 13 files changed, 891 insertions(+), 83 deletions(-) create mode 100644 plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_redirect.py create mode 100644 plugins/Fectivnfy112357/searxng-search/test/searxng-search.test.mjs delete mode 100644 test/searxng-search.test.mjs diff --git a/plugins/Fectivnfy112357/searxng-search/README.md b/plugins/Fectivnfy112357/searxng-search/README.md index 216892f..cf2e58f 100644 --- a/plugins/Fectivnfy112357/searxng-search/README.md +++ b/plugins/Fectivnfy112357/searxng-search/README.md @@ -47,7 +47,10 @@ This skill has **two levels of network destinations**; the difference matters. The script makes a single HTTPS (or loopback HTTP) request to **your configured SearXNG instance** — the `base_url` in your config. The -script does not contact any other host. +script does not contact any other host. It never follows HTTP +redirects: a 30x response is an error, because following a redirect +could forward the Authorization header to an unexpected host. Point +`base_url` directly at the final endpoint. ### Downstream destinations (the SearXNG instance, not the script) diff --git a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/SKILL.md b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/SKILL.md index ca0a0db..1667aa9 100644 --- a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/SKILL.md +++ b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/SKILL.md @@ -19,7 +19,11 @@ Two levels — both matter: for loopback hosts (127.0.0.1, ::1, localhost) or for non-loopback hosts if the config contains `allow_insecure_http = true` (with a warning). If `auth.type` is set, the Authorization header travels to - that single host and nowhere else. + that single host and nowhere else. **Redirects are never followed**: a + 30x response is an error, because following a redirect could forward + the Authorization header to a host you did not configure. Point + `base_url` directly at the final endpoint (or front the instance + with a same-origin reverse proxy). - **Downstream (your SearXNG instance):** the instance then forwards the **query**, **language**, and **categories** to the engines it itself has been configured with (Google, Bing, DuckDuckGo, Brave, diff --git a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/references/configuration.md b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/references/configuration.md index 94112f6..9798929 100644 --- a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/references/configuration.md +++ b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/references/configuration.md @@ -56,7 +56,7 @@ permissive ACLs on Windows. | `auth.token` | `string` | If `auth.type = "bearer"` | Bearer token value; supports `$ENV_VAR` or `${ENV_VAR}` (recommended) | | `auth.user` | `string` | If `auth.type = "basic"` | Basic auth username; supports `$ENV_VAR` or `${ENV_VAR}` (recommended) | | `auth.pass` | `string` | If `auth.type = "basic"` | Basic auth password; supports `$ENV_VAR` or `${ENV_VAR}` (recommended) | -| `headers` | `object` | No | Extra HTTP headers to send with the request | +| `headers` | `object` | No | Extra HTTP headers to send with the request; values support `$ENV_VAR` / `\${ENV_VAR}` references (recommended for keys) | | `default_language` | `string` | No | Default language code, for example `en` or `zh-CN` | | `default_categories` | `string[]` | No | Default categories, for example `["general", "news"]` | | `default_engines` | `string[]` | No | Default engines, for example `["google", "duckduckgo"]` | @@ -296,6 +296,20 @@ Checks: - make sure the instance is reachable from your machine - check VPN, proxy, DNS, or certificate issues +### `ERROR: HTTP 30x: redirect blocked: ...` + +The instance (or a load balancer in front of it) answered with a 30x +redirect. The script never follows redirects — urllib's default redirect +handler would forward the `Authorization` header to the redirect +target, so redirects are treated as errors on purpose. + +Checks: + +- point `base_url` directly at the final endpoint (the URL that + answers `/search?format=json` with 200) +- if your instance is behind a load balancer that 302s, front it with a + same-origin reverse proxy that terminates the redirect instead + ### `ERROR: Invalid JSON response from SearXNG` The endpoint responded, but not with valid JSON. This often means the URL points to the wrong place or a proxy/login page returned HTML. diff --git a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/search.py b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/search.py index 2a29b4d..4eb1092 100644 --- a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/search.py +++ b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/search.py @@ -12,6 +12,20 @@ plaintext never lives in the config file. - All error output is run through `redact_secrets` so a token that leaks into a server response body is masked before reaching stderr. + The set of *exact* secrets used for the current request is tracked + so that even short/non-shape-matching tokens are masked (shape-based + patterns are a fallback, not the primary defense). + - HTTP redirects are blocked entirely. A 30x response raises + HTTPError with a "redirect blocked" diagnostic. This prevents the + default urllib redirect handler from forwarding Authorization + headers to a different origin (e.g. an initial loopback URL + returning 302 to a different port), and prevents silent HTTPS→HTTP + downgrades. Configure base_url to point at the final endpoint, or + front the instance with a same-origin reverse proxy. + - HTTPError bodies for authentication-class status codes (401, 403, + 407) are NOT echoed to stderr. These responses frequently reflect + back the credentials the server saw, and the shape-based redaction + can be bypassed by tokens that don't match a known pattern. - On POSIX, a config file readable beyond the owner (mode & 0o077) is flagged with a warning. """ @@ -35,6 +49,34 @@ # ---------- error / progress helpers ---------- +# Per-process registry of exact credential strings used by the current +# request. `redact_secrets` masks these BEFORE applying shape-based +# patterns, so even a short token that doesn't match any known shape +# is removed from the transcript. Cleared on every call to +# `reset_active_secrets` (driven by `load_config`). +_active_secrets: set[str] = set() + + +def reset_active_secrets() -> None: + """Clear the per-request secret registry. + + Called by `load_config` so a config reload does not let an old + secret linger in the redaction set indefinitely. + """ + _active_secrets.clear() + + +def register_secret(value: str) -> None: + """Record `value` as a secret to mask in any subsequent error output. + + No-op for empty values. The set is deduplicated, so a token that + was registered once does not need to be re-registered on each call. + """ + if not value: + return + _active_secrets.add(value) + + # Credential shapes that must never reach the transcript. Same pattern set # as the github-explore skill (PR #8) so the agent learns one redaction # grammar across skills. Order matters: the most specific label-shaped @@ -51,15 +93,30 @@ def redact_secrets(text: str) -> str: - """Mask credential-shaped substrings in `text`. - - Best-effort: only obvious credential shapes are masked, so ordinary - error text (URLs without embedded tokens, JSON error bodies, rate-limit - messages) passes through intact. + """Mask credential substrings in `text`. + + Two-pass strategy: + 1. Exact substitution of every value registered via + `register_secret` (the actual token/password/basic-credential + used for the current request). This catches short or otherwise + shape-non-conforming secrets that pattern matching would miss. + 2. Shape-based pattern matching for stray credentials that escaped + the registry (e.g. embedded in a third-party server response + that the script never used). + + Best-effort: ordinary error text (URLs without embedded tokens, + JSON error bodies, rate-limit messages) passes through intact. """ if not text: return text out = text + # Pass 1: exact secret values from the active registry. Sort by + # length descending so a substring secret is replaced before its + # containing string (e.g. "abc" before "abcdef"). + for secret in sorted(_active_secrets, key=len, reverse=True): + if secret and secret in out: + out = out.replace(secret, "***") + # Pass 2: shape-based fallback. for pat, repl in _SECRET_PATTERNS: out = pat.sub(repl, out) return out @@ -76,12 +133,25 @@ def warn(msg: str) -> None: print(f"WARN: {redact_secrets(msg)}", file=sys.stderr) -def info(msg: str) -> None: - """Progress message to stderr so stdout stays pipeable.""" - print(f"… {msg}", file=sys.stderr) +# ---------- config loading ---------- +def _require_int(config: dict, key: str, default: int) -> int: + """Return `config[key]` if it is an integer, otherwise exit clearly. + + TOML is typed, but a hand-edited config can contain `timeout = "30"` + or `default_max_results = "5"`. Passing such a value to urllib (or + using it to slice results) raises a raw `TypeError` traceback that + bypasses the redacted-error pipeline; fail with a clear message + instead. Booleans are rejected too (`True` is an `int` subclass). + """ + value = config.get(key, default) + if isinstance(value, bool) or not isinstance(value, int): + die( + f"config field {key!r} must be an integer, " + f"got {type(value).__name__} ({value!r})" + ) + return value -# ---------- config loading ---------- def check_file_permissions(path: str) -> None: """Warn if `path` is readable beyond the owner (POSIX only). @@ -142,6 +212,13 @@ def load_config(): validate_base_url(config["base_url"], config) check_file_permissions(config_file) + # Fail fast on wrong-typed numeric fields instead of surfacing a raw + # TypeError from urllib or from result slicing later. + _require_int(config, "timeout", 30) + _require_int(config, "default_max_results", 5) + # Drop any secrets left over from a previous run; the new config + # will re-register its own values during build_request. + reset_active_secrets() return config @@ -211,6 +288,105 @@ def validate_base_url(url: str, config: dict) -> None: ) +# ---------- request execution (safe opener) ---------- + +# Upper bound on how many bytes we will read from the instance before +# failing. SearXNG search responses are typically a few hundred KB at +# most; an unbounded read lets a misbehaving or hijacked endpoint exhaust +# memory/disk. The error body read is capped with the same constant. +MAX_RESPONSE_BYTES = 10 * 1024 * 1024 # 10 MiB + +# Status codes whose response body we suppress. These codes mean +# "authentication challenge" and the response body frequently reflects +# back the credentials the server saw, in arbitrary shapes that a +# pattern-based redactor cannot reliably cover. Combined with the +# exact-secret registry in `redact_secrets`, suppressing the body for +# these codes closes the residual leak surface. +_AUTH_ERROR_CODES = frozenset({401, 403, 407}) + + +class _SafeRedirectHandler(urllib.request.HTTPRedirectHandler): + """Block all 3xx responses with a clear diagnostic. + + Rationale: the default `urllib.request.HTTPRedirectHandler` will + transparently re-issue the request to a `Location:` URL, INCLUDING + the `Authorization` header from the original request. A misconfigured + `base_url` (e.g. an instance fronted by a load balancer that 302s to + a different port) would leak the Bearer/Basic token to a host the + user never explicitly trusted. It also permits silent HTTPS→HTTP + downgrades, which violates the same-origin HTTPS guarantee the + validator establishes for the initial request. + + Instead of trying to police every hop (same-origin? same-port? + same-scheme? didn't the user already pass validation for the *first* + URL?), we reject all redirects and tell the user to point `base_url` + at the final endpoint, or front the instance with a same-origin + reverse proxy. + """ + + _REDIRECT_LABELS = { + 301: "301 Moved Permanently", + 302: "302 Found", + 303: "303 See Other", + 307: "307 Temporary Redirect", + 308: "308 Permanent Redirect", + } + + def _block(self, req, code, msg, headers): + # Show a *safe* form of Location: scheme://host[:port]/path, no + # query or fragment, so the diagnostic is useful without + # potentially echoing embedded credentials. + loc_raw = headers.get("Location", "") if headers else "" + safe_loc = "" + if loc_raw: + try: + p = urllib.parse.urlparse(loc_raw) + host = p.hostname or "" + if p.port: + host = f"{host}:{p.port}" + safe_loc = f"{p.scheme}://{host}{p.path}" + except Exception: + safe_loc = "(unparseable)" + label = self._REDIRECT_LABELS.get(code, f"{code}") + raise urllib.error.HTTPError( + req.get_full_url(), + code, + f"redirect blocked: {label} to {safe_loc or '(none)'}. " + "Configure base_url to point directly at the final endpoint, " + "or front the instance with a same-origin reverse proxy. " + "This skill does not follow redirects because the " + "Authorization header would otherwise be forwarded to the " + "redirect target.", + headers, + None, + ) + + def http_error_301(self, req, fp, code, msg, headers): + return self._block(req, code, msg, headers) + + def http_error_302(self, req, fp, code, msg, headers): + return self._block(req, code, msg, headers) + + def http_error_303(self, req, fp, code, msg, headers): + return self._block(req, code, msg, headers) + + def http_error_307(self, req, fp, code, msg, headers): + return self._block(req, code, msg, headers) + + def http_error_308(self, req, fp, code, msg, headers): + return self._block(req, code, msg, headers) + + +def _build_opener(): + """Construct a urllib opener that uses `_SafeRedirectHandler`. + + We pass our handler to `build_opener` so it replaces the default + redirect handler (rather than chaining on top of it). All other + default handlers (HTTPSHandler, HTTPHandler, etc.) are kept. + """ + return urllib.request.build_opener(_SafeRedirectHandler()) + + # ---------- request building ---------- def build_request(config: dict, args: argparse.Namespace) -> urllib.request.Request: @@ -251,20 +427,32 @@ def build_request(config: dict, args: argparse.Namespace) -> urllib.request.Requ token = resolve_env(auth.get("token", "")) if not token: die("auth.token required for bearer auth") + register_secret(token) req.add_header("Authorization", f"Bearer {token}") elif auth_type == "basic": user = resolve_env(auth.get("user", "")) password = resolve_env(auth.get("pass", "")) if not user or not password: die("auth.user and auth.pass required for basic auth") + register_secret(user) + register_secret(password) credentials = base64.b64encode(f"{user}:{password}".encode()).decode() + register_secret(credentials) # the base64 form may also be reflected by the server req.add_header("Authorization", f"Basic {credentials}") elif auth_type: die(f"Unknown auth.type '{auth_type}'. Use 'bearer' or 'basic'.") headers = config.get("headers", {}) for key, value in headers.items(): - req.add_header(key, value) + if not isinstance(value, str): + die(f"config field {key!r} in headers must be a string, got {type(value).__name__}") + resolved = resolve_env(value) + if value.startswith("$"): + # An explicit $ENV_VAR reference marks this value as a + # secret: register the resolved value for exact redaction in + # any later error output, exactly like auth.* values. + register_secret(resolved) + req.add_header(key, resolved) if "User-Agent" not in headers and "user-agent" not in {h.lower() for h in headers}: req.add_header( @@ -372,13 +560,38 @@ def main(): timeout = config.get("timeout", 30) req = build_request(config, args) + opener = _build_opener() try: - with urllib.request.urlopen(req, timeout=timeout) as resp: - body = resp.read().decode() + with opener.open(req, timeout=timeout) as resp: + raw = resp.read(MAX_RESPONSE_BYTES + 1) + if len(raw) > MAX_RESPONSE_BYTES: + die(f"SearXNG response exceeded the {MAX_RESPONSE_BYTES}-byte limit") + try: + body = raw.decode("utf-8") + except UnicodeDecodeError: + die("SearXNG returned a non-UTF-8 response body") except urllib.error.HTTPError as e: + # Authentication-class responses (401/403/407) frequently + # reflect back the credentials the server saw. Even with + # exact-secret redaction, refusing to echo the body for these + # codes removes a residual leak surface. + if e.code in _AUTH_ERROR_CODES: + die( + f"HTTP {e.code}: body suppressed to avoid leaking " + "credentials (check base_url, auth credentials, and " + "SearXNG instance configuration)" + ) + # Redirect-class responses (30x) only reach us via + # `_SafeRedirectHandler`, which raises HTTPError with fp=None + # and a diagnostic in `e.msg` / `e.reason`. Show that diagnostic + # instead of an empty body so the user knows how to fix + # `base_url`. + if 300 <= e.code < 400: + diagnostic = getattr(e, "msg", None) or str(e) or "redirect blocked" + die(f"HTTP {e.code}: {diagnostic}") try: - body = e.read().decode() if e.fp else "" + body = e.read(MAX_RESPONSE_BYTES + 1).decode() if e.fp else "" except Exception: body = "" die(f"HTTP {e.code}: {body[:500]}") @@ -392,6 +605,9 @@ def main(): except json.JSONDecodeError: die(f"Invalid JSON response from SearXNG. Response: {body[:500]}") + if not isinstance(data, dict): + die("Invalid JSON response from SearXNG: expected a JSON object at the top level") + if "error" in data: die(f"SearXNG returned error: {data['error']}") diff --git a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/_fixtures.py b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/_fixtures.py index ae8fb73..92baa42 100644 --- a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/_fixtures.py +++ b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/_fixtures.py @@ -11,8 +11,10 @@ def __init__(self, body: bytes = b"", status: int = 200) -> None: self._body = body self.status = status - def read(self) -> bytes: - return self._body + def read(self, size: int = -1) -> bytes: + if size is None or size < 0: + return self._body + return self._body[:size] def __enter__(self): return self @@ -30,8 +32,16 @@ class _FP: def __init__(self, body: bytes): self._body = body - def read(self) -> bytes: - return self._body + def read(self, size: int = -1) -> bytes: + if size is None or size < 0: + return self._body + return self._body[:size] + + def close(self) -> None: + """No-op so urllib's addbase (_TemporaryFileWrapper) GC + cleanup does not raise AttributeError when the FakeHTTPError + is garbage-collected.""" + return None def __init__(self, code: int, body: bytes = b"", reason: str = "HTTP Error") -> None: super().__init__( @@ -43,5 +53,7 @@ def __init__(self, code: int, body: bytes = b"", reason: str = "HTTP Error") -> ) self._body = body - def read(self) -> bytes: - return self._body + def read(self, size: int = -1) -> bytes: + if size is None or size < 0: + return self._body + return self._body[:size] diff --git a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_auth_header.py b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_auth_header.py index f3e468d..9c26b5f 100644 --- a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_auth_header.py +++ b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_auth_header.py @@ -7,6 +7,7 @@ """ import argparse import base64 +import io import os import sys import unittest @@ -71,6 +72,63 @@ def test_token_does_not_leak_into_url(self): self.assertIn("q=q", req.full_url) +def _get_header_ci(req, name): + """urllib stores header names via `key.capitalize()` and `get_header` + is an exact dict lookup, so look the value up case-insensitively.""" + for key, value in req.header_items(): + if key.lower() == name.lower(): + return value + return None + + +class TestBuildRequestCustomHeaders(unittest.TestCase): + """Custom `headers` config values get the same treatment as auth.*: + `$ENV_VAR` references are resolved (instead of being sent as the + literal `$NAME` text), and env-referenced values are registered for + exact redaction in later error output.""" + + def test_plain_header_passes_through(self): + config = {"base_url": "https://searx.example.com", "headers": {"X-Custom": "value"}} + req = search.build_request(config, _args()) + self.assertEqual(_get_header_ci(req, "X-Custom"), "value") + + def test_env_var_header_is_resolved(self): + config = {"base_url": "https://searx.example.com", "headers": {"X-API-Key": "$SEARXNG_API_KEY"}} + with mock.patch.dict(os.environ, {"SEARXNG_API_KEY": "hunter2"}): + req = search.build_request(config, _args()) + self.assertEqual(_get_header_ci(req, "X-API-Key"), "hunter2") + + def test_env_var_header_registers_secret_for_redaction(self): + search.reset_active_secrets() + config = {"base_url": "https://searx.example.com", "headers": {"X-API-Key": "$SEARXNG_API_KEY"}} + with mock.patch.dict(os.environ, {"SEARXNG_API_KEY": "hunter2"}): + search.build_request(config, _args()) + out = search.redact_secrets("server echoed hunter2 in error body") + self.assertNotIn("hunter2", out) + self.assertIn("***", out) + search.reset_active_secrets() + + def test_missing_env_var_header_exits(self): + err = io.StringIO() + config = {"base_url": "https://searx.example.com", "headers": {"X-Key": "$MISSING_VAR"}} + with mock.patch.dict(os.environ, {}, clear=True), \ + mock.patch.object(sys, "stderr", err), \ + mock.patch.object(search.sys, "exit", side_effect=SystemExit) as e: + with self.assertRaises(SystemExit): + search.build_request(config, _args()) + self.assertIn("MISSING_VAR is not set", err.getvalue()) + + def test_non_string_header_value_exits(self): + err = io.StringIO() + config = {"base_url": "https://searx.example.com", "headers": {"X-Num": 42}} + with mock.patch.object(sys, "stderr", err), \ + mock.patch.object(search.sys, "exit", side_effect=SystemExit) as e: + with self.assertRaises(SystemExit): + search.build_request(config, _args()) + self.assertIn("X-Num", err.getvalue()) + self.assertIn("headers", err.getvalue()) + + class TestUserAgent(unittest.TestCase): def test_user_agent_mentions_real_repo(self): req = search.build_request({"base_url": "https://searx.example.com"}, _args()) diff --git a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_config.py b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_config.py index 4975bd5..b9a99dc 100644 --- a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_config.py +++ b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_config.py @@ -99,6 +99,61 @@ def test_empty_string_passes_through(self): self.assertEqual(search.resolve_env(""), "") +class TestConfigFieldTypes(unittest.TestCase): + """Numeric config fields must be integers. A hand-edited TOML with + `timeout = "30"` used to raise a raw TypeError from urllib (and + `default_max_results = "5"` from result slicing); both now fail + fast with a clear message during load_config. + """ + + def _write(self, d, body): + os.makedirs(os.path.join(d, "agents"), exist_ok=True) + with open(os.path.join(d, "agents", "searxng.toml"), "w") as f: + f.write(body) + + def test_timeout_as_string_exits(self): + with tempfile.TemporaryDirectory() as d, \ + mock.patch.dict(os.environ, {"XDG_CONFIG_HOME": d}, clear=True), \ + mock.patch.object(search, "validate_base_url"), \ + mock.patch.object(search, "check_file_permissions"): + self._write(d, 'base_url = "https://searx.example.com"\ntimeout = "30"\n') + rc, err = _capture(search.load_config) + self.assertEqual(rc, 1) + self.assertIn("timeout", err) + self.assertIn("must be an integer", err) + + def test_default_max_results_as_string_exits(self): + with tempfile.TemporaryDirectory() as d, \ + mock.patch.dict(os.environ, {"XDG_CONFIG_HOME": d}, clear=True), \ + mock.patch.object(search, "validate_base_url"), \ + mock.patch.object(search, "check_file_permissions"): + self._write(d, 'base_url = "https://searx.example.com"\ndefault_max_results = "5"\n') + rc, err = _capture(search.load_config) + self.assertEqual(rc, 1) + self.assertIn("default_max_results", err) + self.assertIn("must be an integer", err) + + def test_boolean_is_rejected(self): + # `True` is an int subclass; treat it as a config typo, not a number. + with tempfile.TemporaryDirectory() as d, \ + mock.patch.dict(os.environ, {"XDG_CONFIG_HOME": d}, clear=True), \ + mock.patch.object(search, "validate_base_url"), \ + mock.patch.object(search, "check_file_permissions"): + self._write(d, 'base_url = "https://searx.example.com"\ntimeout = true\n') + rc, err = _capture(search.load_config) + self.assertEqual(rc, 1) + self.assertIn("timeout", err) + + def test_valid_integers_load_ok(self): + with tempfile.TemporaryDirectory() as d, \ + mock.patch.dict(os.environ, {"XDG_CONFIG_HOME": d}, clear=True), \ + mock.patch.object(search, "check_file_permissions"): + self._write(d, 'base_url = "https://searx.example.com"\ntimeout = 12\ndefault_max_results = 3\n') + config = search.load_config() + self.assertEqual(config["timeout"], 12) + self.assertEqual(config["default_max_results"], 3) + + class TestFilePermissions(unittest.TestCase): def test_world_readable_warns(self): if os.name != "posix": diff --git a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_invalid_response.py b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_invalid_response.py index c15070e..719128f 100644 --- a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_invalid_response.py +++ b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_invalid_response.py @@ -1,7 +1,7 @@ -"""Review point 4: error paths for non-JSON responses, HTTPError body -echo, and SearXNG structured error response. The script must exit -non-zero and (in the body-echo case) not leak credentials that the -server reflected back. +"""Review point 4 + P1 review round 2: error paths for non-JSON responses, +HTTPError body echo, SearXNG structured error response, and authentication +error body suppression. The script must exit non-zero and (in the body-echo +case) not leak credentials that the server reflected back. """ import argparse import io @@ -25,18 +25,27 @@ def _args(**over): return argparse.Namespace(**base) -def _capture_main(argv, urlopen_mock, load_config_mock=None): - """Invoke `search.main()` with patched argv / urlopen / load_config, - return (exit_code, stderr_text).""" +def _capture_main(argv, open_side_effect, load_config_mock=None): + """Invoke `search.main()` with patched argv / opener / load_config, + return (exit_code, stderr_text). + + `open_side_effect` is the value/exception that the mocked opener.open + will produce when main() calls it. + """ if load_config_mock is None: load_config_mock = mock.Mock(return_value={"base_url": "https://searx.example.com"}) err = io.StringIO() + open_mock = mock.Mock(side_effect=open_side_effect) if isinstance(open_side_effect, BaseException) or ( + callable(open_side_effect) and not isinstance(open_side_effect, mock.Mock) + ) else mock.Mock(return_value=open_side_effect) + opener = mock.Mock() + opener.open = open_mock with mock.patch.object(sys, "argv", ["search.py"] + argv), \ mock.patch.object(sys, "stderr", err), \ mock.patch.object(search, "load_config", load_config_mock), \ mock.patch.object(search, "build_request", mock.Mock(return_value=mock.Mock(get_header=mock.Mock(return_value=None)))), \ - mock.patch.object(search.urllib.request, "urlopen", urlopen_mock), \ + mock.patch.object(search, "_build_opener", return_value=opener), \ mock.patch.object(search.sys, "exit", side_effect=SystemExit) as e: try: search.main() @@ -48,49 +57,134 @@ def _capture_main(argv, urlopen_mock, load_config_mock=None): class TestInvalidResponse(unittest.TestCase): def test_non_json_response_exits_with_diagnostic(self): body = b"oops not json" - urlopen = mock.Mock(return_value=FakeResponse(body)) - rc, err = _capture_main(["q"], urlopen) + rc, err = _capture_main(["q"], FakeResponse(body)) self.assertEqual(rc, 1) self.assertIn("Invalid JSON", err) self.assertIn("oops not json", err) def test_searxng_structured_error_exits(self): body = b'{"error": "instance disabled"}' - urlopen = mock.Mock(return_value=FakeResponse(body)) - rc, err = _capture_main(["q"], urlopen) + rc, err = _capture_main(["q"], FakeResponse(body)) self.assertEqual(rc, 1) self.assertIn("SearXNG returned error", err) self.assertIn("instance disabled", err) def test_http_error_echoes_body_redacted(self): - # The "server" reflects back an Authorization header in its error body. + # 500 is not auth-class, so body IS echoed. The "server" reflects + # back an Authorization header in its error body which must be + # masked via shape-based redaction. body = b'{"detail": "got Authorization: Basic ' + (b"A" * 60) + b'"}' - urlopen = mock.Mock(side_effect=FakeHTTPError(401, body)) - rc, err = _capture_main(["q"], urlopen) + rc, err = _capture_main(["q"], FakeHTTPError(500, body)) self.assertEqual(rc, 1) - self.assertIn("HTTP 401", err) + self.assertIn("HTTP 500", err) # The reflected credential must be masked, not echoed. self.assertNotIn(b"A" * 60, err.encode()) self.assertIn("Authorization: Basic ***", err) def test_url_error_reports_reason(self): from urllib.error import URLError - urlopen = mock.Mock(side_effect=URLError("dns failure")) - rc, err = _capture_main(["q"], urlopen) + rc, err = _capture_main(["q"], URLError("dns failure")) self.assertEqual(rc, 1) self.assertIn("Request failed", err) self.assertIn("dns failure", err) def test_timeout_error_reports_configured_timeout(self): - urlopen = mock.Mock(side_effect=TimeoutError("")) - # We can't easily test the actual timeout from urlopen's perspective - # without hitting the network, but we can verify the error message - # uses the configured value. - rc, err = _capture_main(["q"], urlopen) + rc, err = _capture_main(["q"], TimeoutError("")) + # We can't easily test the actual timeout from the opener's + # perspective without hitting the network, but we can verify + # the error message uses the configured value. self.assertEqual(rc, 1) self.assertIn("Request timed out", err) self.assertIn("30s", err) # default +class TestAuthErrorBodySuppression(unittest.TestCase): + """P1 review round 2: 401/403/407 bodies must not be echoed to stderr. + + These codes are the canonical "your credentials are wrong" responses, + and SearXNG-style instances have been observed to reflect the + Authorization header back in the body. Shape-based redaction can't + cover every variant; suppressing the body entirely for these codes + closes the leak surface. + """ + + def test_401_body_is_suppressed(self): + # Even if the body contains a long-shape bearer token, it must + # not appear in the error output. + body = b'{"detail": "got Bearer ghp_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}' + rc, err = _capture_main(["q"], FakeHTTPError(401, body)) + self.assertEqual(rc, 1) + self.assertIn("HTTP 401", err) + self.assertIn("body suppressed", err) + # Body bytes must not leak. + self.assertNotIn(b"Bearer ghp_", err.encode()) + self.assertNotIn(b"got Bearer", err.encode()) + + def test_403_body_is_suppressed(self): + body = b'{"detail": "forbidden: bad credentials xyz123"}' + rc, err = _capture_main(["q"], FakeHTTPError(403, body)) + self.assertEqual(rc, 1) + self.assertIn("HTTP 403", err) + self.assertIn("body suppressed", err) + self.assertNotIn(b"xyz123", err.encode()) + self.assertNotIn(b"forbidden: bad", err.encode()) + + def test_407_body_is_suppressed(self): + # Proxy auth challenge — same class of leak. + body = b'{"detail": "Proxy-Authenticate: Basic realm=svc"}' + rc, err = _capture_main(["q"], FakeHTTPError(407, body)) + self.assertEqual(rc, 1) + self.assertIn("HTTP 407", err) + self.assertIn("body suppressed", err) + self.assertNotIn(b"Proxy-Authenticate", err.encode()) + + def test_500_body_is_NOT_suppressed(self): + # 500 is not auth-class; the body should still be echoed (and + # run through redact_secrets as usual). This guards against + # accidentally widening the suppression to all HTTPError codes. + body = b'{"error": "internal server error"}' + rc, err = _capture_main(["q"], FakeHTTPError(500, body)) + self.assertEqual(rc, 1) + self.assertIn("HTTP 500", err) + self.assertIn("internal server error", err) + self.assertNotIn("body suppressed", err) + + +class TestResponseShapeRobustness(unittest.TestCase): + """Robustness beyond the P1 review: a malformed-but-200 response must + produce a clean diagnostic, not a raw traceback that bypasses the + redacted-error pipeline.""" + + def test_json_array_root_exits_with_diagnostic(self): + # A proxy/login page or misconfigured endpoint can return a JSON + # array; the script used to crash with AttributeError on data.get(). + rc, err = _capture_main(["q"], FakeResponse(b'[{"x": 1}]')) + self.assertEqual(rc, 1) + self.assertIn("JSON object", err) + + def test_non_utf8_body_exits_with_diagnostic(self): + # A 200 response in latin-1/GBK used to raise an uncaught + # UnicodeDecodeError. + rc, err = _capture_main(["q"], FakeResponse('{"results": [{"title": "caf\u00e9"}]}'.encode("latin-1"))) + self.assertEqual(rc, 1) + self.assertIn("non-UTF-8", err) + + def test_oversized_response_exits_with_diagnostic(self): + with mock.patch.object(search, "MAX_RESPONSE_BYTES", 16): + rc, err = _capture_main(["q"], FakeResponse(b"x" * 100)) + self.assertEqual(rc, 1) + self.assertIn("limit", err) + self.assertNotIn(b"x" * 16, err.encode()) # body not echoed + + def test_oversized_http_error_body_is_capped(self): + # The HTTPError path reads the body with the same cap; a huge + # error body must not be slurped into memory or echoed wholesale. + with mock.patch.object(search, "MAX_RESPONSE_BYTES", 32): + rc, err = _capture_main(["q"], FakeHTTPError(500, b"z" * 1000)) + self.assertEqual(rc, 1) + self.assertIn("HTTP 500", err) + self.assertLess(len(err), 200) # truncated, not 1000 bytes + + if __name__ == "__main__": unittest.main() diff --git a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_redaction.py b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_redaction.py index 094c108..008196e 100644 --- a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_redaction.py +++ b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_redaction.py @@ -3,6 +3,10 @@ `redact_secrets` covers 5 credential shapes. `die` and `warn` route through it before printing. We also assert the `Authorization: Basic` header value is masked in error output. + +P1 review round 2 (point 3): the exact-secret registry (`register_secret` / +`reset_active_secrets`) must mask short or otherwise shape-non-conforming +tokens that the pattern-based redaction cannot catch. """ import contextlib import io @@ -23,6 +27,13 @@ class TestRedactSecrets(unittest.TestCase): + def setUp(self): + # Ensure each test starts with a clean registry. + search.reset_active_secrets() + + def tearDown(self): + search.reset_active_secrets() + def test_redacts_credential_shapes(self): cases = { TOKEN: "ghx_***", @@ -42,7 +53,107 @@ def test_leaves_ordinary_text_intact(self): self.assertEqual(search.redact_secrets(msg), msg) +class TestExactSecretRegistry(unittest.TestCase): + """P1 review round 2 (point 3): shape-based redaction cannot cover + every token variant. The active-request secret registry + (`register_secret`) provides exact-value replacement as a primary + defense; shape patterns are a fallback. + + These tests pin down the registry's contract: + - Short tokens that don't match any shape ARE masked after registration. + - A substring secret is replaced before its containing string + (longest-first). + - `reset_active_secrets` clears the registry. + - Empty / falsy values are no-ops. + - Duplicates don't grow the set. + """ + + def setUp(self): + search.reset_active_secrets() + + def tearDown(self): + search.reset_active_secrets() + + def test_short_token_is_masked_after_registration(self): + # 8 chars — well below the 20-char shape threshold for `gh[pousr]_`. + # Without the registry, this would pass through; with it, it must + # be replaced. + short = "abc12345" + search.register_secret(short) + out = search.redact_secrets(f"server saw header value {short} in trace") + self.assertNotIn(short, out) + self.assertIn("***", out) + self.assertIn("server saw header value *** in trace", out) + + def test_non_shape_token_is_masked_after_registration(self): + # A bespoke internal token format with no matching shape pattern. + bespoke = "internal-token-zzzz-9999" + search.register_secret(bespoke) + out = search.redact_secrets(f"upstream error: {bespoke} is not authorized") + self.assertNotIn(bespoke, out) + self.assertIn("upstream error: *** is not authorized", out) + + def test_unregistered_short_token_is_NOT_masked(self): + # Counter-test: if a token was never registered, we don't claim + # to mask it. The pattern-based fallback is best-effort and may + # miss it. This pins down the contract that the registry is the + # primary defense. + short = "abc12345" + out = search.redact_secrets(f"server saw {short} in trace") + self.assertIn(short, out) + + def test_substring_secret_replaced_before_container(self): + # "abc" is a substring of "abcdef". When both are registered, the + # longer one must be replaced first so the shorter one isn't + # masked by a leftover. + search.register_secret("abc") + search.register_secret("abcdef") + out = search.redact_secrets("value=abcdef in payload") + self.assertNotIn("abcdef", out) + # After both replacements: "value=*** in payload" (if "abcdef" is + # replaced first) or "value=***def in payload" (if "abc" is first + # but the leftover `def` is harmless). + # The contract is: no registered value may remain. + self.assertNotIn("abc", out) + self.assertNotIn("abcdef", out) + + def test_reset_clears_registry(self): + search.register_secret("abc12345") + out1 = search.redact_secrets("leak abc12345 here") + self.assertNotIn("abc12345", out1) + search.reset_active_secrets() + # After reset, the same value is no longer masked. + out2 = search.redact_secrets("leak abc12345 here") + self.assertIn("abc12345", out2) + + def test_empty_value_is_noop(self): + # Empty / falsy registrations must not pollute the registry or + # match unrelated text. + search.register_secret("") + search.register_secret(None) if False else None # type: ignore[arg-type] + out = search.redact_secrets("normal text with *** placeholders") + # No replacement should happen for an empty registered value + # (replace("", "***") would replace every empty position, so we + # guard against that). + self.assertNotEqual(out, "normal text with ******** placeholders") + + def test_duplicate_registration_is_idempotent(self): + # The set is deduplicated; repeated registration must not change + # output and must not change set size meaningfully. + for _ in range(5): + search.register_secret("abc12345") + out = search.redact_secrets("leak abc12345 here") + self.assertNotIn("abc12345", out) + self.assertIn("***", out) + + class TestStderrRedaction(unittest.TestCase): + def setUp(self): + search.reset_active_secrets() + + def tearDown(self): + search.reset_active_secrets() + def test_die_redacts_token_and_exits(self): err = io.StringIO() with mock.patch.object(sys, "stderr", err), self.assertRaises(SystemExit) as cm: diff --git a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_redirect.py b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_redirect.py new file mode 100644 index 0000000..783189f --- /dev/null +++ b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_redirect.py @@ -0,0 +1,198 @@ +"""P1 review round 2 (point 1): HTTP redirects must not forward the +Authorization header to a different origin (or a different scheme). + +The default `urllib.request.HTTPRedirectHandler` re-issues a 30x response +to the `Location:` URL *with* the original Authorization header. A +misconfigured `base_url` (e.g. an instance fronted by a load balancer +that 302s to a different port) would leak the Bearer / Basic token to a +host the user never explicitly trusted, and would also permit a silent +HTTPS->HTTP downgrade. + +`_SafeRedirectHandler` rejects all 3xx with a clear diagnostic. These +tests assert the rejection path and the diagnostic content for every +supported status code, and assert the *Location:* URL printed to the +user does not echo credentials from the original request. +""" +import argparse +import io +import os +import sys +import unittest +import urllib.error +import urllib.request +from unittest import mock + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import search +from _fixtures import FakeResponse # noqa: F401 (parity with other test files) + + +class TestSafeRedirectHandlerBlocksAllRedirects(unittest.TestCase): + """Every supported 3xx code must raise HTTPError with a clear + diagnostic, and the diagnostic must mention the redirect so the user + knows to fix their `base_url` or front the instance with a same-origin + reverse proxy.""" + + def _assert_blocked(self, code: int, label_fragment: str, headers=None): + handler = search._SafeRedirectHandler() + req = urllib.request.Request("https://searx.example.com/search?q=x") + req.add_header("Authorization", "Bearer ghp_secretvalue_aaaaaaaaaaaaaaaa") + loc = headers if headers is not None else {"Location": "https://other.example.com:9000/search"} + method = getattr(handler, f"http_error_{code}") + with self.assertRaises(urllib.error.HTTPError) as cm: + method(req, mock.Mock(), code, "Found", loc) + self.assertEqual(cm.exception.code, code) + msg = str(cm.exception) + self.assertIn("redirect blocked", msg) + self.assertIn(label_fragment, msg) + # The diagnostic must show the redirect target so the user can fix + # their config — but stripped of any query/fragment that could carry + # credentials. + if "Location" in loc: + self.assertIn("other.example.com:9000", msg) + # Critical: the Authorization header value must NOT be echoed into + # the error message, even though it is on the request object. + self.assertNotIn("ghp_secretvalue_aaaaaaa", msg) + + def test_301_is_blocked(self): + self._assert_blocked(301, "301 Moved Permanently") + + def test_302_is_blocked(self): + self._assert_blocked(302, "302 Found") + + def test_303_is_blocked(self): + self._assert_blocked(303, "303 See Other") + + def test_307_is_blocked(self): + self._assert_blocked(307, "307 Temporary Redirect") + + def test_308_is_blocked(self): + self._assert_blocked(308, "308 Permanent Redirect") + + +class TestSafeRedirectHandlerDiagnosticShape(unittest.TestCase): + """The diagnostic must be useful (show scheme/host/port/path) but must + not echo the query string or fragment of the Location header. Those + are common places to embed tokens.""" + + def test_diagnostic_strips_query_and_fragment(self): + handler = search._SafeRedirectHandler() + req = urllib.request.Request("https://searx.example.com/search?q=x") + location = "https://other.example.com:9000/path?token=ghp_SECRETabcdefghij1234#frag" + with self.assertRaises(urllib.error.HTTPError) as cm: + handler.http_error_302(req, mock.Mock(), 302, "Found", {"Location": location}) + msg = str(cm.exception) + # Path is preserved. + self.assertIn("/path", msg) + # Query is stripped. + self.assertNotIn("token=", msg) + self.assertNotIn("ghp_SECRET", msg) + # Fragment is stripped. + self.assertNotIn("frag", msg) + + def test_diagnostic_handles_unparseable_location(self): + handler = search._SafeRedirectHandler() + req = urllib.request.Request("https://searx.example.com/search?q=x") + with mock.patch.object(search.urllib.parse, "urlparse", side_effect=ValueError("bad url")): + with self.assertRaises(urllib.error.HTTPError) as cm: + handler.http_error_302(req, mock.Mock(), 302, "Found", {"Location": "http://x"}) + msg = str(cm.exception) + self.assertIn("redirect blocked", msg) + self.assertIn("(unparseable)", msg) + + def test_diagnostic_handles_missing_location(self): + handler = search._SafeRedirectHandler() + req = urllib.request.Request("https://searx.example.com/search?q=x") + with self.assertRaises(urllib.error.HTTPError) as cm: + handler.http_error_302(req, mock.Mock(), 302, "Found", {}) + msg = str(cm.exception) + self.assertIn("redirect blocked", msg) + self.assertIn("(none)", msg) + + +class TestBuildOpenerUsesSafeHandler(unittest.TestCase): + """`_build_opener` must install the safe handler so the default urllib + redirect handler is NOT in play (which would forward Authorization).""" + + def test_opener_includes_safe_redirect_handler(self): + opener = search._build_opener() + # urllib stores handlers in `handlers` (list). We assert an instance + # of our safe class is registered. + safe_instances = [h for h in opener.handlers if isinstance(h, search._SafeRedirectHandler)] + self.assertEqual(len(safe_instances), 1, "_SafeRedirectHandler must be installed exactly once") + + def test_opener_does_not_install_default_redirect_handler(self): + opener = search._build_opener() + # If a generic HTTPRedirectHandler (not our subclass) is also + # installed, it would still chase redirects. urllib's build_opener + # REPLACES handlers passed to it, so only the safe one should be + # present. + from urllib.request import HTTPRedirectHandler + non_safe = [h for h in opener.handlers + if isinstance(h, HTTPRedirectHandler) and not isinstance(h, search._SafeRedirectHandler)] + self.assertEqual(non_safe, [], f"non-safe redirect handlers installed: {non_safe}") + + +class TestRedirectIntegrationInMain(unittest.TestCase): + """End-to-end: when the opener returns a 302, the script must exit + non-zero with the redirect diagnostic, and the Authorization header + must not be visible in the error output. + + This is the regression test for the *exact* bug the P1 review cited: + a loopback URL returning 302 to a different port, with the Bearer + token forwarded to the second origin. + """ + + def _run(self, base_url: str, auth_token: str = ""): + config = {"base_url": base_url} + if auth_token: + config["auth"] = {"type": "bearer", "token": auth_token} + + # Simulate a 302 from the opener: the safe handler would raise + # HTTPError("redirect blocked: ...") in real life, but here we + # mock the opener to short-circuit and raise the same error so + # the integration path through main() is exercised. + def _open_redirect(*a, **kw): + raise urllib.error.HTTPError( + url=base_url + "/search", + code=302, + msg="redirect blocked: 302 Found to https://other.example.com:9000/path", + hdrs=None, + fp=None, + ) + + err = io.StringIO() + open_mock = mock.Mock(side_effect=_open_redirect) + opener = mock.Mock() + opener.open = open_mock + + with mock.patch.object(sys, "argv", ["search.py", "q"]), \ + mock.patch.object(sys, "stderr", err), \ + mock.patch.object(search, "load_config", return_value=config), \ + mock.patch.object(search, "build_request", + mock.Mock(return_value=mock.Mock(get_header=mock.Mock(return_value=None)))), \ + mock.patch.object(search, "_build_opener", return_value=opener), \ + mock.patch.object(search.sys, "exit", side_effect=SystemExit) as e: + try: + search.main() + except SystemExit: + pass + return e.call_args[0][0] if e.call_args else None, err.getvalue() + + def test_main_exits_nonzero_on_redirect(self): + rc, err = self._run("https://searx.example.com") + self.assertEqual(rc, 1) + self.assertIn("redirect blocked", err) + + def test_main_does_not_echo_authorization_in_redirect_diagnostic(self): + # The Authorization header is on the request, but the error + # message must not echo it. + rc, err = self._run("https://searx.example.com", auth_token="ghp_aabbccddeeff00112233445566778899") + self.assertEqual(rc, 1) + self.assertIn("redirect blocked", err) + self.assertNotIn("ghp_aabbccddeeff", err) + + +if __name__ == "__main__": + unittest.main() diff --git a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_timeout.py b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_timeout.py index 207743b..2846869 100644 --- a/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_timeout.py +++ b/plugins/Fectivnfy112357/searxng-search/skills/searxng-search/scripts/tests/test_timeout.py @@ -1,5 +1,6 @@ -"""Review point 4: the configured `timeout` is passed through to -`urllib.request.urlopen`. We mock urlopen and inspect the call. +"""Review point 4: the configured `timeout` is passed through to the +HTTP opener. We mock `_build_opener` (returns an opener whose `.open` +is the call we inspect) so the timeout kwarg is observable. """ import os import sys @@ -14,18 +15,20 @@ def _run_main(load_config_return, argv=("search.py", "q")): """Run `search.main()` with all network/config side effects mocked, - and return the urlopen mock so the test can inspect its call args.""" - u = mock.Mock(return_value=FakeResponse(b'{"results":[]}')) + and return the opener.open mock so the test can inspect its call args.""" + open_mock = mock.Mock(return_value=FakeResponse(b'{"results":[]}')) + opener = mock.Mock() + opener.open = open_mock with mock.patch.object(sys, "argv", list(argv)), \ mock.patch.object(search, "load_config", return_value=load_config_return), \ mock.patch.object(search, "build_request", return_value=mock.Mock()), \ - mock.patch.object(search.urllib.request, "urlopen", u), \ + mock.patch.object(search, "_build_opener", return_value=opener), \ mock.patch.object(search.sys, "exit", side_effect=SystemExit): try: search.main() except SystemExit: pass - return u + return open_mock class TestTimeoutConfig(unittest.TestCase): diff --git a/plugins/Fectivnfy112357/searxng-search/test/searxng-search.test.mjs b/plugins/Fectivnfy112357/searxng-search/test/searxng-search.test.mjs new file mode 100644 index 0000000..0acf713 --- /dev/null +++ b/plugins/Fectivnfy112357/searxng-search/test/searxng-search.test.mjs @@ -0,0 +1,72 @@ +// Bridge from the repo's `node --test` to this plugin's Python regression +// suite. Lives inside the Plugin directory so the Plugin stays a portable, +// self-contained contribution: nothing outside plugins// is +// modified. `node --test` at the repository root discovers this file +// recursively. +// +// Spawns the first available Python interpreter (python3, python, py -3) +// and asserts a zero exit code. No network, no gh binary required — the +// Python suite is self-contained. +// +// Interpreter discovery: +// - On Windows, prefer `py -3` (the official Python launcher), then +// fall back to `python` and `python3`. +// - On POSIX, prefer `python3`, then `python`. +// - Each candidate is checked by `version --version` so a non-Python +// stub (a Windows command name collision, a `py` shim that is +// actually pip) fails fast and we move on to the next candidate. +// - If no interpreter is found, the test fails with an explicit +// diagnostic instead of producing a pass that misleads CI. +import { test } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const SCRIPT_DIR = join( + __dirname, + '..', + 'skills', + 'searxng-search', + 'scripts', +); + +const isWindows = process.platform === 'win32'; + +const CANDIDATES = isWindows + ? [ + { cmd: 'py', args: ['-3', '--version'] }, + { cmd: 'py', args: ['--version'] }, + { cmd: 'python', args: ['--version'] }, + { cmd: 'python3', args: ['--version'] }, + ] + : [ + { cmd: 'python3', args: ['--version'] }, + { cmd: 'python', args: ['--version'] }, + ]; + +function findPython() { + for (const c of CANDIDATES) { + const r = spawnSync(c.cmd, c.args, { encoding: 'utf-8' }); + if (r.status === 0) { + return { cmd: c.cmd, prefix: c.cmd === 'py' ? ['-3'] : [] }; + } + } + return null; +} + +test('searxng-search Python regression tests', () => { + const py = findPython(); + assert.ok(py, `no usable Python interpreter found; tried: ${CANDIDATES.map((c) => `${c.cmd} ${c.args.join(' ')}`).join(', ')}`); + + const r = spawnSync(py.cmd, [...py.prefix, join(SCRIPT_DIR, 'run_tests.py')], { + cwd: SCRIPT_DIR, + encoding: 'utf-8', + }); + if (r.status !== 0) { + console.error(r.stdout); + console.error(r.stderr); + } + assert.equal(r.status, 0, 'python run_tests.py must exit 0'); +}); diff --git a/test/searxng-search.test.mjs b/test/searxng-search.test.mjs deleted file mode 100644 index 998ad0e..0000000 --- a/test/searxng-search.test.mjs +++ /dev/null @@ -1,32 +0,0 @@ -// Bridge from the repo's `node --test` to the Python regression suite. -// Spawns `python run_tests.py` and asserts a zero exit code. No network, -// no gh binary required — the Python suite is self-contained. -import { test } from 'node:test'; -import { strict as assert } from 'node:assert'; -import { spawnSync } from 'node:child_process'; -import { fileURLToPath } from 'node:url'; -import { dirname, join } from 'node:path'; - -const __dirname = dirname(fileURLToPath(import.meta.url)); -const SCRIPT_DIR = join( - __dirname, - '..', - 'plugins', - 'Fectivnfy112357', - 'searxng-search', - 'skills', - 'searxng-search', - 'scripts', -); - -test('searxng-search Python regression tests', () => { - const r = spawnSync('python', [join(SCRIPT_DIR, 'run_tests.py')], { - cwd: SCRIPT_DIR, - encoding: 'utf-8', - }); - if (r.status !== 0) { - console.error(r.stdout); - console.error(r.stderr); - } - assert.equal(r.status, 0, 'python run_tests.py must exit 0'); -});