diff --git a/README.md b/README.md index a83df68..7140aa0 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,47 @@ Add to your `opencode.json`: Restart OpenCode. The plugin will be installed automatically. +## Proxy Configuration + +You can configure outbound proxying in the plugin entry. The proxy is used only by this plugin's search requests; it does not affect OpenCode or the configured model provider. Use an absolute path for a local checkout because OpenCode does not expand `~` in plugin paths. + +```json +{ + "plugin": [ + [ + "/absolute/path/to/opencode-search", + { + "proxy": { + "socksProxy": "socks5://127.0.0.1:9050", + "noProxy": "localhost,127.0.0.1" + } + } + ] + ] +} +``` + +Use `httpProxy` and `httpsProxy` for standard HTTP or HTTPS proxies. If only one is set, it is used for both HTTP and HTTPS requests. `noProxy` contains an optional comma-separated bypass list. Use `socksProxy` for a SOCKS5 proxy; it takes precedence over the HTTP proxy settings. + +As an alternative, set `OPENCODE_SEARCH_HTTP_PROXY`, `OPENCODE_SEARCH_HTTPS_PROXY`, `OPENCODE_SEARCH_SOCKS_PROXY`, or `OPENCODE_SEARCH_NO_PROXY` before starting OpenCode. Values in `opencode.json` take precedence. Generic variables such as `HTTP_PROXY` are deliberately ignored so this plugin cannot accidentally inherit OpenCode's proxy configuration. + +When a SOCKS proxy is configured, the plugin checks Tor's official IP endpoint once at startup and caches the result. Over Tor: + +- DuckDuckGo automatically uses its official onion `/html` endpoint. +- Wikipedia, MDN, and Bluesky use their normal APIs and have been verified to work. +- Google fails immediately with a clear error because it requires a JavaScript challenge over Tor. +- Standard.site may return an upstream 502; this has also occurred without Tor. + +All proxied requests have a 20-second timeout and fail closed; they never retry over a direct connection. + +For a local checkout, build the package and reference its directory in `opencode.json`. Remove any automatic `.opencode/plugins/` symlink for this plugin so it is not loaded twice: + +```bash +cd /absolute/path/to/opencode-search +npm install +npm run build +``` + ## Tools ### ddg-search diff --git a/bun.lock b/bun.lock index 4bc8f9c..e745ad0 100644 --- a/bun.lock +++ b/bun.lock @@ -7,12 +7,14 @@ "dependencies": { "@mozilla/readability": "0.5.0", "jsdom": "26.1.0", + "socks": "2.8.7", + "undici-package": "npm:undici@7.28.0", }, "devDependencies": { "@opencode-ai/plugin": "1.1.12", - "@types/bun": "latest", + "@types/bun": "1.3.9", "@types/jsdom": "27.0.0", - "dprint": "^0.49.1", + "dprint": "latest", "typescript": "5.9.3", }, "peerDependencies": { @@ -33,23 +35,35 @@ "@csstools/css-tokenizer": ["@csstools/css-tokenizer@3.0.4", "", {}, "sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw=="], - "@dprint/darwin-arm64": ["@dprint/darwin-arm64@0.49.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-ib6KcJWo/M5RJWXOQKhP664FG1hAvG7nrbkh+j8n+oXdzmbyDdXTP+zW+aM3/sIQUkGaZky1xy1j2VeScMEEHQ=="], + "@dprint/android-arm64": ["@dprint/android-arm64@0.55.1", "", { "os": "android", "cpu": "arm64" }, "sha512-H1UQIcBJEo1dA8AVrkPnw5AYMMGHNe4w0lq9rEDSY3Lds0VUKACu2MK3JLNARpjljh3WKeO109J26Wpq8PTuGQ=="], - "@dprint/darwin-x64": ["@dprint/darwin-x64@0.49.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-vIVgnYxV7YYa1d6Uyz707RbgB9rwefGPam+rzaueFNPQjdOxPOTQDuMEJDS+Z3BlI00MfeoupIfIUGsXoM4dpQ=="], + "@dprint/android-x64": ["@dprint/android-x64@0.55.1", "", { "os": "android", "cpu": "x64" }, "sha512-8dPiGpEo4S4cQhnLaoHtNm4HiqR9DIPr/81z3gWb5qTmJp8bKj+3DO4mAmmldMRBicmpXCss0xudniNTxnWOMg=="], - "@dprint/linux-arm64-glibc": ["@dprint/linux-arm64-glibc@0.49.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-ZeIh6qMPWLBBifDtU0XadpK36b4WoaTqCOt0rWKfoTjq1RAt78EgqETWp43Dbr6et/HvTgYdoWF0ZNEu2FJFFA=="], + "@dprint/darwin-arm64": ["@dprint/darwin-arm64@0.55.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-c65u8f63R/etCKP0CZ/RkbMCRC99wgZDt/vhwjY2QdO0k23dMKXKEV6+ruUEwV2toh+QIRT02dRL7BtxjA0V7w=="], - "@dprint/linux-arm64-musl": ["@dprint/linux-arm64-musl@0.49.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-/nuRyx+TykN6MqhlSCRs/t3o1XXlikiwTc9emWdzMeLGllYvJrcht9gRJ1/q1SqwCFhzgnD9H7roxxfji1tc+Q=="], + "@dprint/darwin-x64": ["@dprint/darwin-x64@0.55.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-AsET9+4rMk7ZFEMq3HYl/hYNBMBrR/juiSzclEU0oUCf3koxFbVKL8Srad8Vhyv8jEnOaMn78TCnahmHhp1SCQ=="], - "@dprint/linux-riscv64-glibc": ["@dprint/linux-riscv64-glibc@0.49.1", "", { "os": "linux", "cpu": "none" }, "sha512-RHBqrnvGO+xW4Oh0QuToBqWtkXMcfjqa1TqbBFF03yopFzZA2oRKX83PhjTWgd/IglaOns0BgmaLJy/JBSxOfQ=="], + "@dprint/linux-arm64-glibc": ["@dprint/linux-arm64-glibc@0.55.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-vt7w+aL2MjtD3RRfnUTDuK/b7xg1/feBe3WoV0S2rGNmPEJI+K7wwBXkOl/I77V9w3PP2zvN/aRY5dMOZrkbSg=="], - "@dprint/linux-x64-glibc": ["@dprint/linux-x64-glibc@0.49.1", "", { "os": "linux", "cpu": "x64" }, "sha512-MjFE894mIQXOKBencuakKyzAI4KcDe/p0Y9lRp9YSw/FneR4QWH9VBH90h8fRxcIlWMArjFFJJAtsBnn5qgxeg=="], + "@dprint/linux-arm64-musl": ["@dprint/linux-arm64-musl@0.55.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-7shUIOvJcTn4IYEPlb/Up4KLt5kbDuCTdc3DN68i88XT+nRBFQatD7Qok+6Ysra34KJMxF88QVHiimjH1ABy4A=="], - "@dprint/linux-x64-musl": ["@dprint/linux-x64-musl@0.49.1", "", { "os": "linux", "cpu": "x64" }, "sha512-CvGBWOksHgrL1uzYqtPFvZz0+E82BzgoCIEHJeuYaveEn37qWZS5jqoCm/vz6BfoivE1dVuyyOT78Begj9KxkQ=="], + "@dprint/linux-loong64-glibc": ["@dprint/linux-loong64-glibc@0.55.1", "", { "os": "linux", "cpu": "none" }, "sha512-G2mXiOGpkeC3LP8huSnImxU7hMvRY6rHIW+yBuw1vDDm5r6+H9CVhcXr3TMOk3poZ7IlWwQnDmP0/RuGQUEpYw=="], - "@dprint/win32-arm64": ["@dprint/win32-arm64@0.49.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-gQa4s82lMcXjfdxjWBQun6IJlXdPZZaIj2/2cqXWVEOYPKxAZ/JvGzt2pPG+i73h9KHjNLIV8M9ckqEH3oHufg=="], + "@dprint/linux-loong64-musl": ["@dprint/linux-loong64-musl@0.55.1", "", { "os": "linux", "cpu": "none" }, "sha512-uO2r4QPYawDfQUyATr0opqfMkG2hVpz3yScRKs5ve9PWzqx2f5kJaaLuLVoXwSbWqDkozMGPcldAVSpOwGJ24Q=="], - "@dprint/win32-x64": ["@dprint/win32-x64@0.49.1", "", { "os": "win32", "cpu": "x64" }, "sha512-nPU6+hoVze5JJlgET7woYWElBw0IUaB/9XKTaglknQuUUfsmD75D9pkgJTxdIxl9Bg/i5O7c9wb3Nj4XNiTIfw=="], + "@dprint/linux-ppc64-glibc": ["@dprint/linux-ppc64-glibc@0.55.1", "", { "os": "linux", "cpu": "ppc64" }, "sha512-NVR+BN4kTQd/vJ91YOGSkBotAM2f1hFD+HD5zq5Oc1djxjl1U2Zvc667UWKamxaXdv/ssZxkLuj0lbvbx9E3/A=="], + + "@dprint/linux-ppc64-musl": ["@dprint/linux-ppc64-musl@0.55.1", "", { "os": "linux", "cpu": "ppc64" }, "sha512-34Db83XGoKbwmvIY52sokywIMCmmIZM1vzw4ILlOkKugG51TltBOzLCssT9Rkfxz/TjR9dgwtBHEQJZjftuQ6Q=="], + + "@dprint/linux-riscv64-glibc": ["@dprint/linux-riscv64-glibc@0.55.1", "", { "os": "linux", "cpu": "none" }, "sha512-MVeYH48GhR3QkfhMhhYbXl9x+iO2sK1nNpr2ByJ4IKogHIlvlFff0eeukqAZZAod+rhAzJnLMBGgIDbIUHWK1g=="], + + "@dprint/linux-x64-glibc": ["@dprint/linux-x64-glibc@0.55.1", "", { "os": "linux", "cpu": "x64" }, "sha512-ILHgXOIIXeZDjt51egaC4kmu744UuNfaauUM1El+dG9QGan4Vv4f0RRDDbQMKN95zTpacECuNKEhuUtS8euNWA=="], + + "@dprint/linux-x64-musl": ["@dprint/linux-x64-musl@0.55.1", "", { "os": "linux", "cpu": "x64" }, "sha512-SwWxWd313VkeH63fR4IlQCDfJNJ+mSw50BidP0PQInCNbUHYOIeWTcyzAaVa8GQdLvmhiNcJ0pmTDOL0F98QIQ=="], + + "@dprint/win32-arm64": ["@dprint/win32-arm64@0.55.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-9jiZWZ2AkaaXIkaOpOoiP6v0XxuBFp1Hyi3kUMUAY4NVCmvlSEDQhr+UZgnDhozYygDbAV13iNMLERDeoUa0nA=="], + + "@dprint/win32-x64": ["@dprint/win32-x64@0.55.1", "", { "os": "win32", "cpu": "x64" }, "sha512-+RmeJL8zzlEWTcrvyFN51rz0S98tV43Socs7xeVGE2S/EH5dG2fG2/fBe6RB3kuyZSfGqByApYeicIdshYIAgw=="], "@mozilla/readability": ["@mozilla/readability@0.5.0", "", {}, "sha512-Z+CZ3QaosfFaTqvhQsIktyGrjFjSC0Fa4EMph4mqKnWhmyoGICsV/8QK+8HpXut6zV7zwfWwqDmEjtk1Qf6EgQ=="], @@ -77,7 +91,7 @@ "decimal.js": ["decimal.js@10.6.0", "", {}, "sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg=="], - "dprint": ["dprint@0.49.1", "", { "optionalDependencies": { "@dprint/darwin-arm64": "0.49.1", "@dprint/darwin-x64": "0.49.1", "@dprint/linux-arm64-glibc": "0.49.1", "@dprint/linux-arm64-musl": "0.49.1", "@dprint/linux-riscv64-glibc": "0.49.1", "@dprint/linux-x64-glibc": "0.49.1", "@dprint/linux-x64-musl": "0.49.1", "@dprint/win32-arm64": "0.49.1", "@dprint/win32-x64": "0.49.1" }, "bin": { "dprint": "bin.js" } }, "sha512-pO9XH79SyXybj2Vhc9ITZMEI8cJkdlQQRoD8oEfPH6Jjpp/7WX5kIgECVd3DBOjjAdCSiW6R47v3gJBx/qZVkw=="], + "dprint": ["dprint@0.55.1", "", { "optionalDependencies": { "@dprint/android-arm64": "0.55.1", "@dprint/android-x64": "0.55.1", "@dprint/darwin-arm64": "0.55.1", "@dprint/darwin-x64": "0.55.1", "@dprint/linux-arm64-glibc": "0.55.1", "@dprint/linux-arm64-musl": "0.55.1", "@dprint/linux-loong64-glibc": "0.55.1", "@dprint/linux-loong64-musl": "0.55.1", "@dprint/linux-ppc64-glibc": "0.55.1", "@dprint/linux-ppc64-musl": "0.55.1", "@dprint/linux-riscv64-glibc": "0.55.1", "@dprint/linux-x64-glibc": "0.55.1", "@dprint/linux-x64-musl": "0.55.1", "@dprint/win32-arm64": "0.55.1", "@dprint/win32-x64": "0.55.1" }, "bin": { "dprint": "bin.cjs" } }, "sha512-tgUCT9gAM7veMvLX5NmRoYVLnKGKrIYRXpNpJDJj5U7/YIIJef5rLJhPZaJXwB7ad2Vo0Kv//nQM1xkrn5j8kQ=="], "entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="], @@ -89,6 +103,8 @@ "iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="], + "ip-address": ["ip-address@10.2.0", "", {}, "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA=="], + "is-potential-custom-element-name": ["is-potential-custom-element-name@1.0.1", "", {}, "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ=="], "jsdom": ["jsdom@26.1.0", "", { "dependencies": { "cssstyle": "^4.2.1", "data-urls": "^5.0.0", "decimal.js": "^10.5.0", "html-encoding-sniffer": "^4.0.0", "http-proxy-agent": "^7.0.2", "https-proxy-agent": "^7.0.6", "is-potential-custom-element-name": "^1.0.1", "nwsapi": "^2.2.16", "parse5": "^7.2.1", "rrweb-cssom": "^0.8.0", "saxes": "^6.0.0", "symbol-tree": "^3.2.4", "tough-cookie": "^5.1.1", "w3c-xmlserializer": "^5.0.0", "webidl-conversions": "^7.0.0", "whatwg-encoding": "^3.1.1", "whatwg-mimetype": "^4.0.0", "whatwg-url": "^14.1.1", "ws": "^8.18.0", "xml-name-validator": "^5.0.0" }, "peerDependencies": { "canvas": "^3.0.0" }, "optionalPeers": ["canvas"] }, "sha512-Cvc9WUhxSMEo4McES3P7oK3QaXldCfNWp7pl2NNeiIFlCoLr3kfq9kb1fxftiwk1FLV7CvpvDfonxtzUDeSOPg=="], @@ -109,6 +125,10 @@ "saxes": ["saxes@6.0.0", "", { "dependencies": { "xmlchars": "^2.2.0" } }, "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA=="], + "smart-buffer": ["smart-buffer@4.2.0", "", {}, "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg=="], + + "socks": ["socks@2.8.7", "", { "dependencies": { "ip-address": "^10.0.1", "smart-buffer": "^4.2.0" } }, "sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A=="], + "symbol-tree": ["symbol-tree@3.2.4", "", {}, "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw=="], "tldts": ["tldts@6.1.86", "", { "dependencies": { "tldts-core": "^6.1.86" }, "bin": { "tldts": "bin/cli.js" } }, "sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ=="], @@ -121,6 +141,8 @@ "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + "undici-package": ["undici@7.28.0", "", {}, "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA=="], + "undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="], "w3c-xmlserializer": ["w3c-xmlserializer@5.0.0", "", { "dependencies": { "xml-name-validator": "^5.0.0" } }, "sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA=="], diff --git a/opencode.json b/opencode.json index ebb33bd..1a495e6 100644 --- a/opencode.json +++ b/opencode.json @@ -1,7 +1,6 @@ { "$schema": "https://opencode.ai/config.json", "theme": "opencode", - "plugin": ["opencode-search@0.0.8"], "permission": { "bash": { "*": "ask" diff --git a/package.json b/package.json index db447fa..d7f901e 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,9 @@ }, "dependencies": { "@mozilla/readability": "0.5.0", - "jsdom": "26.1.0" + "jsdom": "26.1.0", + "socks": "2.8.7", + "undici-package": "npm:undici@7.28.0" }, "devDependencies": { "@opencode-ai/plugin": "1.1.12", diff --git a/search.ts b/search.ts index e5fa931..26ae7f2 100644 --- a/search.ts +++ b/search.ts @@ -1,24 +1,8 @@ -import type { Plugin } from '@opencode-ai/plugin' -import { - createBskySearchTool, - createDdgSearchTool, - createGoogleSearchTool, - createMdnSearchTool, - createStandardSearchTool, - createWikiSearchTool, -} from './src/search.ts' +import type { PluginInput } from '@opencode-ai/plugin' +import { createPlugin } from './src/search.ts' -const plugin: Plugin = async ({ client }) => { - return { - tool: { - 'bsky-search': createBskySearchTool(client), - 'ddg-search': createDdgSearchTool(client), - 'ggl-search': createGoogleSearchTool(client), - 'mdn-search': createMdnSearchTool(client), - 'standard-search': createStandardSearchTool(client), - 'wiki-search': createWikiSearchTool(client), - }, - } -} +// Current OpenCode passes configured plugin options as the second argument. +// Keeping this as a function also preserves compatibility with legacy loaders. +const plugin = (input: PluginInput, options?: unknown) => createPlugin(input, options) export default plugin diff --git a/src/providers/bsky-search.ts b/src/providers/bsky-search.ts index 86fccff..19019a4 100644 --- a/src/providers/bsky-search.ts +++ b/src/providers/bsky-search.ts @@ -1,5 +1,6 @@ import { z } from 'zod' import * as cache from '../cache.ts' +import { getProxyCacheScope, proxyFetch } from '../proxy.ts' const BSKY_API = 'https://api.bsky.app/xrpc/app.bsky.feed.searchPosts' @@ -46,7 +47,7 @@ type BskySearchOptions = { export const bskySearch = async (options: BskySearchOptions): Promise => { const cacheKey = [ - 'bsky', + getProxyCacheScope() + 'bsky', options.query, options.sort ?? 'latest', options.limit ?? 25, @@ -68,7 +69,7 @@ export const bskySearch = async (options: BskySearchOptions): Promise => { - const cacheKey = 'ddg:' + query + const cacheKey = getProxyCacheScope() + 'ddg:' + query const cached = await cache.get(cacheKey) if (cached) { return cached } - const response = await fetch(DDG_URL, { - method: 'POST', - headers: { - 'User-Agent': USER_AGENT, - 'Accept-Encoding': 'gzip', - 'Content-Type': 'application/x-www-form-urlencoded', - 'DNT': '1', - }, - body: new URLSearchParams({ - q: query, - b: '', - kf: '-1', - kh: '1', - kl: 'us-en', - kp: '1', - k1: '-1', - }), + const params = new URLSearchParams({ + q: query, + b: '', + kf: '-1', + kh: '1', + kl: 'us-en', + kp: '1', + k1: '-1', }) + const useOnion = await isTorProxy() + let responseUrl = useOnion ? DDG_ONION_URL : DDG_URL + let response = useOnion + ? await fetchOnion(params) + : await fetchClearnet(params) + + if (response.status === 403 && getProxyStatus().proxyType === 'socks') { + const blockedHtml = await response.text() + if (blockedHtml.includes('detected that you have connected over Tor')) { + responseUrl = DDG_ONION_URL + response = await fetchOnion(params) + } + } if (!response.ok) { throw new Error(response.status + ' ' + response.statusText) @@ -53,7 +60,7 @@ export const ddgSearch = async (query: string): Promise => { // Silently ignore JSDOM errors }) - const dom = new JSDOM(html, { url: DDG_URL, virtualConsole }) + const dom = new JSDOM(html, { url: responseUrl, virtualConsole }) const doc = dom.window.document const links = doc.querySelectorAll('.result__a') @@ -68,7 +75,7 @@ export const ddgSearch = async (query: string): Promise => { } const title = link.textContent?.trim() ?? '' - const href = link.getAttribute('href') ?? '' + const href = normalizeResultUrl(link.getAttribute('href') ?? '', responseUrl) const abstract = snippets[i]?.textContent?.trim() ?? '' if (title && href) { @@ -80,3 +87,35 @@ export const ddgSearch = async (query: string): Promise => { return results } + +const fetchClearnet = (params: URLSearchParams) => + proxyFetch(DDG_URL, { + method: 'POST', + headers: { + 'User-Agent': USER_AGENT, + 'Accept-Encoding': 'gzip', + 'Content-Type': 'application/x-www-form-urlencoded', + 'DNT': '1', + }, + body: params, + }) + +const fetchOnion = (params: URLSearchParams) => + proxyFetch(DDG_ONION_URL + '?' + params.toString(), { + headers: { + 'User-Agent': USER_AGENT, + 'Accept-Encoding': 'gzip', + 'DNT': '1', + }, + }) + +const normalizeResultUrl = (href: string, baseUrl: string): string => { + if (!href) return '' + + try { + const url = new URL(href, baseUrl) + return url.searchParams.get('uddg') ?? url.toString() + } catch { + return href + } +} diff --git a/src/providers/ggl-search.ts b/src/providers/ggl-search.ts index 97ae703..0e15435 100644 --- a/src/providers/ggl-search.ts +++ b/src/providers/ggl-search.ts @@ -1,5 +1,6 @@ import { JSDOM, VirtualConsole } from 'jsdom' import * as cache from '../cache.ts' +import { getProxyCacheScope, isTorProxy, proxyFetch } from '../proxy.ts' const GOOGLE_URL = 'https://www.google.com/search' const USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' @@ -27,13 +28,17 @@ const unwrapUrl = (href: string): string => { } export const googleSearch = async (query: string): Promise => { - const cacheKey = 'ggl:' + query + const cacheKey = getProxyCacheScope() + 'ggl:' + query const cached = await cache.get(cacheKey) if (cached) { return cached } + if (await isTorProxy()) { + throw new Error('Google search is unavailable over Tor because Google requires a JavaScript challenge') + } + const params = new URLSearchParams({ q: query, ie: 'UTF-8', @@ -53,7 +58,7 @@ export const googleSearch = async (query: string): Promise => { } // Initial request to capture cookies (like googler does) - const init = await fetch(GOOGLE_URL + '?' + params, { + const init = await proxyFetch(GOOGLE_URL + '?' + params, { headers, redirect: 'manual', }) @@ -73,7 +78,7 @@ export const googleSearch = async (query: string): Promise => { // Follow through if we got a redirect, otherwise use the initial response const res = init.redirected || init.status >= 300 - ? await fetch(GOOGLE_URL + '?' + params, { headers }) + ? await proxyFetch(GOOGLE_URL + '?' + params, { headers }) : init if (!res.ok) { @@ -86,6 +91,10 @@ export const googleSearch = async (query: string): Promise => { throw new Error('Google returned a CAPTCHA/block page') } + if (html.includes('/httpservice/retry/enablejs')) { + throw new Error('Google requires JavaScript for this network route') + } + const virtualConsole = new VirtualConsole() virtualConsole.on('jsdomError', () => { // Silently ignore JSDOM errors @@ -114,6 +123,10 @@ export const googleSearch = async (query: string): Promise => { } } + if (results.length === 0) { + throw new Error('Google returned no parseable results; this network route may be blocked') + } + await cache.set(cacheKey, results) return results diff --git a/src/providers/mdn-search.ts b/src/providers/mdn-search.ts index 3706bef..f43f744 100644 --- a/src/providers/mdn-search.ts +++ b/src/providers/mdn-search.ts @@ -1,5 +1,6 @@ import { z } from 'zod' import * as cache from '../cache.ts' +import { getProxyCacheScope, proxyFetch } from '../proxy.ts' import { USER_AGENT } from '../user-agent.ts' const MDN_API = 'https://developer.mozilla.org/api/v1/search' @@ -37,7 +38,7 @@ type MdnSearchOptions = { export const mdnSearch = async (options: MdnSearchOptions): Promise => { const cacheKey = [ - 'mdn', + getProxyCacheScope() + 'mdn', options.query, options.limit ?? 10, options.page ?? 1, @@ -62,7 +63,7 @@ export const mdnSearch = async (options: MdnSearchOptions): Promise => { - const cacheKey = ['standard', options.query, options.limit ?? 0, options.offset ?? 0].join(':') + const cacheKey = [getProxyCacheScope() + 'standard', options.query, options.limit ?? 0, options.offset ?? 0].join(':') const cached = await cache.get(cacheKey) if (cached && cached.documents && cached.documents.length > 0) { @@ -56,7 +57,7 @@ export const standardSearch = async (options: StandardSearchOptions): Promise => { - const cacheKey = 'wiki:' + options.query + ':' + (options.limit ?? 10) + const cacheKey = getProxyCacheScope() + 'wiki:' + options.query + ':' + (options.limit ?? 10) const cached = await cache.get(cacheKey) if (cached && cached.pages && cached.pages.length > 0) { @@ -51,7 +52,7 @@ export const wikiSearch = async (options: WikiSearchOptions): Promise { + afterEach(async () => { + await configureProxy({}) + }) + + it('reads only plugin-specific environment variables', () => { + expect(proxyConfigFromEnv({ + HTTP_PROXY: 'http://unrelated.example:8080', + HTTPS_PROXY: 'http://unrelated.example:8080', + NO_PROXY: '*', + OPENCODE_SEARCH_HTTP_PROXY: 'http://plugin.example:8081', + })).toEqual({ + httpProxy: 'http://plugin.example:8081', + httpsProxy: undefined, + socksProxy: undefined, + noProxy: undefined, + }) + }) + + it('uses plugin options before environment variables', () => { + expect(proxyConfigFromOptions({ + proxy: { + socksProxy: 'socks5://configured.example:9050', + noProxy: '', + }, + }, { + OPENCODE_SEARCH_HTTP_PROXY: 'http://environment.example:8080', + OPENCODE_SEARCH_SOCKS_PROXY: 'socks5://environment.example:1080', + OPENCODE_SEARCH_NO_PROXY: 'localhost', + })).toEqual({ + httpProxy: 'http://environment.example:8080', + httpsProxy: undefined, + socksProxy: 'socks5://configured.example:9050', + noProxy: '', + }) + }) + + it('rejects malformed plugin options', () => { + expect(() => proxyConfigFromOptions({ proxy: 'http://127.0.0.1:8080' })).toThrow( + 'proxy options must be an object', + ) + }) + + it('configures and resets an HTTP proxy', async () => { + await configureProxy({ httpProxy: 'http://127.0.0.1:8080' }) + expect(getProxyStatus()).toEqual({ configured: true, proxyType: 'http', dispatcher: true }) + expect(getProxyCacheScope()).toBe('http:') + expect(await isTorProxy()).toBe(false) + + await configureProxy({}) + expect(getProxyStatus()).toEqual({ configured: false, proxyType: 'none', dispatcher: false }) + expect(getProxyCacheScope()).toBe('none:') + }) + + it('configures a SOCKS5 proxy', async () => { + await configureProxy({ socksProxy: 'socks5://127.0.0.1:1080' }) + expect(getProxyStatus()).toEqual({ configured: true, proxyType: 'socks', dispatcher: true }) + expect(getProxyCacheScope()).toBe('socks:') + }) + + it('fails closed when the configured SOCKS proxy is unavailable', async () => { + await configureProxy({ socksProxy: 'socks5://127.0.0.1:1' }) + + await expect(proxyFetch('https://example.com')).rejects.toThrow() + }) + + it('rejects unsupported proxy protocols', async () => { + expect(configureProxy({ httpProxy: 'file:///tmp/socket' })).rejects.toThrow('httpProxy must use') + expect(configureProxy({ socksProxy: 'http://127.0.0.1:1080' })).rejects.toThrow('socksProxy must use') + }) +}) diff --git a/src/proxy.ts b/src/proxy.ts new file mode 100644 index 0000000..efd407e --- /dev/null +++ b/src/proxy.ts @@ -0,0 +1,200 @@ +// The alias avoids Bun's incomplete built-in `undici` compatibility shim. +import { connect as connectTls } from 'node:tls' +import { SocksClient } from 'socks' +import * as undici from 'undici-package' +import type { Dispatcher, RequestInit as UndiciRequestInit } from 'undici-package' + +const { Agent, EnvHttpProxyAgent, fetch: undiciFetch } = undici + +export type ProxyConfig = { + httpProxy?: string + httpsProxy?: string + socksProxy?: string + noProxy?: string +} + +const isRecord = (value: unknown): value is Record => ( + typeof value === 'object' && value !== null && !Array.isArray(value) +) + +let httpProxyDispatcher: Dispatcher | undefined +let socksProxyDispatcher: Dispatcher | undefined +let torProxyCheck: Promise | undefined +let torProxyKey: string | undefined +let proxyStatus = { + configured: false, + proxyType: 'none' as 'none' | 'http' | 'socks', +} + +const normalizeProxyUrl = ( + value: unknown, + name: string, + allowedProtocols: readonly string[], +): string | undefined => { + if (value === undefined || value === null || value === '') return undefined + if (typeof value !== 'string') throw new TypeError(`${name} must be a URL string`) + + let url: URL + try { + url = new URL(value.trim()) + } catch { + throw new TypeError(`${name} must be a valid URL`) + } + + if (!url.hostname || !allowedProtocols.includes(url.protocol)) { + throw new TypeError(`${name} must use ${allowedProtocols.join(' or ')}`) + } + + return url.toString() +} + +export const proxyConfigFromEnv = (env: NodeJS.ProcessEnv = process.env): ProxyConfig => ({ + httpProxy: env.OPENCODE_SEARCH_HTTP_PROXY, + httpsProxy: env.OPENCODE_SEARCH_HTTPS_PROXY, + socksProxy: env.OPENCODE_SEARCH_SOCKS_PROXY, + noProxy: env.OPENCODE_SEARCH_NO_PROXY, +}) + +export const proxyConfigFromOptions = (options: unknown, env: NodeJS.ProcessEnv = process.env): ProxyConfig => { + const fallback = proxyConfigFromEnv(env) + if (options === undefined) return fallback + if (!isRecord(options)) throw new TypeError('plugin options must be an object') + + const proxy = options.proxy + if (proxy === undefined) return fallback + if (!isRecord(proxy)) throw new TypeError('proxy options must be an object') + + const value = (key: keyof ProxyConfig) => Object.hasOwn(proxy, key) ? proxy[key] : fallback[key] + return { + httpProxy: value('httpProxy') as string | undefined, + httpsProxy: value('httpsProxy') as string | undefined, + socksProxy: value('socksProxy') as string | undefined, + noProxy: value('noProxy') as string | undefined, + } +} + +export const configureProxy = async (config: ProxyConfig = proxyConfigFromEnv()): Promise => { + const httpProxy = normalizeProxyUrl(config.httpProxy, 'httpProxy', ['http:', 'https:']) + const httpsProxy = normalizeProxyUrl(config.httpsProxy, 'httpsProxy', ['http:', 'https:']) + const socksProxy = normalizeProxyUrl(config.socksProxy, 'socksProxy', ['socks:', 'socks5:']) + const noProxy = typeof config.noProxy === 'string' ? config.noProxy.trim() : '' + + const sharedHttpProxy = httpProxy ?? httpsProxy + const sharedHttpsProxy = httpsProxy ?? httpProxy + const nextSocksDispatcher = socksProxy ? createSocksDispatcher(new URL(socksProxy)) : undefined + const nextHttpDispatcher = !socksProxy && (sharedHttpProxy || sharedHttpsProxy) + ? new EnvHttpProxyAgent({ + // Supplying every value prevents EnvHttpProxyAgent from falling back to + // process-wide HTTP_PROXY, HTTPS_PROXY, or NO_PROXY variables. + httpProxy: sharedHttpProxy ?? '', + httpsProxy: sharedHttpsProxy ?? '', + noProxy, + }) + : undefined + + const previousHttpDispatcher = httpProxyDispatcher + const previousSocksDispatcher = socksProxyDispatcher + httpProxyDispatcher = nextHttpDispatcher + socksProxyDispatcher = nextSocksDispatcher + if (torProxyKey !== socksProxy) { + torProxyKey = socksProxy + torProxyCheck = undefined + } + proxyStatus = { + configured: Boolean(nextSocksDispatcher || nextHttpDispatcher), + proxyType: nextSocksDispatcher ? 'socks' : nextHttpDispatcher ? 'http' : 'none', + } + + if (previousHttpDispatcher && previousHttpDispatcher !== nextHttpDispatcher) { + await previousHttpDispatcher.close() + } + if (previousSocksDispatcher && previousSocksDispatcher !== nextSocksDispatcher) { + await previousSocksDispatcher.close() + } + + if (nextSocksDispatcher) { + await isTorProxy() + } +} + +export const getProxyStatus = () => ({ + ...proxyStatus, + dispatcher: Boolean(httpProxyDispatcher || socksProxyDispatcher), +}) + +// Keep results fetched through different transports separate. In particular, +// a direct result must never satisfy a search while proxying is enabled. +export const getProxyCacheScope = (): string => `${proxyStatus.proxyType}:` + +export const isTorProxy = async (): Promise => { + if (!socksProxyDispatcher) return false + + torProxyCheck ??= undiciFetch('https://check.torproject.org/api/ip', { + dispatcher: socksProxyDispatcher, + signal: AbortSignal.timeout(10_000), + }).then(async (response) => { + if (!response.ok) return false + const data = await response.json() as { IsTor?: unknown } + return data.IsTor === true + }).catch(() => false) + + return torProxyCheck +} + +export const proxyFetch = async (input: string | URL, init?: globalThis.RequestInit): Promise => { + const requestInit = { + ...init, + signal: init?.signal ?? AbortSignal.timeout(20_000), + } + const dispatcher = socksProxyDispatcher ?? httpProxyDispatcher + if (!dispatcher) { + return fetch(input, requestInit) + } + + return undiciFetch(input, { + ...requestInit, + dispatcher, + } as UndiciRequestInit) as unknown as Promise +} + +const createSocksDispatcher = (proxy: URL): Dispatcher => { + const proxyPort = Number(proxy.port || 1080) + const username = proxy.username ? decodeURIComponent(proxy.username) : undefined + const password = proxy.password ? decodeURIComponent(proxy.password) : undefined + + return new Agent({ + connect: async (options, callback) => { + try { + const { socket } = await SocksClient.createConnection({ + command: 'connect', + proxy: { + host: proxy.hostname, + port: proxyPort, + type: 5, + userId: username, + password, + }, + destination: { + host: options.hostname, + port: Number(options.port || (options.protocol === 'https:' ? 443 : 80)), + }, + }) + + if (options.protocol !== 'https:') { + callback(null, socket) + return + } + + const secureSocket = connectTls({ + socket, + servername: options.servername || options.hostname, + ALPNProtocols: ['http/1.1'], + }) + secureSocket.once('secureConnect', () => callback(null, secureSocket)) + secureSocket.once('error', callback) + } catch (error) { + callback(error as Error, null) + } + }, + }) +} diff --git a/src/search.ts b/src/search.ts index 2ea4709..cde6e47 100644 --- a/src/search.ts +++ b/src/search.ts @@ -8,7 +8,15 @@ import { googleSearch } from './providers/ggl-search.ts' import { mdnSearch } from './providers/mdn-search.ts' import { standardSearch } from './providers/standard-search.ts' import { wikiSearch } from './providers/wiki-search.ts' -import { renderBskyPost, renderDdgResult, renderGoogleResult, renderMdnDoc, renderStandardDoc, renderWikiPage } from './renderers.ts' +import { configureProxy, getProxyStatus, proxyConfigFromOptions } from './proxy.ts' +import { + renderBskyPost, + renderDdgResult, + renderGoogleResult, + renderMdnDoc, + renderStandardDoc, + renderWikiPage, +} from './renderers.ts' import { safeAsync } from './safe.ts' type Client = PluginInput['client'] @@ -205,3 +213,27 @@ export const createMdnSearchTool = (client: Client) => { }, }) } + +export const createPlugin = async (input: PluginInput, options?: unknown) => { + await configureProxy(proxyConfigFromOptions(options)) + + await input.client.app.log({ + body: { + service: 'opencode-search', + level: 'info', + message: 'proxy configuration initialized', + extra: getProxyStatus(), + }, + }).catch(() => {}) + + return { + tool: { + 'bsky-search': createBskySearchTool(input.client), + 'ddg-search': createDdgSearchTool(input.client), + 'ggl-search': createGoogleSearchTool(input.client), + 'mdn-search': createMdnSearchTool(input.client), + 'standard-search': createStandardSearchTool(input.client), + 'wiki-search': createWikiSearchTool(input.client), + }, + } +}