diff --git a/README.md b/README.md index 8bb59eaf..5e170df1 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ | Package | Location | | - | - | | `@exodus/stasis` | [stasis](stasis/) | +| `@exodus/stasis-api` | [stasis-api](stasis-api/) | | `@exodus/stasis-core` | [stasis-core](stasis-core/) | | `@exodus/stasis-plugins` | [stasis-plugins](stasis-plugins/) | diff --git a/package.json b/package.json index fc95d6c5..c32ff429 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@exodus/stasis-workspace", "version": "0.0.0", - "description": "Workspace root for @exodus/stasis, @exodus/stasis-core, and @exodus/stasis-plugins", + "description": "Workspace root for @exodus/stasis, @exodus/stasis-api, @exodus/stasis-core, and @exodus/stasis-plugins", "private": true, "type": "module", "engines": { @@ -24,6 +24,7 @@ "@babel/plugin-transform-classes": "^7.29.7", "@exodus/bytes": "^1.15.0", "@exodus/stasis": "workspace:*", + "@exodus/stasis-api": "workspace:*", "@exodus/stasis-core": "workspace:*", "@exodus/stasis-plugins": "workspace:*", "babel-plugin-module-resolver": "^3.2.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a6cd8b85..89d71e6c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -23,6 +23,9 @@ importers: '@exodus/stasis': specifier: workspace:* version: link:stasis + '@exodus/stasis-api': + specifier: workspace:* + version: link:stasis-api '@exodus/stasis-core': specifier: workspace:* version: link:stasis-core @@ -59,6 +62,9 @@ importers: stasis: dependencies: + '@exodus/stasis-api': + specifier: 1.0.0-beta.2 + version: link:../stasis-api '@exodus/stasis-core': specifier: 1.0.0-beta.2 version: link:../stasis-core @@ -75,6 +81,8 @@ importers: specifier: ^0.94.0 version: 0.94.0(@emnapi/core@1.11.0)(@emnapi/runtime@1.11.0) + stasis-api: {} + stasis-core: {} stasis-plugins: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index e751cb64..0dc1d9b7 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,6 +1,7 @@ packages: - stasis + - stasis-api - stasis-core - stasis-plugins diff --git a/stasis-api/LICENSE b/stasis-api/LICENSE new file mode 100644 index 00000000..7fbaf3ab --- /dev/null +++ b/stasis-api/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Exodus Movement + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/stasis-api/README.md b/stasis-api/README.md new file mode 100644 index 00000000..1bf8ac61 --- /dev/null +++ b/stasis-api/README.md @@ -0,0 +1,55 @@ +# `@exodus/stasis-api` + +Zero-dependency registry API clients used by `@exodus/stasis`. + +Transport only: no credential discovery, no caching, and no disk access — except the +`npm/semver` shim, which binds to the semver already bundled with the running Node's npm CLI +rather than installing one. + +| Export | What it provides | +| - | - | +| `@exodus/stasis-api/npm` | `advisories(list)` — npm bulk security advisories for `{ name, version }` pairs | +| `@exodus/stasis-api/npm/semver` | lazily-bound `semver` from the bundled npm CLI, so nothing is installed for it | +| `@exodus/stasis-api/github` | `releases()`, `release()`, `latestRelease()`, `asset()` — GitHub releases and their attachments; `subtree()` — a repo tree at an exact ref | + +```js +import { asset, latestRelease } from '@exodus/stasis-api/github' + +const { tag, assets } = await latestRelease('ExodusOSS/stasis') +const bundle = assets.find((a) => a.name.endsWith('.stasis.code.br')) +// `digest` (when GitHub reports one) is verified before the bytes are returned +const bytes = await asset('ExodusOSS/stasis', bundle.id, { digest: bundle.digest }) +console.log(tag, bundle.name, bytes.byteLength) +``` + +`asset()` buffers the attachment in memory, so a caller fetching several large assets should +bound its own concurrency rather than firing them all at once. + +`subtree()` reads a repo tree at an exact commit, tag or branch — no git client, nothing +written to disk. One archive request per call, decompressed and parsed in memory: + +```js +import { subtree } from '@exodus/stasis-api/github' + +// omit `path` for the whole tree +const { root, files } = await subtree('ExodusOSS/bytes', 'v1.15.1', { path: 'benchmarks' }) +console.log(root) // 'ExodusOSS-bytes-c33d586' — the commit the ref resolved to +for (const [path, bytes] of files) console.log(path, bytes.byteLength) // 'benchmarks/…', repo-relative +``` + +Keys stay repo-relative, so a subtree's paths keep their `path` prefix. Only regular files are +returned: directories and symlinks carry no usable content, and a symlink target is exactly +what could point outside the tree. Rejected rather than sanitized: entries that escape the +tree, a path that appears twice (the second would shadow the first), and archives with more +than one top-level directory. The whole archive is held in memory while it is parsed, bounded +by `maxBytes` (256 MiB by default) — enforced on the download and again on the decompressed +bytes, so a compression bomb fails the same way an oversized repo does. + +Every function takes an optional `signal` (defaulting to a timeout) and, for GitHub, an +optional `token` — no token is ever read from the environment. + +See main package [GitHub](https://github.com/ExodusOSS/stasis/tree/main/stasis) or [npm](https://npmjs.com/package/@exodus/stasis) for full README. + +## License + +[MIT](./LICENSE) diff --git a/stasis-api/package.json b/stasis-api/package.json new file mode 100644 index 00000000..04d8623a --- /dev/null +++ b/stasis-api/package.json @@ -0,0 +1,28 @@ +{ + "name": "@exodus/stasis-api", + "version": "1.0.0-beta.2", + "description": "Zero-dependency registry API clients (npm, GitHub) for @exodus/stasis", + "type": "module", + "exports": { + "./npm": "./src/npm/index.js", + "./npm/semver": "./src/npm/semver.cjs", + "./github": "./src/github/index.js" + }, + "files": [ + "src" + ], + "engines": { + "node": ">=24.14.0" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/ExodusOSS/stasis.git", + "directory": "stasis-api" + }, + "author": "Exodus Movement, Inc.", + "license": "MIT", + "bugs": { + "url": "https://github.com/ExodusOSS/stasis/issues" + }, + "homepage": "https://github.com/ExodusOSS/stasis#readme" +} diff --git a/stasis-api/src/archive.js b/stasis-api/src/archive.js new file mode 100644 index 00000000..1543bfd9 --- /dev/null +++ b/stasis-api/src/archive.js @@ -0,0 +1,145 @@ +import assert from 'node:assert/strict' +import { promisify } from 'node:util' +import { gunzip } from 'node:zlib' + +// In-memory reader for a gzipped ustar archive that has already arrived as bytes (see +// github/subtree.js, the consumer). Returns a `Map` of entry path -> bytes; nothing touches +// disk and no external tar is involved -- Node's zlib decompresses, the ustar framing is +// parsed here, so the package stays dependency-free. +// +// Only regular files are kept. Directories, symlinks, hardlinks and device nodes carry no +// content a caller can use, and a symlink target is precisely the thing that could point +// out of the tree, so they are dropped rather than represented. + +const BLOCK = 512 + +const dotdotRegex = /(?:^|\/)\.\.(?:\/|$)/u + +// An entry must land inside the tree it claims to be part of. Rejected: absolute paths, any +// `..` segment, a backslash (a separator that would slip past a posix-only check), and NUL. +export function isSafePath(path) { + if (path === '' || path.startsWith('/') || path.includes('\\') || path.includes('\0')) return false + return !dotdotRegex.test(path) +} + +// Copy an entry out of the archive buffer instead of returning a view into it: callers +// typically keep a subtree and drop the rest, and a subarray would pin the entire archive in +// memory for as long as any single file is referenced. +const detach = (bytes) => new Uint8Array(bytes) + +// tar pads its text fields with NUL. Read the range directly rather than slicing a view out +// of it first -- this runs three times per entry, so the views alone would outnumber the +// files in the archive several times over. +const field = (buf, start, end) => { + const nul = buf.indexOf(0, start) + return buf.toString('utf8', start, nul === -1 || nul > end ? end : nul) +} + +// tar numbers are octal ASCII. GNU/star write sizes above 8 GiB in a base-256 form instead, +// flagged by the high bit of the first byte. +function readNumber(buf, start, end) { + if ((buf[start] & 0x80) !== 0) { + let n = 0n + for (let i = start; i < end; i++) n = (n << 8n) | BigInt(i === start ? buf[i] & 0x7f : buf[i]) + assert(n <= BigInt(Number.MAX_SAFE_INTEGER), 'Malformed tar archive: entry too large') + return Number(n) + } + const text = field(buf, start, end).trim() + if (text === '') return 0 + const n = Number.parseInt(text, 8) + assert(Number.isSafeInteger(n) && n >= 0, `Malformed tar archive: bad size ${text}`) + return n +} + +// ustar splits a long path across `prefix` (345..500) and `name` (0..100). +function ustarName(header) { + const name = field(header, 0, 100) + const prefix = field(header, 345, 500) + return prefix === '' ? name : `${prefix}/${name}` +} + +// pax extended headers are a run of ` =\n` records, where the length +// counts itself. Only `path` matters here (it overrides the ustar name of the next entry). +function paxPath(data) { + let i = 0 + while (i < data.length) { + const space = data.indexOf(0x20, i) + if (space === -1) return null + const length = Number.parseInt(data.subarray(i, space).toString('latin1'), 10) + if (!Number.isSafeInteger(length) || length <= 0 || i + length > data.length) return null + const record = data.subarray(space + 1, i + length - 1).toString('utf8') + const eq = record.indexOf('=') + if (eq !== -1 && record.slice(0, eq) === 'path') return record.slice(eq + 1) + i += length + } + return null +} + +function readTar(tar, select) { + const files = new Map() + // Names are tracked separately from the returned entries so the checks below cover the + // whole archive even when `select` keeps only part of it. + const seen = new Set() + // A pax ('x') or GNU longname ('L') block names the entry that FOLLOWS it. + let pending = null + let offset = 0 + while (offset + BLOCK <= tar.length) { + const header = tar.subarray(offset, offset + BLOCK) + // The archive ends with zero blocks; the first one is enough to stop. + if (header.every((b) => b === 0)) break + + const size = readNumber(header, 124, 136) + const type = String.fromCharCode(header[156]) + const start = offset + BLOCK + const end = start + size + assert(end <= tar.length, 'Malformed tar archive: truncated entry') + + if (type === 'x' || type === 'L') { + pending = type === 'x' ? paxPath(tar.subarray(start, end)) : field(tar, start, end) + } else if (type !== 'g') { + const name = pending ?? ustarName(header) + pending = null + // '0' and NUL both mean a regular file; every other type carries nothing to keep. + if (type === '0' || type === '\0') { + assert(isSafePath(name), `Unsafe archive path: ${name}`) + // tar can legally hold the same path twice and the reader would keep whichever came + // last, so a second entry could shadow the first -- and the shadowed bytes would + // never be seen. Refuse the archive instead of silently picking a winner. + assert(!seen.has(name), `Duplicate archive path: ${name}`) + seen.add(name) + // `select` maps an entry to the key it is stored under, or drops it by returning + // null. Both checks above run either way, so a dropped entry still cannot smuggle an + // unsafe or duplicated path past them -- it only skips being copied. + const key = select(name) + if (key !== null) files.set(key, detach(tar.subarray(start, end))) + } + } + + offset = end + ((BLOCK - (size % BLOCK)) % BLOCK) + } + return files +} + +// zlib has no promise API of its own; inflating on the threadpool keeps a decompression +// that can run to hundreds of MB from blocking the event loop for seconds. +const gunzipAsync = promisify(gunzip) + +// `select` narrows and re-keys the archive as it is read, so the bytes of an entry the caller +// does not want are never copied out of it -- filtering afterwards would memcpy the whole +// tree to keep a fraction of it. +// +// `maxBytes` bounds the DECOMPRESSED size. gzip can expand a tiny input a thousandfold, so a +// cap on the downloaded bytes alone would still let a compression bomb exhaust the heap here. +export async function readTarGz(bytes, select, maxBytes) { + let tar + try { + tar = await gunzipAsync(bytes, { maxOutputLength: maxBytes }) + } catch (cause) { + // The output cap tripping is a size refusal, not a damaged archive -- say which. + if (cause.code === 'ERR_BUFFER_TOO_LARGE') { + throw new Error(`Archive is over the ${maxBytes} byte limit once decompressed; raise maxBytes`, { cause }) + } + throw new Error(`Malformed tar.gz archive: ${cause.message}`, { cause }) + } + return readTar(tar, select) +} diff --git a/stasis-api/src/github/core.js b/stasis-api/src/github/core.js new file mode 100644 index 00000000..4a4afa5d --- /dev/null +++ b/stasis-api/src/github/core.js @@ -0,0 +1,99 @@ +import assert from 'node:assert/strict' + +import { requestFailed, requestOk } from '../request.js' + +// Core GitHub REST plumbing: the host, the headers every call sends, slug and ref +// validation, and next-link parsing. Knows how to talk to GitHub but not what to ask it -- +// each endpoint family builds on this (see releases.js, subtree.js). +// +// Transport only: no disk, and no credential discovery -- a caller that needs a private repo +// or a higher rate limit (60 requests/hour per IP unauthenticated, 5000 authenticated) passes +// `token` explicitly, so a token is never picked up from the environment behind its back. + +const API = 'https://api.github.com' +const API_VERSION = '2022-11-28' +const JSON_MEDIA_TYPE = 'application/vnd.github+json' +// GitHub rejects requests that carry no User-Agent, so one is always sent. +const USER_AGENT = '@exodus/stasis-api' + +// Each half of a slug must be exactly one path segment, so a hand-assembled value can't +// reshape the endpoint it is interpolated into -- `.` and `..` would climb out of /repos/ +// once fetch normalizes the path. Deliberately a segment rule and not GitHub's account +// naming policy: that policy drifts, and encoding it here only turns a loosened rule into a +// local false rejection (it is how a leading-dot repo like `ExodusOSS/.github` gets refused). +const segmentRegex = /^[\w.-]{1,100}$/u +const isSegment = (s) => segmentRegex.test(s) && s !== '.' && s !== '..' +// Refs hold nearly anything, so one is percent-encoded rather than matched; these are the +// characters and sequences git itself forbids, rejected up front for a local error message. +const badRefRegex = /[\s~^:?*[\]\\]|\.\./u + +function assertRepo(repo) { + assert(typeof repo === 'string', `Unexpected repo: ${repo}`) + const parts = repo.split('/') + assert.equal(parts.length, 2, `Expected an \`owner/repo\` slug: ${repo}`) + const [owner, name] = parts + assert(isSegment(owner), `Unexpected repo owner: ${owner}`) + assert(isSegment(name), `Unexpected repo name: ${name}`) +} + +// The one route by which a repo slug becomes a URL. Validating here rather than at each +// endpoint means a new endpoint cannot assemble a URL and forget the check -- which is the +// whole point of the segment rule above. +export function repoUrl(repo, ...segments) { + assertRepo(repo) + return `${API}/repos/${repo}/${segments.join('/')}` +} + +export function encodeRef(ref, what = 'ref') { + assert(typeof ref === 'string' && ref !== '', `Unexpected ${what}: ${ref}`) + assert(!badRefRegex.test(ref), `Unexpected ${what}: ${ref}`) + return encodeURIComponent(ref) +} + +// Absent and explicitly-null both mean unauthenticated, decided here so an endpoint cannot +// half-declare the policy: a missing default would otherwise reach the assert below as +// `undefined` and be reported as a bad token rather than as no token. +export function request(what, url, { accept = JSON_MEDIA_TYPE, token = null, signal }) { + const headers = { Accept: accept, 'User-Agent': USER_AGENT, 'X-GitHub-Api-Version': API_VERSION } + if (token !== null) { + assert(typeof token === 'string' && token !== '', 'Expected a non-empty token') + headers.Authorization = `Bearer ${token}` + } + // A download answers with a cross-origin 302 to a storage host. fetch drops Authorization + // when following a redirect to another origin, so the token stays with GitHub -- which is + // why redirects are followed rather than handled by hand. + return requestOk(`github ${what}`, url, { headers, signal }) +} + +export async function parseJson(what, res) { + try { + return await res.json() + } catch (cause) { + throw new Error(`github ${what} response was not JSON: ${cause.message}`, { cause }) + } +} + +// An OK response's body can still die mid-read (a reset during the transfer): the same +// transport failure class as the request itself, reported through the same shape and the +// same `github ` label -- owned here so an endpoint cannot hand-assemble either. +export async function readBody(what, read) { + try { + return await read() + } catch (cause) { + throw requestFailed(`github ${what}`, cause) + } +} + +const nextLinkRegex = /^\s*<([^>]+)>\s*;\s*rel="?next"?/u + +// Follow the `next` URL of a GitHub `Link` header, ignoring the other rels. Only URLs on the +// API host are accepted: pagination must not walk a response into following a link to +// somewhere else (which would carry the token there). +export function nextLink(header) { + if (!header) return null + for (const part of header.split(',')) { + const m = nextLinkRegex.exec(part) + if (m && m[1].startsWith(`${API}/`)) return m[1] + } + return null +} diff --git a/stasis-api/src/github/index.js b/stasis-api/src/github/index.js new file mode 100644 index 00000000..41a6a526 --- /dev/null +++ b/stasis-api/src/github/index.js @@ -0,0 +1,7 @@ +// Public face of the GitHub client. The shared plumbing lives in core.js (host, headers, +// auth, slug/ref validation, next-link parsing); each endpoint family is its own module +// beside it. Only what is re-exported here is API of this package -- core.js is internal, +// so it can change without breaking a consumer. + +export { asset, latestRelease, release, releases } from './releases.js' +export { subtree } from './subtree.js' diff --git a/stasis-api/src/github/releases.js b/stasis-api/src/github/releases.js new file mode 100644 index 00000000..ba6b897a --- /dev/null +++ b/stasis-api/src/github/releases.js @@ -0,0 +1,117 @@ +import assert from 'node:assert/strict' +import { hash } from 'node:crypto' + +import { METADATA_TIMEOUT, TRANSFER_TIMEOUT } from '../request.js' +import { encodeRef, nextLink, parseJson, readBody, repoUrl, request } from './core.js' + +// Releases and their attachments ("assets" in API terms). + +// GitHub reports an asset's content digest as `:`. The hex length is pinned +// per algorithm so a truncated digest fails as a bad argument, not as a content mismatch. +const digestRegex = /^(?:sha256:[\da-f]{64}|sha384:[\da-f]{96}|sha512:[\da-f]{128})$/u + +// A fixed, curated view of GitHub's payloads rather than a passthrough: volatile or +// caller-irrelevant fields (node_id, author/uploader, download_count, upload_url) are +// dropped and the rest renamed to a stable shape, so consumers can't come to depend on the +// raw API shape. Everything kept here is public API of this package, so a field is added +// when a caller needs one rather than pre-emptively -- adding is cheap, removing breaks. +function normalizeAsset(raw) { + assert(raw !== null && typeof raw === 'object' && !Array.isArray(raw), 'Expected a GitHub asset object') + return { + id: raw.id, + name: raw.name, + label: raw.label ?? null, + size: raw.size ?? null, + contentType: raw.content_type ?? null, + // Absent on assets uploaded before GitHub began reporting digests. + digest: raw.digest ?? null, + // 'uploaded' once the upload completed; anything else is not fetchable yet. + state: raw.state ?? null, + downloadUrl: raw.browser_download_url ?? null, + apiUrl: raw.url ?? null, + } +} + +function normalizeRelease(raw) { + assert(raw !== null && typeof raw === 'object' && !Array.isArray(raw), 'Expected a GitHub release object') + return { + id: raw.id, + tag: raw.tag_name, + name: raw.name ?? null, + body: raw.body ?? null, + draft: Boolean(raw.draft), + prerelease: Boolean(raw.prerelease), + commitish: raw.target_commitish ?? null, + createdAt: raw.created_at ?? null, + publishedAt: raw.published_at ?? null, + url: raw.html_url ?? null, + // Only the tarball: GitHub serves it for every repo, so carrying `zipball_url` beside it + // would be a second way to do the same thing (subtree.js reads tarballs for the same reason). + tarballUrl: raw.tarball_url ?? null, + assets: (Array.isArray(raw.assets) ? raw.assets : []).map(normalizeAsset), + } +} + +// List a repo's releases, newest first, including drafts and prereleases (for those the +// token must be able to see them). `limit` caps how many are returned; pass `Infinity` to +// walk every page. +export async function releases(repo, { limit = 100, signal = AbortSignal.timeout(METADATA_TIMEOUT), token } = {}) { + assert((Number.isInteger(limit) || limit === Infinity) && limit > 0, `Unexpected limit: ${limit}`) + + const out = [] + // GitHub caps a page at 100 entries. Walking `Link: rel="next"` instead of incrementing + // `page=` keeps the walk on URLs GitHub itself handed back. One `signal` covers the + // whole walk, so `limit` also bounds how long a repo with many releases can take. + let url = `${repoUrl(repo, 'releases')}?per_page=${Math.min(limit, 100)}` + while (url !== null) { + // eslint-disable-next-line no-await-in-loop -- pagination is sequential by nature: the next URL comes from this page's Link header + const res = await request('releases', url, { token, signal }) + // eslint-disable-next-line no-await-in-loop -- same walk, one page at a time + const page = await parseJson('releases', res) + assert(Array.isArray(page), 'Expected an array of GitHub releases') + for (const entry of page) { + out.push(normalizeRelease(entry)) + if (out.length === limit) return out + } + url = nextLink(res.headers.get('link')) + } + return out +} + +// `release()` and `latestRelease()` differ only in how the one release is addressed. +async function oneRelease(repo, tail, { signal = AbortSignal.timeout(METADATA_TIMEOUT), token } = {}) { + const res = await request('release', repoUrl(repo, 'releases', tail), { token, signal }) + return normalizeRelease(await parseJson('release', res)) +} + +// Fetch a single release by its tag. +export async function release(repo, tag, options) { + return oneRelease(repo, `tags/${encodeRef(tag, 'tag')}`, options) +} + +// Fetch the latest release. GitHub's notion of "latest" skips drafts and prereleases -- +// use `releases()` when those matter. +export async function latestRelease(repo, options) { + return oneRelease(repo, 'latest', options) +} + +// Download one attachment's bytes by asset id (from a listing's `assets[].id`). Pass the +// listing's `digest` to have it verified before the bytes are handed back, so a truncated or +// swapped download fails here rather than downstream. The default timeout is far longer than +// the metadata calls': release assets can be very large. +export async function asset(repo, id, { digest = null, signal = AbortSignal.timeout(TRANSFER_TIMEOUT), token } = {}) { + assert(Number.isInteger(id) && id > 0, `Unexpected asset id: ${id}`) + const expected = digest === null ? null : String(digest).toLowerCase() + assert(expected === null || digestRegex.test(expected), `Unexpected digest: ${digest}`) + + const url = repoUrl(repo, 'releases/assets', id) + const res = await request('asset', url, { accept: 'application/octet-stream', token, signal }) + const bytes = new Uint8Array(await readBody('asset', () => res.arrayBuffer())) + + if (expected !== null) { + const algorithm = expected.slice(0, expected.indexOf(':')) + const actual = `${algorithm}:${hash(algorithm, bytes, 'hex')}` + assert.equal(actual, expected, `Asset ${id} digest mismatch: expected ${expected}, got ${actual}`) + } + return bytes +} diff --git a/stasis-api/src/github/subtree.js b/stasis-api/src/github/subtree.js new file mode 100644 index 00000000..283a68df --- /dev/null +++ b/stasis-api/src/github/subtree.js @@ -0,0 +1,94 @@ +import assert from 'node:assert/strict' + +import { isSafePath, readTarGz } from '../archive.js' +import { TRANSFER_TIMEOUT } from '../request.js' +import { encodeRef, repoUrl, request } from './core.js' + +// A repo's source tree at an exact ref, read into memory. + +// The archive is decoded in memory, so it needs a ceiling: a runaway repo should error +// instead of exhausting the heap. Enforced twice with the one number -- on the downloaded +// bytes and again on what they decompress to -- because either form alone can be the huge +// one (a checkout of large blobs downloads big; a compression bomb decompresses big). Set +// well clear of ordinary repos -- a small library's archive already measures in the tens of +// MB -- since the cap is a backstop, not a budget. +const ARCHIVE_MAX_BYTES = 256 * 1024 * 1024 + +// Normalize a subtree path to a `dir/` prefix; '' means the whole tree. +function subtreePrefix(path) { + assert(typeof path === 'string', `Unexpected path: ${path}`) + const trimmed = path.replaceAll(/^\/+|\/+$/gu, '') + if (trimmed === '') return '' + assert(isSafePath(trimmed), `Unexpected path: ${path}`) + return `${trimmed}/` +} + +// Read the archive body with a hard ceiling. Lives here rather than in core.js because the +// ceiling is this endpoint's `maxBytes` option -- the asset endpoint takes the whole body, +// whose size the caller already knows from the release listing. +async function readCapped(res, maxBytes) { + const over = (n) => `github archive is over the ${maxBytes} byte limit (${n} bytes); raise maxBytes` + // Trust `content-length` only to fail early -- it is absent on a chunked response. + const declared = Number(res.headers.get('content-length')) + assert(!(declared > maxBytes), over(declared)) + + const chunks = [] + let total = 0 + for await (const chunk of res.body) { + total += chunk.length + assert(!(total > maxBytes), over(total)) + chunks.push(chunk) + } + return Buffer.concat(chunks) +} + +// Fetch a repo subtree at an exact commit, tag or branch, into memory. +// +// One request to GitHub's tarball endpoint returns the whole tree at `ref`, which is then +// narrowed to `path` (default: everything). No git client is involved and nothing is written +// to disk -- the archive is decompressed and parsed in memory, and only regular files +// survive (see ../archive.js). +// +// Returns `{ root, files }` where `files` maps REPO-relative posix paths -> bytes (so a +// subtree's keys keep their `path` prefix), and `root` is the archive's stripped top-level +// directory, which records the commit the ref resolved to. +export async function subtree(repo, ref, options = {}) { + const { + path = '', + maxBytes = ARCHIVE_MAX_BYTES, + signal = AbortSignal.timeout(TRANSFER_TIMEOUT), + token, + } = options + assert(Number.isInteger(maxBytes) && maxBytes > 0, `Unexpected maxBytes: ${maxBytes}`) + const prefix = subtreePrefix(path) + + const url = repoUrl(repo, 'tarball', encodeRef(ref)) + // Keeps the default JSON Accept even though the body is an archive: unlike the asset + // endpoint, /tarball answers 415 to `application/octet-stream`. GitHub redirects to + // codeload, which serves the bytes regardless of what was negotiated here. + const res = await request('archive', url, { token, signal }) + const bytes = await readCapped(res, maxBytes) + + // Every GitHub archive nests the tree under one top-level directory whose name carries the + // resolved commit (`owner-repo-`). Take that directory from the entries themselves + // rather than reconstructing the name and trusting it to match. + // + // Stripping the root and applying `path` as the archive is read, rather than filtering the + // finished Map, is what keeps a small subtree cheap: an entry outside it is checked for + // safety and then skipped, so the whole tree is never copied to keep a fraction of it. + let root = null + const files = await readTarGz(bytes, (name) => { + const slash = name.indexOf('/') + assert(slash > 0, `Unexpected archive entry outside the root directory: ${name}`) + const first = name.slice(0, slash) + if (root === null) root = first + else assert.equal(first, root, 'Unexpected archive with more than one top-level directory') + // '' prefixes everything, so the whole-tree case needs no branch of its own. + return name.startsWith(prefix, slash + 1) ? name.slice(slash + 1) : null + }, maxBytes) + + // A path that matches nothing is a typo far more often than an intentionally empty + // subtree, and an empty Map would be indistinguishable from a successful read. + assert(prefix === '' || files.size > 0, `No files under '${path}' in ${repo} at ${ref}`) + return { root, files } +} diff --git a/stasis/src/apis/npm/index.js b/stasis-api/src/npm/index.js similarity index 58% rename from stasis/src/apis/npm/index.js rename to stasis-api/src/npm/index.js index 60b8f15f..652fff64 100644 --- a/stasis/src/apis/npm/index.js +++ b/stasis-api/src/npm/index.js @@ -1,10 +1,13 @@ import assert from 'node:assert/strict' +import { METADATA_TIMEOUT, requestOk } from '../request.js' import semver from './semver.cjs' +const BULK_ADVISORIES_URL = 'https://registry.npmjs.org/-/npm/v1/security/advisories/bulk' + const packageNameRegex = /^(@[\da-z-]+\/)?[\w-]+(\.[\w-]+)*$/u -export async function advisories(list, { signal = AbortSignal.timeout(30_000) } = {}) { +export async function advisories(list, { signal = AbortSignal.timeout(METADATA_TIMEOUT) } = {}) { const groups = new Map() for (const { name, version } of list) { assert(typeof name === 'string' && typeof version === 'string') @@ -17,20 +20,11 @@ export async function advisories(list, { signal = AbortSignal.timeout(30_000) } const entries = [...groups].map(([k, v]) => [k, [...v].toSorted((a, b) => semver.compare(a, b))]) const body = Object.fromEntries(entries.toSorted((a, b) => a[0] < b[0] ? -1 : 1)) - let res - try { - res = await fetch('https://registry.npmjs.org/-/npm/v1/security/advisories/bulk', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(body), - signal, - }) - } catch (cause) { - throw new Error(`npm advisories request failed: ${cause.message}`, { cause }) - } - if (!res.ok) { - const text = await res.text().catch(() => '') - throw new Error(`npm advisories request failed: ${res.status} ${res.statusText}${text ? ` — ${text.slice(0, 200)}` : ''}`) - } + const res = await requestOk('npm advisories', BULK_ADVISORIES_URL, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + signal, + }) return res.json() } diff --git a/stasis/src/apis/npm/semver.cjs b/stasis-api/src/npm/semver.cjs similarity index 100% rename from stasis/src/apis/npm/semver.cjs rename to stasis-api/src/npm/semver.cjs diff --git a/stasis-api/src/request.js b/stasis-api/src/request.js new file mode 100644 index 00000000..44da9bde --- /dev/null +++ b/stasis-api/src/request.js @@ -0,0 +1,32 @@ +// Shared transport for this package's registry clients: make the request and turn both +// failure modes -- fetch itself rejecting, and a non-2xx answer -- into one Error shape, +// `