From 4623aad2794212c246534fe6d7d8518501a91df0 Mon Sep 17 00:00:00 2001 From: Walter Pinto Date: Sun, 4 Oct 2026 08:41:20 +0100 Subject: [PATCH 1/3] feat(ts): add ts_npm_module, npm packages through Deno's own resolution (prototype) Provide an npm package to Deno's npm: resolution, offline: a sandboxed build step extracts the sha256-pinned tarball into a slice of a Deno npm cache (registry.json + extracted package), bundling the slices of its dependencies and failing if one is not declared. Targets merge the slices into their DENO_DIR, the import map sends the package name to an npm: specifier, and deno check / deno test run with --cached-only so a module missing from deps fails instead of being downloaded. No node_modules and no lockfile. Works for CommonJS packages, subpath imports and exports-only ESM packages (fixtures: debug, highlight.js, @codemirror/legacy-modes). Not wired yet: ts_bundle, ts_binary, Vitest and browser runners, a helper that prints the declarations. Refs #61 Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 16 + build_defs/ts/ts.build_defs | 61 ++++ docs/ts/usage.md | 63 ++++ test/ts/npm_cache/BUILD | 69 +++++ test/ts/npm_cache/highlight.ts | 10 + test/ts/npm_cache/highlight_test.ts | 8 + test/ts/npm_cache/humanize.ts | 6 + test/ts/npm_cache/humanize_test.ts | 8 + test/ts/npm_cache/modes.ts | 7 + test/ts/npm_cache/modes_test.ts | 8 + test/ts/npm_cache/modules/BUILD | 115 +++++++ tools/please_ts/commands.go | 2 + tools/please_ts/compile/BUILD | 10 +- tools/please_ts/compile/compile.go | 15 + tools/please_ts/importmap/BUILD | 2 + tools/please_ts/importmap/importmap.go | 13 + tools/please_ts/importmap/loader.go | 24 +- tools/please_ts/importmap/npm_slice_test.go | 106 +++++++ tools/please_ts/npmcache/BUILD | 15 + tools/please_ts/npmcache/npmcache.go | 184 +++++++++++ tools/please_ts/npmcache/npmcache_test.go | 176 +++++++++++ tools/please_ts/testrunner/BUILD | 2 + tools/please_ts/testrunner/deno.go | 12 + tools/please_ts/unpack/BUILD | 17 +- tools/please_ts/unpack/npmcache.go | 240 +++++++++++++++ tools/please_ts/unpack/npmcache_test.go | 321 ++++++++++++++++++++ tools/please_ts/unpack/unpack.go | 4 + 27 files changed, 1508 insertions(+), 6 deletions(-) create mode 100644 test/ts/npm_cache/BUILD create mode 100644 test/ts/npm_cache/highlight.ts create mode 100644 test/ts/npm_cache/highlight_test.ts create mode 100644 test/ts/npm_cache/humanize.ts create mode 100644 test/ts/npm_cache/humanize_test.ts create mode 100644 test/ts/npm_cache/modes.ts create mode 100644 test/ts/npm_cache/modes_test.ts create mode 100644 test/ts/npm_cache/modules/BUILD create mode 100644 tools/please_ts/importmap/npm_slice_test.go create mode 100644 tools/please_ts/npmcache/BUILD create mode 100644 tools/please_ts/npmcache/npmcache.go create mode 100644 tools/please_ts/npmcache/npmcache_test.go create mode 100644 tools/please_ts/unpack/npmcache.go create mode 100644 tools/please_ts/unpack/npmcache_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 4609941..ffd8890 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,22 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- Experimental `ts_npm_module` rule: provides an npm package to Deno's own + `npm:` resolution, offline. The sha256-pinned tarball is extracted by a + sandboxed build step into a slice of a Deno npm cache that targets merge into + their `DENO_DIR`; no `node_modules`, no lockfile. CommonJS packages, subpath + imports and packages with only an `exports` map work as Deno resolves them. + Dependencies are separate `ts_npm_module` targets and the build fails if one + is missing; targets run with `--cached-only` so a missing module fails instead + of being downloaded. +- `please_ts unpack --npm-cache` and the `npmcache` package behind it. +- Fixtures for `debug` (CommonJS, transitive dependency), `highlight.js` + (CommonJS, subpaths) and `@codemirror/legacy-modes` (exports map only). + ## [0.6.0] - 2026-10-03 ### Added diff --git a/build_defs/ts/ts.build_defs b/build_defs/ts/ts.build_defs index 10f4622..738af02 100644 --- a/build_defs/ts/ts.build_defs +++ b/build_defs/ts/ts.build_defs @@ -219,6 +219,67 @@ def ts_module( building_description = f"Unpacking npm module {pkg}@{ver}...", ) +def ts_npm_module( + name: str, + package: str = "", + version: str = "", + hashes: list = None, + url: str = "", + deps: list = None, + visibility: list = None, + labels: list = None): + """Provides one npm package to Deno's own npm resolution, hermetically and offline. + + The tarball is pinned by `hashes` (sha256, checked by Please). The build step, which + has no network access, extracts it into a slice of a Deno npm cache; targets that + depend on it get the slices merged into their DENO_DIR and import the package through + an `npm:` specifier. Deno then resolves `exports`, CommonJS and subpaths itself: there + is no node_modules directory and no lockfile, and a package that Deno cannot resolve + from the cache fails the build rather than reaching for the network. + + Unlike ts_module, dependencies are not resolved at build time: every dependency of the + package (and theirs) must be its own ts_npm_module, listed in `deps`. The build fails + with the missing names if one is absent. + + Args: + name: Name of the rule. + package: npm package name (defaults to `name`), e.g. "debug" or "@scope/pkg". + version: Exact package version. + hashes: sha256 of the package tarball. + url: Tarball URL (defaults to the npm registry). + deps: ts_npm_module targets for the package's dependencies. + visibility: Visibility of the rule. + labels: Labels for the rule. + """ + pkg = package or name + if not url: + pkg_base = pkg.split("/")[-1] if "/" in pkg else pkg + url = f"https://registry.npmjs.org/{pkg}/-/{pkg_base}-{version}.tgz" + + dl_target = f"_{name}#download" + remote_file( + name = dl_target, + url = url, + hashes = hashes or [], + out = f"{name}.tgz", + ) + + tools = {"TOOL": [_ts_tool()]} + return build_rule( + name = name, + srcs = [f":{dl_target}"], + deps = deps or [], + exported_deps = deps or [], + outs = [name], + cmd = f'$TOOLS_TOOL unpack --npm-cache --tarball "$SRCS" --out "$OUT" --name "{pkg}"', + sandbox = True, + needs_transitive_deps = True, + tools = tools, + visibility = visibility or ["PUBLIC"], + labels = labels or ["ts", "module", "npm", "npm_cache"], + building_description = f"Building Deno npm cache slice for {pkg}@{version}...", + ) + def ts_library( name: str, srcs: list, diff --git a/docs/ts/usage.md b/docs/ts/usage.md index 5ea5c70..faf1c1f 100644 --- a/docs/ts/usage.md +++ b/docs/ts/usage.md @@ -219,3 +219,66 @@ branch (`BRDA`) records that `coverage.xml` and `coverage.json` do not: `tools/common/lcov` package. Do not merge branch records of the Deno and Vitest runners for the same file: they number the arms of a branch differently. + +--- + +## npm Packages Through Deno's Own Resolution (`ts_npm_module`, experimental) + +`ts_module` unpacks a package and describes it with a single entry file, which +cannot express CommonJS packages, subpath imports (`highlight.js/lib/core`) or +packages that only have an `exports` map. `ts_npm_module` takes a different +route: Deno resolves the package itself through an `npm:` specifier, and the +rule only has to make the package available **offline**: + +```starlark +ts_npm_module( + name = "ms", + hashes = ["f6616e15e530ed552f9daa2d3ce71963947c6bc7c98c9b64fd3e673fd02622c6"], + version = "2.1.3", +) + +ts_npm_module( + name = "debug", + hashes = ["c803a8ca9b835b7a75f7150ee52f7f640675515bafbe8f5da78fcc0ae12914ac"], + version = "4.3.7", + deps = [":ms"], # every dependency is its own ts_npm_module +) + +ts_library( + name = "humanize", + srcs = ["humanize.ts"], + module_name = "@app/humanize", + deps = [":debug"], +) + +ts_test( + name = "humanize_test", + srcs = ["humanize_test.ts"], + deps = [":debug", ":humanize"], # list the npm module itself, as with ts_module +) +``` + +- **Hermetic**: the tarball is pinned by `hashes` (sha256, checked by Please). + The build step runs in the sandbox, without network access, and extracts it + into a slice of a Deno npm cache. There is no `node_modules` directory and no + lockfile; the BUILD files are the lock. +- **Dependencies are explicit**: nothing is resolved at build time. If a package + depends on something that is not in `deps`, the build fails and names it. Each + module bundles the packages of its dependencies, so a target lists only the + modules it imports. +- **Resolution is Deno's**: `exports` maps, CommonJS, subpaths and a package's + own types work as they do for `npm:` specifiers. Imports use the plain package + name, for example `import createDebug from "debug"`. +- **Offline is enforced**: targets whose import map has npm modules run Deno + with `--cached-only`. A module missing from a target's `deps` fails at once + with `npm package not found in cache`; Deno does not download it. +- **Not wired yet**: `ts_bundle` and `ts_binary`, the Vitest and browser + runners, and a helper that prints the declarations for a package and its + dependencies. The cache layout (`registry.json`, including its + `_deno.packumentFormat` key) is internal to Deno, so it is tied to the Deno + version the plugin pins. + +The fixtures in `test/ts/npm_cache` cover a CommonJS package with a transitive +dependency (`debug`), a CommonJS package imported through subpaths +(`highlight.js`), and an ES module package that only has an `exports` map and no +`main` (`@codemirror/legacy-modes`, with its eleven dependencies). diff --git a/test/ts/npm_cache/BUILD b/test/ts/npm_cache/BUILD new file mode 100644 index 0000000..bfb44e9 --- /dev/null +++ b/test/ts/npm_cache/BUILD @@ -0,0 +1,69 @@ +subinclude("//build_defs/ts:ts") + +# A CommonJS package and its transitive dependency, each pinned by the sha256 of its tarball. +ts_npm_module( + name = "ms", + hashes = ["f6616e15e530ed552f9daa2d3ce71963947c6bc7c98c9b64fd3e673fd02622c6"], + version = "2.1.3", +) + +ts_npm_module( + name = "debug", + hashes = ["c803a8ca9b835b7a75f7150ee52f7f640675515bafbe8f5da78fcc0ae12914ac"], + version = "4.3.7", + deps = [":ms"], +) + +ts_library( + name = "humanize", + srcs = ["humanize.ts"], + module_name = "@test/humanize", + deps = [":debug"], +) + +ts_test( + name = "humanize_test", + srcs = ["humanize_test.ts"], + deps = [ + ":debug", + ":humanize", + ], +) + +# highlight.js is CommonJS and is imported through subpaths (highlight.js/lib/core). +ts_library( + name = "highlight", + srcs = ["highlight.ts"], + module_name = "@test/highlight", + deps = ["//test/ts/npm_cache/modules:highlight_js"], +) + +ts_test( + name = "highlight_test", + srcs = ["highlight_test.ts"], + deps = [ + ":highlight", + "//test/ts/npm_cache/modules:highlight_js", + ], +) + +# @codemirror/legacy-modes is ESM with only an exports map (./mode/*) and no main or module. +ts_library( + name = "modes", + srcs = ["modes.ts"], + module_name = "@test/modes", + deps = [ + "//test/ts/npm_cache/modules:codemirror_language", + "//test/ts/npm_cache/modules:codemirror_legacy_modes", + ], +) + +ts_test( + name = "modes_test", + srcs = ["modes_test.ts"], + deps = [ + ":modes", + "//test/ts/npm_cache/modules:codemirror_language", + "//test/ts/npm_cache/modules:codemirror_legacy_modes", + ], +) diff --git a/test/ts/npm_cache/highlight.ts b/test/ts/npm_cache/highlight.ts new file mode 100644 index 0000000..496c427 --- /dev/null +++ b/test/ts/npm_cache/highlight.ts @@ -0,0 +1,10 @@ +import core from "highlight.js/lib/core"; +import go from "highlight.js/lib/languages/go"; + +const hljs: any = core; +hljs.registerLanguage("go", go); + +/** Highlights Go source as HTML. */ +export function highlight(source: string): string { + return hljs.highlight(source, { language: "go" }).value; +} diff --git a/test/ts/npm_cache/highlight_test.ts b/test/ts/npm_cache/highlight_test.ts new file mode 100644 index 0000000..00fe786 --- /dev/null +++ b/test/ts/npm_cache/highlight_test.ts @@ -0,0 +1,8 @@ +import { highlight } from "@test/highlight"; + +Deno.test("highlight.js: subpath imports of a CommonJS package", () => { + const html = highlight("package main"); + if (!html.includes("hljs-keyword")) { + throw new Error(`expected a highlighted keyword, got ${html}`); + } +}); diff --git a/test/ts/npm_cache/humanize.ts b/test/ts/npm_cache/humanize.ts new file mode 100644 index 0000000..cbdba6d --- /dev/null +++ b/test/ts/npm_cache/humanize.ts @@ -0,0 +1,6 @@ +import createDebug from "debug"; + +/** Formats a duration in milliseconds with the `ms` package, which `debug` re-exports. */ +export function humanize(milliseconds: number): string { + return (createDebug as any).humanize(milliseconds); +} diff --git a/test/ts/npm_cache/humanize_test.ts b/test/ts/npm_cache/humanize_test.ts new file mode 100644 index 0000000..e32cb45 --- /dev/null +++ b/test/ts/npm_cache/humanize_test.ts @@ -0,0 +1,8 @@ +import { humanize } from "@test/humanize"; + +Deno.test("humanize formats durations through a CommonJS package and its dependency", () => { + const got = humanize(90061000); + if (got !== "1d") { + throw new Error(`expected 1d, got ${got}`); + } +}); diff --git a/test/ts/npm_cache/modes.ts b/test/ts/npm_cache/modes.ts new file mode 100644 index 0000000..ca14661 --- /dev/null +++ b/test/ts/npm_cache/modes.ts @@ -0,0 +1,7 @@ +import { go } from "@codemirror/legacy-modes/mode/go"; +import { StreamLanguage } from "@codemirror/language"; + +/** A CodeMirror language for Go, built from a legacy mode. */ +export function goLanguage(): StreamLanguage { + return StreamLanguage.define(go); +} diff --git a/test/ts/npm_cache/modes_test.ts b/test/ts/npm_cache/modes_test.ts new file mode 100644 index 0000000..6809a59 --- /dev/null +++ b/test/ts/npm_cache/modes_test.ts @@ -0,0 +1,8 @@ +import { goLanguage } from "@test/modes"; + +Deno.test("legacy-modes: an ESM package that only has an exports map", () => { + const language = goLanguage(); + if (!language || typeof language.parser !== "object") { + throw new Error("expected a StreamLanguage with a parser"); + } +}); diff --git a/test/ts/npm_cache/modules/BUILD b/test/ts/npm_cache/modules/BUILD new file mode 100644 index 0000000..3da194b --- /dev/null +++ b/test/ts/npm_cache/modules/BUILD @@ -0,0 +1,115 @@ +subinclude("//build_defs/ts:ts") + +# Pinned npm packages for the fixtures: highlight.js (CommonJS, subpath imports) and +# @codemirror/legacy-modes (ESM with only an exports map, no main/module) and its dependencies. +# Every package is its own target with the sha256 of its tarball; dependencies are listed in deps. +# (Generated from an npm lock; the planned helper would print these declarations.) + +ts_npm_module( + name = "codemirror_language", + package = "@codemirror/language", + hashes = ["5e49acf55fde65ce9848068f0f06478be9ec71f818e91de6eca00824e9152226"], + version = "6.12.4", + deps = [ + ":codemirror_state", + ":codemirror_view", + ":lezer_common", + ":lezer_highlight", + ":lezer_lr", + ":style_mod", + ], +) + +ts_npm_module( + name = "codemirror_legacy_modes", + package = "@codemirror/legacy-modes", + hashes = ["61143cdb9c375dc1521895e4e536d75036baaf6e9df6b4431691f7ea5d273463"], + version = "6.5.1", + deps = [ + ":codemirror_language", + ], +) + +ts_npm_module( + name = "codemirror_state", + package = "@codemirror/state", + hashes = ["91d75acc955ceeaf86af9396c11cda7694fe6222aac8427fa033d89eff5ac839"], + version = "6.7.6", + deps = [ + ":marijn_find_cluster_break", + ], +) + +ts_npm_module( + name = "codemirror_view", + package = "@codemirror/view", + hashes = ["1246c9b6e1ad1d9d870b3e67672d3a8f8c9bc73af3b63c081c1549a94d5850db"], + version = "6.43.13", + deps = [ + ":codemirror_state", + ":crelt", + ":style_mod", + ":w3c_keyname", + ], +) + +ts_npm_module( + name = "lezer_common", + package = "@lezer/common", + hashes = ["a8854639c04adabe5e72c26fa2876475116dc099ae5aa02250a5a938bb5ea4ae"], + version = "1.5.3", +) + +ts_npm_module( + name = "lezer_highlight", + package = "@lezer/highlight", + hashes = ["e349eec1ff4382439c1e6286ae937e0203a8349c5192e2e133863391775ac7ff"], + version = "1.2.5", + deps = [ + ":lezer_common", + ], +) + +ts_npm_module( + name = "lezer_lr", + package = "@lezer/lr", + hashes = ["c71e654cd0d153898e744a613bb82714fcd3c7d8e4df8cef7d193fd5ddfa6015"], + version = "1.4.10", + deps = [ + ":lezer_common", + ], +) + +ts_npm_module( + name = "marijn_find_cluster_break", + package = "@marijn/find-cluster-break", + hashes = ["5d12b770b64d46632c84fcd18e1d4c09f195ed0f5dbc71f94a41f0af333eaeae"], + version = "1.0.4", +) + +ts_npm_module( + name = "crelt", + hashes = ["3542299c0278fdc26aceb2b13d31bffbcb89d7367706bbc457130e3fd103c03e"], + version = "1.0.7", +) + +ts_npm_module( + name = "highlight_js", + package = "highlight.js", + hashes = ["accbfaaab745088609b4eea2bdca2ad62f1f1dd27304e0f8df65cfe0fe042143"], + version = "11.12.0", +) + +ts_npm_module( + name = "style_mod", + package = "style-mod", + hashes = ["769073434ab698eb035cbaa9df5d028231c25d95bc7b8676a46d603fb2e89e39"], + version = "4.1.4", +) + +ts_npm_module( + name = "w3c_keyname", + package = "w3c-keyname", + hashes = ["05d21484026d1f2dd842c843181e391e66d60f1d5dff292d562c3f40302dab36"], + version = "2.2.8", +) diff --git a/tools/please_ts/commands.go b/tools/please_ts/commands.go index a4010d1..fc7d59e 100644 --- a/tools/please_ts/commands.go +++ b/tools/please_ts/commands.go @@ -194,6 +194,7 @@ func handleUnpack(args []string) error { symlink := cmd.String("symlink", "", "Optional symlink name for extracted binary") resolveTransitive := cmd.Bool("resolve-transitive", false, "Recursively resolve and download transitive dependencies") registry := cmd.String("registry", "", "NPM registry base URL") + npmCache := cmd.Bool("npm-cache", false, "Build a Deno npm cache slice from the tarball (offline, with a dependency closure check)") if err := cmd.Parse(args); err != nil { return err @@ -213,6 +214,7 @@ func handleUnpack(args []string) error { Symlink: *symlink, ResolveTransitive: *resolveTransitive, Registry: *registry, + NpmCache: *npmCache, } return unpack.Run(opts) } diff --git a/tools/please_ts/compile/BUILD b/tools/please_ts/compile/BUILD index 5309481..cc53324 100644 --- a/tools/please_ts/compile/BUILD +++ b/tools/please_ts/compile/BUILD @@ -4,7 +4,10 @@ go_library( name = "compile", srcs = ["compile.go"], visibility = ["//tools/please_ts/..."], - deps = ["//tools/please_ts/importmap"], + deps = [ + "//tools/please_ts/importmap", + "//tools/please_ts/npmcache", + ], ) go_test( @@ -13,5 +16,8 @@ go_test( "compile.go", "compile_test.go", ], - deps = ["//tools/please_ts/importmap"], + deps = [ + "//tools/please_ts/importmap", + "//tools/please_ts/npmcache", + ], ) diff --git a/tools/please_ts/compile/compile.go b/tools/please_ts/compile/compile.go index 9d6ae75..0f69456 100644 --- a/tools/please_ts/compile/compile.go +++ b/tools/please_ts/compile/compile.go @@ -10,6 +10,7 @@ import ( "strings" "tools/please_ts/importmap" + "tools/please_ts/npmcache" ) // Options holds configuration for TypeScript library compilation. @@ -53,6 +54,14 @@ func Run(opts Options) error { } } + // npm packages provided by ts_npm_module targets are merged into the per-run Deno cache + // and resolved from there, offline. The shared Vitest cache is left as it is. + if opts.VitestDir == "" { + if _, err := npmcache.Prepare(".", denoCacheDir); err != nil { + return fmt.Errorf("failed preparing the npm cache: %w", err) + } + } + // 1. Synthesize target-local import map importMapPath := ".import_map.json" im, err := importmap.Synthesize(opts.ModuleName, opts.Srcs, opts.Deps, ".") @@ -73,6 +82,12 @@ func Run(opts Options) error { "--no-remote", "--import-map", importMapPath, } + // npm packages come from the merged cache only: a ts_npm_module missing from the + // target's deps must fail here, not be downloaded. (The shared Vitest cache is a + // different mechanism and keeps its own behaviour.) + if opts.VitestDir == "" && im.HasNpmSpecifiers() { + args = append(args, "--cached-only") + } args = append(args, opts.Flags...) args = append(args, opts.Srcs...) diff --git a/tools/please_ts/importmap/BUILD b/tools/please_ts/importmap/BUILD index e74ef99..9870036 100644 --- a/tools/please_ts/importmap/BUILD +++ b/tools/please_ts/importmap/BUILD @@ -4,9 +4,11 @@ go_library( name = "importmap", srcs = glob(["*.go"], exclude = ["*_test.go"]), visibility = ["//tools/please_ts/..."], + deps = ["//tools/please_ts/npmcache"], ) go_test( name = "importmap_test", srcs = glob(["*.go"]), + deps = ["//tools/please_ts/npmcache"], ) diff --git a/tools/please_ts/importmap/importmap.go b/tools/please_ts/importmap/importmap.go index 085f751..965b933 100644 --- a/tools/please_ts/importmap/importmap.go +++ b/tools/please_ts/importmap/importmap.go @@ -117,3 +117,16 @@ func (im *ImportMap) WriteToFile(filePath string) error { } return os.WriteFile(filePath, data, 0644) } + +// HasNpmSpecifiers reports whether the import map sends any module to an npm: specifier. +// Those are resolved from the Deno cache that the npm slices are merged into, so a target +// whose import map has them must run with --cached-only: a slice that is missing from the +// target's dependencies then fails at once, instead of Deno quietly downloading the package. +func (im *ImportMap) HasNpmSpecifiers() bool { + for _, target := range im.Imports { + if strings.HasPrefix(target, "npm:") { + return true + } + } + return false +} diff --git a/tools/please_ts/importmap/loader.go b/tools/please_ts/importmap/loader.go index 65ff787..87533eb 100644 --- a/tools/please_ts/importmap/loader.go +++ b/tools/please_ts/importmap/loader.go @@ -6,6 +6,8 @@ import ( "os" "path/filepath" "strings" + + "tools/please_ts/npmcache" ) // discoverDepPaths collects explicit dependencies and auto-discovers module metadata files in workingDir. @@ -15,7 +17,7 @@ func discoverDepPaths(deps []string, workingDir string) []string { _ = filepath.Walk(workingDir, func(path string, info os.FileInfo, err error) error { if err == nil && !info.IsDir() { - if info.Name() == "ts_metadata.json" || info.Name() == "ts_module.json" { + if info.Name() == "ts_metadata.json" || info.Name() == "ts_module.json" || info.Name() == npmcache.MetadataFile { allDeps = append(allDeps, path) } } @@ -51,6 +53,9 @@ func (im *ImportMap) LoadDependencies(depPaths []string, workingDir string) ([]* // LoadDepItem handles a single dependency path: metadata file, directory, or source file. func (im *ImportMap) LoadDepItem(dep string, workingDir string) (*loadedModule, error) { + if filepath.Base(dep) == npmcache.MetadataFile { + return nil, im.registerNpmSlice(filepath.Dir(dep)) + } if isMetadataFile(dep) { mod, err := loadMetadataFile(dep, workingDir) if err != nil { @@ -73,6 +78,10 @@ func (im *ImportMap) LoadDepItem(dep string, workingDir string) (*loadedModule, // LoadDirectoryDep checks for metadata inside a directory or maps it directly. func (im *ImportMap) LoadDirectoryDep(dir, workingDir string) (*loadedModule, error) { + if _, err := os.Stat(filepath.Join(dir, npmcache.MetadataFile)); err == nil { + return nil, im.registerNpmSlice(dir) + } + metaPath := filepath.Join(dir, "ts_module.json") if _, err := os.Stat(metaPath); err == nil { return loadMetadataFile(metaPath, workingDir) @@ -87,6 +96,19 @@ func (im *ImportMap) LoadDirectoryDep(dir, workingDir string) (*loadedModule, er return nil, nil } +// registerNpmSlice maps an npm package provided by a ts_npm_module to npm: specifiers that +// Deno resolves from its cache (see package npmcache): the bare name for the package +// entry, and name/ for subpaths, which Deno resolves with the package's own exports map. +func (im *ImportMap) registerNpmSlice(dir string) error { + slice, err := npmcache.Read(dir) + if err != nil { + return fmt.Errorf("failed loading npm slice %s: %w", dir, err) + } + im.Imports[slice.Name] = slice.Specifier + im.Imports[slice.Name+"/"] = "npm:/" + slice.Name + "@" + slice.Version + "/" + return nil +} + // MapDirectSourceFile maps a single source file to imports. func (im *ImportMap) MapDirectSourceFile(filePath, workingDir string) { relPath, err := relativeTo(workingDir, filePath) diff --git a/tools/please_ts/importmap/npm_slice_test.go b/tools/please_ts/importmap/npm_slice_test.go new file mode 100644 index 0000000..ecc5310 --- /dev/null +++ b/tools/please_ts/importmap/npm_slice_test.go @@ -0,0 +1,106 @@ +package importmap + +import ( + "os" + "path/filepath" + "testing" + + "tools/please_ts/npmcache" +) + +func writeNpmSlice(t *testing.T, root, dir string, s npmcache.Slice) string { + t.Helper() + d := filepath.Join(root, dir) + if err := os.MkdirAll(d, 0755); err != nil { + t.Fatal(err) + } + if err := npmcache.Write(d, s); err != nil { + t.Fatal(err) + } + return d +} + +func TestSynthesizeMapsNpmSlicesToNpmSpecifiers(t *testing.T) { + root := t.TempDir() + writeNpmSlice(t, root, "third_party/debug", npmcache.Slice{Name: "debug", Version: "4.3.7", Specifier: "npm:debug@4.3.7"}) + writeNpmSlice(t, root, "third_party/scoped", npmcache.Slice{Name: "@codemirror/legacy-modes", Version: "6.5.1", Specifier: "npm:@codemirror/legacy-modes@6.5.1"}) + + im, err := Synthesize("", nil, nil, root) + if err != nil { + t.Fatal(err) + } + want := map[string]string{ + "debug": "npm:debug@4.3.7", + "debug/": "npm:/debug@4.3.7/", + "@codemirror/legacy-modes": "npm:@codemirror/legacy-modes@6.5.1", + "@codemirror/legacy-modes/": "npm:/@codemirror/legacy-modes@6.5.1/", + } + for k, v := range want { + if im.Imports[k] != v { + t.Errorf("imports[%q] = %q, want %q", k, im.Imports[k], v) + } + } +} + +func TestSynthesizeMapsExplicitNpmSliceDirectoryAndMetadataFile(t *testing.T) { + root := t.TempDir() + dir := writeNpmSlice(t, root, "third_party/ms", npmcache.Slice{Name: "ms", Version: "2.1.3", Specifier: "npm:ms@2.1.3"}) + + for name, dep := range map[string]string{"directory": dir, "metadata file": filepath.Join(dir, npmcache.MetadataFile)} { + im := New() + if _, err := im.LoadDependencies([]string{dep}, root); err != nil { + t.Fatalf("%s: %v", name, err) + } + if im.Imports["ms"] != "npm:ms@2.1.3" || im.Imports["ms/"] != "npm:/ms@2.1.3/" { + t.Errorf("%s: imports = %v", name, im.Imports) + } + } +} + +func TestSynthesizeReportsAnUnreadableNpmSlice(t *testing.T) { + root := t.TempDir() + d := filepath.Join(root, "third_party", "broken") + if err := os.MkdirAll(d, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(d, npmcache.MetadataFile), []byte("{"), 0644); err != nil { + t.Fatal(err) + } + if _, err := Synthesize("", nil, nil, root); err == nil { + t.Error("expected an error for invalid ts_npm.json") + } +} + +func TestNpmSlicesLeaveFirstPartyAliasesAlone(t *testing.T) { + root := t.TempDir() + writeNpmSlice(t, root, "third_party/ms", npmcache.Slice{Name: "ms", Version: "2.1.3", Specifier: "npm:ms@2.1.3"}) + src := filepath.Join(root, "app", "app.ts") + if err := os.MkdirAll(filepath.Dir(src), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(src, []byte("export {};"), 0644); err != nil { + t.Fatal(err) + } + im, err := Synthesize("@app/app", []string{src}, nil, root) + if err != nil { + t.Fatal(err) + } + if im.Imports["@app/app"] == "" || im.Imports["ms"] != "npm:ms@2.1.3" { + t.Errorf("imports = %v", im.Imports) + } +} + +func TestHasNpmSpecifiers(t *testing.T) { + im := New() + if im.HasNpmSpecifiers() { + t.Error("an empty import map has no npm specifiers") + } + im.Imports["@app/lib"] = "./lib/lib.ts" + if im.HasNpmSpecifiers() { + t.Error("first-party aliases are not npm specifiers") + } + im.Imports["ms/"] = "npm:/ms@2.1.3/" + if !im.HasNpmSpecifiers() { + t.Error("an npm:/ prefix mapping is an npm specifier") + } +} diff --git a/tools/please_ts/npmcache/BUILD b/tools/please_ts/npmcache/BUILD new file mode 100644 index 0000000..a92da92 --- /dev/null +++ b/tools/please_ts/npmcache/BUILD @@ -0,0 +1,15 @@ +subinclude("///go//build_defs:go") + +go_library( + name = "npmcache", + srcs = ["npmcache.go"], + visibility = ["//tools/please_ts/..."], +) + +go_test( + name = "npmcache_test", + srcs = [ + "npmcache.go", + "npmcache_test.go", + ], +) diff --git a/tools/please_ts/npmcache/npmcache.go b/tools/please_ts/npmcache/npmcache.go new file mode 100644 index 0000000..909215a --- /dev/null +++ b/tools/please_ts/npmcache/npmcache.go @@ -0,0 +1,184 @@ +// Package npmcache handles npm packages as slices of a Deno npm cache. +// +// A slice is the output of a ts_npm_module target: a directory holding the package +// extracted into Deno's cache layout (npm/registry.npmjs.org/// plus a +// registry.json), and a ts_npm.json describing it. Slices are built offline from a +// hash-pinned tarball; the runners merge the slices a target depends on into the +// per-run DENO_DIR, so Deno resolves npm: specifiers without network access, without a +// node_modules directory and without a lockfile. +package npmcache + +import ( + "encoding/json" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" +) + +// MetadataFile is the name of the file that marks a directory as a slice. +const MetadataFile = "ts_npm.json" + +// registryDir is the directory under the Deno cache that holds npm packages. +const registryDir = "npm/registry.npmjs.org" + +// Slice describes one npm package of the cache. +type Slice struct { + Name string `json:"name"` + Version string `json:"version"` + Specifier string `json:"specifier"` + Dependencies map[string]string `json:"dependencies,omitempty"` +} + +// Specifier returns the npm: specifier of an exact package version. +func Specifier(name, version string) string { + return "npm:" + name + "@" + version +} + +// Read loads the slice described by dir/ts_npm.json. +func Read(dir string) (*Slice, error) { + data, err := os.ReadFile(filepath.Join(dir, MetadataFile)) + if err != nil { + return nil, err + } + var s Slice + if err := json.Unmarshal(data, &s); err != nil { + return nil, fmt.Errorf("%s: %w", filepath.Join(dir, MetadataFile), err) + } + if s.Name == "" || s.Version == "" { + return nil, fmt.Errorf("%s: name and version are required", filepath.Join(dir, MetadataFile)) + } + return &s, nil +} + +// Write stores the slice metadata in dir/ts_npm.json. +func Write(dir string, s Slice) error { + data, err := json.MarshalIndent(s, "", " ") + if err != nil { + return err + } + return os.WriteFile(filepath.Join(dir, MetadataFile), data, 0644) +} + +// Discover returns the slice directories below root, sorted. +func Discover(root string) []string { + var dirs []string + _ = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err == nil && !d.IsDir() && d.Name() == MetadataFile { + dirs = append(dirs, filepath.Dir(path)) + } + return nil + }) + sort.Strings(dirs) + return dirs +} + +// Merge copies the npm cache content of every slice into denoDir (a DENO_DIR). Files +// that already exist are kept, except registry.json, whose versions are unioned so that +// two slices holding different versions of one package both stay resolvable. +func Merge(slices []string, denoDir string) error { + for _, slice := range slices { + src := filepath.Join(slice, "npm") + if _, err := os.Stat(src); err != nil { + continue + } + err := filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(slice, path) + if err != nil { + return err + } + dst := filepath.Join(denoDir, rel) + if d.IsDir() { + return os.MkdirAll(dst, 0755) + } + if _, err := os.Stat(dst); err == nil { + if d.Name() == "registry.json" && strings.Contains(filepath.ToSlash(dst), registryDir) { + return mergePackument(path, dst) + } + return nil + } + return copyFile(path, dst) + }) + if err != nil { + return fmt.Errorf("merging npm slice %s: %w", slice, err) + } + } + return nil +} + +// Prepare merges the slices found below root into denoDir and reports whether there were +// any. Callers add --cached-only to the Deno command when it returns true, so a package +// missing from the cache fails at once instead of reaching for the network. +func Prepare(root, denoDir string) (bool, error) { + slices := Discover(root) + if len(slices) == 0 { + return false, nil + } + if err := os.MkdirAll(denoDir, 0755); err != nil { + return false, err + } + return true, Merge(slices, denoDir) +} + +func copyFile(src, dst string) error { + info, err := os.Stat(src) + if err != nil { + return err + } + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, info.Mode().Perm()|0200) + if err != nil { + return err + } + if _, err := io.Copy(out, in); err != nil { + out.Close() + return err + } + return out.Close() +} + +// mergePackument adds the versions of the registry.json at src to the one at dst. +func mergePackument(src, dst string) error { + var a, b map[string]json.RawMessage + for path, target := range map[string]*map[string]json.RawMessage{dst: &a, src: &b} { + data, err := os.ReadFile(path) + if err != nil { + return err + } + if err := json.Unmarshal(data, target); err != nil { + return fmt.Errorf("%s: %w", path, err) + } + } + var va, vb map[string]json.RawMessage + if err := json.Unmarshal(a["versions"], &va); err != nil { + return err + } + if err := json.Unmarshal(b["versions"], &vb); err != nil { + return err + } + for v, raw := range vb { + if _, ok := va[v]; !ok { + va[v] = raw + } + } + merged, err := json.Marshal(va) + if err != nil { + return err + } + a["versions"] = merged + out, err := json.Marshal(a) + if err != nil { + return err + } + return os.WriteFile(dst, out, 0644) +} diff --git a/tools/please_ts/npmcache/npmcache_test.go b/tools/please_ts/npmcache/npmcache_test.go new file mode 100644 index 0000000..6a07336 --- /dev/null +++ b/tools/please_ts/npmcache/npmcache_test.go @@ -0,0 +1,176 @@ +package npmcache + +import ( + "encoding/json" + "os" + "path/filepath" + "reflect" + "testing" +) + +func makeSlice(t *testing.T, root, name, version string, deps map[string]string) string { + t.Helper() + dir := filepath.Join(root, "slices", name+"-"+version) + pkgDir := filepath.Join(dir, "npm", "registry.npmjs.org", name, version) + if err := os.MkdirAll(pkgDir, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(pkgDir, "index.js"), []byte("module.exports = '"+version+"';"), 0644); err != nil { + t.Fatal(err) + } + packument := map[string]any{ + "name": name, + "versions": map[string]any{version: map[string]any{"name": name, "version": version}}, + "dist-tags": map[string]string{"latest": version}, + } + data, _ := json.Marshal(packument) + if err := os.WriteFile(filepath.Join(dir, "npm", "registry.npmjs.org", name, "registry.json"), data, 0644); err != nil { + t.Fatal(err) + } + if err := Write(dir, Slice{Name: name, Version: version, Specifier: Specifier(name, version), Dependencies: deps}); err != nil { + t.Fatal(err) + } + return dir +} + +func TestWriteReadRoundTrip(t *testing.T) { + dir := t.TempDir() + want := Slice{Name: "@scope/pkg", Version: "1.2.3", Specifier: "npm:@scope/pkg@1.2.3", Dependencies: map[string]string{"ms": "^2.1.3"}} + if err := Write(dir, want); err != nil { + t.Fatal(err) + } + got, err := Read(dir) + if err != nil || !reflect.DeepEqual(*got, want) { + t.Fatalf("Read = %+v, %v; want %+v", got, err, want) + } +} + +func TestReadRejectsIncompleteMetadata(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, MetadataFile), []byte(`{"name":"x"}`), 0644); err != nil { + t.Fatal(err) + } + if _, err := Read(dir); err == nil { + t.Error("expected an error for metadata without a version") + } + if _, err := Read(t.TempDir()); err == nil { + t.Error("expected an error for a missing file") + } +} + +func TestDiscoverFindsSlicesSorted(t *testing.T) { + root := t.TempDir() + b := makeSlice(t, root, "b", "1.0.0", nil) + a := makeSlice(t, root, "a", "1.0.0", nil) + got := Discover(root) + if !reflect.DeepEqual(got, []string{a, b}) { + t.Errorf("Discover = %v, want [%s %s]", got, a, b) + } + if len(Discover(t.TempDir())) != 0 { + t.Error("an empty tree has no slices") + } +} + +func TestMergeCopiesPackagesIntoDenoDir(t *testing.T) { + root := t.TempDir() + debug := makeSlice(t, root, "debug", "4.3.7", map[string]string{"ms": "^2.1.3"}) + ms := makeSlice(t, root, "ms", "2.1.3", nil) + denoDir := filepath.Join(t.TempDir(), "deno") + + if err := Merge([]string{debug, ms}, denoDir); err != nil { + t.Fatal(err) + } + for _, p := range []string{ + "npm/registry.npmjs.org/debug/4.3.7/index.js", + "npm/registry.npmjs.org/debug/registry.json", + "npm/registry.npmjs.org/ms/2.1.3/index.js", + "npm/registry.npmjs.org/ms/registry.json", + } { + if _, err := os.Stat(filepath.Join(denoDir, p)); err != nil { + t.Errorf("missing %s: %v", p, err) + } + } + if _, err := os.Stat(filepath.Join(denoDir, MetadataFile)); err == nil { + t.Error("slice metadata must not be copied into the cache") + } +} + +func TestMergeUnionsVersionsOfTheSamePackage(t *testing.T) { + root := t.TempDir() + old := makeSlice(t, root, "ms", "2.0.0", nil) + cur := makeSlice(t, root, "ms", "2.1.3", nil) + denoDir := t.TempDir() + + if err := Merge([]string{old, cur}, denoDir); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(denoDir, "npm/registry.npmjs.org/ms/registry.json")) + if err != nil { + t.Fatal(err) + } + var packument struct { + Versions map[string]json.RawMessage `json:"versions"` + } + if err := json.Unmarshal(data, &packument); err != nil { + t.Fatal(err) + } + if len(packument.Versions) != 2 { + t.Errorf("versions = %v, want both 2.0.0 and 2.1.3", packument.Versions) + } + for _, v := range []string{"2.0.0", "2.1.3"} { + if _, err := os.Stat(filepath.Join(denoDir, "npm/registry.npmjs.org/ms", v, "index.js")); err != nil { + t.Errorf("version %s missing: %v", v, err) + } + } +} + +func TestMergeIsIdempotentAndKeepsExistingFiles(t *testing.T) { + root := t.TempDir() + s := makeSlice(t, root, "ms", "2.1.3", nil) + denoDir := t.TempDir() + for i := 0; i < 2; i++ { + if err := Merge([]string{s}, denoDir); err != nil { + t.Fatal(err) + } + } + target := filepath.Join(denoDir, "npm/registry.npmjs.org/ms/2.1.3/index.js") + if err := os.WriteFile(target, []byte("kept"), 0644); err != nil { + t.Fatal(err) + } + if err := Merge([]string{s}, denoDir); err != nil { + t.Fatal(err) + } + if got, _ := os.ReadFile(target); string(got) != "kept" { + t.Errorf("an existing file was overwritten: %q", got) + } +} + +func TestMergeSkipsSlicesWithoutCacheContent(t *testing.T) { + dir := t.TempDir() + if err := Merge([]string{dir}, t.TempDir()); err != nil { + t.Errorf("a directory without npm/ must be skipped, got %v", err) + } +} + +func TestPrepareMergesAndReportsWhetherThereWereSlices(t *testing.T) { + root := t.TempDir() + makeSlice(t, root, "ms", "2.1.3", nil) + denoDir := filepath.Join(t.TempDir(), "deno") + + found, err := Prepare(root, denoDir) + if err != nil || !found { + t.Fatalf("Prepare = %v, %v; want true", found, err) + } + if _, err := os.Stat(filepath.Join(denoDir, "npm/registry.npmjs.org/ms/2.1.3/index.js")); err != nil { + t.Errorf("package not merged: %v", err) + } + + empty := filepath.Join(t.TempDir(), "deno") + found, err = Prepare(t.TempDir(), empty) + if err != nil || found { + t.Fatalf("Prepare without slices = %v, %v; want false", found, err) + } + if _, err := os.Stat(empty); err == nil { + t.Error("the Deno directory must be left alone when there is nothing to merge") + } +} diff --git a/tools/please_ts/testrunner/BUILD b/tools/please_ts/testrunner/BUILD index b443715..4cc5484 100644 --- a/tools/please_ts/testrunner/BUILD +++ b/tools/please_ts/testrunner/BUILD @@ -8,6 +8,7 @@ go_library( "//tools/common/lcov", "//tools/please_ts/bundle", "//tools/please_ts/importmap", + "//tools/please_ts/npmcache", ], ) @@ -18,5 +19,6 @@ go_test( "//tools/common/lcov", "//tools/please_ts/bundle", "//tools/please_ts/importmap", + "//tools/please_ts/npmcache", ], ) diff --git a/tools/please_ts/testrunner/deno.go b/tools/please_ts/testrunner/deno.go index acf863c..5983368 100644 --- a/tools/please_ts/testrunner/deno.go +++ b/tools/please_ts/testrunner/deno.go @@ -8,6 +8,7 @@ import ( "strings" "tools/please_ts/importmap" + "tools/please_ts/npmcache" ) func (opts RunOptions) runDeno(resultsFile string) error { @@ -29,6 +30,12 @@ func (opts RunOptions) runDeno(resultsFile string) error { return fmt.Errorf("failed creating DENO_DIR: %w", err) } + // npm packages provided by ts_npm_module targets are merged into the per-run Deno cache + // and resolved from there, offline. + if _, err := npmcache.Prepare(".", denoCacheDir); err != nil { + return fmt.Errorf("failed preparing the npm cache: %w", err) + } + // 1. Synthesize target-local import map importMapPath := ".import_map.json" im, err := importmap.Synthesize(opts.ModuleName, opts.Srcs, opts.Deps, ".") @@ -52,6 +59,11 @@ func (opts RunOptions) runDeno(resultsFile string) error { "--import-map", importMapPath, "--junit-path", resultsFile, } + // npm packages come from the merged cache only: a ts_npm_module missing from the + // target's deps must fail here, not be downloaded. + if im.HasNpmSpecifiers() { + args = append(args, "--cached-only") + } covDir := "" if coverageActive { diff --git a/tools/please_ts/unpack/BUILD b/tools/please_ts/unpack/BUILD index 80f2af6..4aceab7 100644 --- a/tools/please_ts/unpack/BUILD +++ b/tools/please_ts/unpack/BUILD @@ -2,16 +2,27 @@ subinclude("///go//build_defs:go") go_library( name = "unpack", - srcs = ["unpack.go"], + srcs = [ + "npmcache.go", + "unpack.go", + ], visibility = ["//tools/please_ts/..."], - deps = ["//tools/please_ts/importmap"], + deps = [ + "//tools/please_ts/importmap", + "//tools/please_ts/npmcache", + ], ) go_test( name = "unpack_test", srcs = [ + "npmcache.go", + "npmcache_test.go", "unpack.go", "unpack_test.go", ], - deps = ["//tools/please_ts/importmap"], + deps = [ + "//tools/please_ts/importmap", + "//tools/please_ts/npmcache", + ], ) diff --git a/tools/please_ts/unpack/npmcache.go b/tools/please_ts/unpack/npmcache.go new file mode 100644 index 0000000..05a9f0f --- /dev/null +++ b/tools/please_ts/unpack/npmcache.go @@ -0,0 +1,240 @@ +package unpack + +import ( + "crypto/sha512" + "encoding/base64" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "tools/please_ts/npmcache" +) + +// buildNpmCache turns a hash-pinned npm tarball into a slice of a Deno npm cache, with no +// network access: the package is extracted into npm/registry.npmjs.org/// +// and described by a minimal registry.json, so that Deno resolves npm:@ +// from its cache. Deno does not verify the cache, so the integrity of the package is the +// sha256 Please checked on the tarball. +// +// Every dependency the package declares must be provided by another slice staged in the +// build directory (a ts_npm_module in deps); otherwise Deno would only notice at run time, +// by trying the network. +func buildNpmCache(opts Options) error { + tmp, err := os.MkdirTemp("", "please_ts_npm_*") + if err != nil { + return err + } + defer os.RemoveAll(tmp) + + if err := extractArchive(opts.ArchivePath(), tmp); err != nil { + return fmt.Errorf("failed extracting %s: %w", opts.ArchivePath(), err) + } + pkg, raw, err := readPackageManifest(tmp) + if err != nil { + return err + } + if pkg.Name == "" || pkg.Version == "" { + return fmt.Errorf("package.json has no name or version") + } + if opts.Name != "" && opts.Name != pkg.Name { + return fmt.Errorf("tarball holds package %q, expected %q", pkg.Name, opts.Name) + } + + staged, err := stagedSlices(opts.Out) + if err != nil { + return err + } + if err := checkDependencyClosure(pkg, staged); err != nil { + return err + } + + pkgDir := filepath.Join(opts.Out, "npm", "registry.npmjs.org", filepath.FromSlash(pkg.Name)) + versionDir := filepath.Join(pkgDir, pkg.Version) + if err := os.MkdirAll(filepath.Dir(versionDir), 0755); err != nil { + return err + } + if err := copyPath(tmp, versionDir); err != nil { + return fmt.Errorf("failed staging the package: %w", err) + } + + integrity, err := tarballIntegrity(opts.ArchivePath()) + if err != nil { + return err + } + if err := writePackument(pkgDir, pkg, raw, integrity); err != nil { + return err + } + + // Bundle the dependencies' packages into this slice, so that a target only needs to list + // this module, like ts_module: Please stages the direct dependencies of a target, not + // the transitive ones. Each staged slice already contains its own dependencies. + dirs := make([]string, 0, len(staged)) + for _, s := range staged { + dirs = append(dirs, s.dir) + } + if err := npmcache.Merge(dirs, opts.Out); err != nil { + return fmt.Errorf("failed bundling dependencies: %w", err) + } + + return npmcache.Write(opts.Out, npmcache.Slice{ + Name: pkg.Name, + Version: pkg.Version, + Specifier: npmcache.Specifier(pkg.Name, pkg.Version), + Dependencies: pkg.Dependencies, + }) +} + +// readPackageManifest reads package.json, returning both the typed fields and the raw +// object (to carry dependency fields into registry.json unchanged). +func readPackageManifest(dir string) (PackageJSON, map[string]json.RawMessage, error) { + var pkg PackageJSON + data, err := os.ReadFile(filepath.Join(dir, "package.json")) + if err != nil { + return pkg, nil, fmt.Errorf("package.json not found in the tarball: %w", err) + } + if err := json.Unmarshal(data, &pkg); err != nil { + return pkg, nil, fmt.Errorf("invalid package.json: %w", err) + } + var raw map[string]json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + return pkg, nil, err + } + return pkg, raw, nil +} + +func tarballIntegrity(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + sum := sha512.Sum512(data) + return "sha512-" + base64.StdEncoding.EncodeToString(sum[:]), nil +} + +// writePackument writes the registry.json Deno reads from its cache. The +// _deno.packumentFormat key tells Deno that the packument is complete; without it Deno +// tries to download the real one. +func writePackument(pkgDir string, pkg PackageJSON, raw map[string]json.RawMessage, integrity string) error { + version := map[string]any{ + "name": pkg.Name, + "version": pkg.Version, + "dist": map[string]string{ + "tarball": fmt.Sprintf("https://registry.npmjs.org/%s/-/%s-%s.tgz", pkg.Name, tarballBase(pkg.Name), pkg.Version), + "integrity": integrity, + }, + } + for _, key := range []string{"dependencies", "peerDependencies", "peerDependenciesMeta", "optionalDependencies", "bin"} { + if v, ok := raw[key]; ok { + version[key] = v + } + } + packument := map[string]any{ + "name": pkg.Name, + "dist-tags": map[string]string{"latest": pkg.Version}, + "versions": map[string]any{pkg.Version: version}, + "_deno.packumentFormat": "full", + } + data, err := json.Marshal(packument) + if err != nil { + return err + } + return os.WriteFile(filepath.Join(pkgDir, "registry.json"), data, 0644) +} + +func tarballBase(name string) string { + if i := strings.LastIndex(name, "/"); i >= 0 { + return name[i+1:] + } + return name +} + +// stagedSlice is a slice found in the build directory. +type stagedSlice struct { + dir string + npmcache.Slice +} + +// stagedSlices returns the slices staged under the working directory, which are the +// ts_npm_module targets in deps; outDir itself is skipped. +func stagedSlices(outDir string) ([]stagedSlice, error) { + absOut, _ := filepath.Abs(outDir) + var staged []stagedSlice + for _, dir := range npmcache.Discover(".") { + if abs, _ := filepath.Abs(dir); abs == absOut { + continue + } + s, err := npmcache.Read(dir) + if err != nil { + return nil, err + } + staged = append(staged, stagedSlice{dir: dir, Slice: *s}) + } + return staged, nil +} + +// checkDependencyClosure verifies that each required dependency of pkg is provided by a +// staged slice. Optional dependencies and peer dependencies, which the consumer provides, +// are not required. +func checkDependencyClosure(pkg PackageJSON, staged []stagedSlice) error { + if len(pkg.Dependencies) == 0 { + return nil + } + versions := map[string][]string{} + for _, s := range staged { + versions[s.Name] = append(versions[s.Name], s.Version) + } + + names := make([]string, 0, len(pkg.Dependencies)) + for name := range pkg.Dependencies { + names = append(names, name) + } + sort.Strings(names) + + var missing []string + for _, name := range names { + rng := pkg.Dependencies[name] + if !anySatisfies(versions[name], rng) { + missing = append(missing, fmt.Sprintf("%s@%s", name, rng)) + } + } + if len(missing) > 0 { + return fmt.Errorf("%s@%s depends on %s, which no ts_npm_module in deps provides; "+ + "add a ts_npm_module for each (with its tarball hash) to deps", + pkg.Name, pkg.Version, strings.Join(missing, ", ")) + } + return nil +} + +func anySatisfies(versions []string, constraint string) bool { + for _, v := range versions { + if satisfies(v, constraint) { + return true + } + } + return false +} + +// satisfies reports whether version meets an npm range. It understands exact versions, +// "*", "x"-less ranges with ^, ~ and >= prefixes; any other range form (||, spaces, <, +// hyphen ranges, x-ranges) is accepted when a version is staged at all, since the +// closure check only has to catch a dependency that was forgotten, not pick versions. +func satisfies(version, constraint string) bool { + c := strings.TrimSpace(constraint) + if c == "" || c == "*" || c == "latest" || version == c { + return true + } + prefix := "" + switch { + case strings.HasPrefix(c, "^"), strings.HasPrefix(c, "~"): + prefix, c = c[:1], c[1:] + case strings.HasPrefix(c, ">="): + prefix, c = ">=", strings.TrimSpace(c[2:]) + } + if strings.ContainsAny(c, " |<>=xX-") || c == "" { + return true + } + return matchesConstraint(parseSemver(version), parseSemver(c), prefix) +} diff --git a/tools/please_ts/unpack/npmcache_test.go b/tools/please_ts/unpack/npmcache_test.go new file mode 100644 index 0000000..bfc9a74 --- /dev/null +++ b/tools/please_ts/unpack/npmcache_test.go @@ -0,0 +1,321 @@ +package unpack + +import ( + "archive/tar" + "compress/gzip" + "crypto/sha512" + "encoding/base64" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "tools/please_ts/npmcache" +) + +// makeNpmTarball writes a gzipped tarball laid out like an npm package ("package/" prefix). +func makeNpmTarball(t *testing.T, dir string, files map[string]string) string { + t.Helper() + path := filepath.Join(dir, "pkg.tgz") + f, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + gz := gzip.NewWriter(f) + tw := tar.NewWriter(gz) + for name, content := range files { + if err := tw.WriteHeader(&tar.Header{Name: "package/" + name, Mode: 0644, Size: int64(len(content)), Typeflag: tar.TypeReg}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte(content)); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + return path +} + +func chdir(t *testing.T, dir string) { + t.Helper() + old, _ := os.Getwd() + if err := os.Chdir(dir); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chdir(old) }) +} + +func stageSlice(t *testing.T, root, name, version string) { + t.Helper() + dir := filepath.Join(root, "staged", name) + if err := os.MkdirAll(dir, 0755); err != nil { + t.Fatal(err) + } + if err := npmcache.Write(dir, npmcache.Slice{Name: name, Version: version, Specifier: npmcache.Specifier(name, version)}); err != nil { + t.Fatal(err) + } +} + +func TestBuildNpmCacheLaysOutTheDenoCache(t *testing.T) { + work := t.TempDir() + chdir(t, work) + tgz := makeNpmTarball(t, work, map[string]string{ + "package.json": `{"name":"ms","version":"2.1.3"}`, + "index.js": "module.exports = 1;", + "lib/util.js": "exports.x = 1;", + }) + out := filepath.Join(work, "out") + + if err := Run(Options{Archive: tgz, Out: out, Name: "ms", NpmCache: true}); err != nil { + t.Fatal(err) + } + + for _, p := range []string{"index.js", "package.json", "lib/util.js"} { + if _, err := os.Stat(filepath.Join(out, "npm/registry.npmjs.org/ms/2.1.3", p)); err != nil { + t.Errorf("package file %s missing: %v", p, err) + } + } + slice, err := npmcache.Read(out) + if err != nil || slice.Specifier != "npm:ms@2.1.3" { + t.Fatalf("slice = %+v, %v", slice, err) + } + + data, err := os.ReadFile(filepath.Join(out, "npm/registry.npmjs.org/ms/registry.json")) + if err != nil { + t.Fatal(err) + } + var packument struct { + Name string `json:"name"` + Format string `json:"_deno.packumentFormat"` + DistTags map[string]string + Versions map[string]struct { + Dist struct { + Integrity string `json:"integrity"` + Tarball string `json:"tarball"` + } `json:"dist"` + } `json:"versions"` + } + if err := json.Unmarshal(data, &packument); err != nil { + t.Fatal(err) + } + raw, _ := os.ReadFile(tgz) + sum := sha512.Sum512(raw) + wantIntegrity := "sha512-" + base64.StdEncoding.EncodeToString(sum[:]) + v := packument.Versions["2.1.3"] + if packument.Name != "ms" || packument.Format != "full" || v.Dist.Integrity != wantIntegrity || + v.Dist.Tarball != "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz" { + t.Errorf("packument = %s", data) + } +} + +func TestBuildNpmCacheScopedPackageAndDependencyFields(t *testing.T) { + work := t.TempDir() + chdir(t, work) + stageSlice(t, work, "lib", "1.0.0") + tgz := makeNpmTarball(t, work, map[string]string{ + "package.json": `{"name":"@scope/pkg","version":"1.0.0","dependencies":{"lib":"1.0.0"}, + "peerDependencies":{"react":"^18"},"peerDependenciesMeta":{"react":{"optional":true}}}`, + "index.js": "", + }) + out := filepath.Join(work, "out") + if err := Run(Options{Archive: tgz, Out: out, NpmCache: true}); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(out, "npm/registry.npmjs.org/@scope/pkg/registry.json")) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{`"dependencies":{"lib":"1.0.0"}`, `"peerDependencies"`, `"peerDependenciesMeta"`, "pkg-1.0.0.tgz"} { + if !strings.Contains(string(data), want) { + t.Errorf("registry.json lacks %s:\n%s", want, data) + } + } +} + +func TestBuildNpmCacheDependencyClosure(t *testing.T) { + depPkg := map[string]string{ + "package.json": `{"name":"debug","version":"4.3.7","dependencies":{"ms":"^2.1.3"}}`, + "index.js": "", + } + for name, tc := range map[string]struct { + staged map[string]string // name -> version + wantErr string + }{ + "missing": {nil, `ms@^2.1.3`}, + "unsatisfied": {map[string]string{"ms": "1.0.0"}, `ms@^2.1.3`}, + "different major": {map[string]string{"ms": "3.0.0"}, `ms@^2.1.3`}, + "satisfied": {map[string]string{"ms": "2.1.3"}, ""}, + "newer minor": {map[string]string{"ms": "2.4.0"}, ""}, + } { + t.Run(name, func(t *testing.T) { + work := t.TempDir() + chdir(t, work) + for n, v := range tc.staged { + stageSlice(t, work, n, v) + } + tgz := makeNpmTarball(t, work, depPkg) + err := Run(Options{Archive: tgz, Out: filepath.Join(work, "out"), Name: "debug", NpmCache: true}) + if tc.wantErr == "" { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tc.wantErr) || !strings.Contains(err.Error(), "ts_npm_module") { + t.Fatalf("error = %v, want one naming %s and ts_npm_module", err, tc.wantErr) + } + }) + } +} + +func TestBuildNpmCacheIgnoresItsOwnOutputInTheClosureCheck(t *testing.T) { + work := t.TempDir() + chdir(t, work) + out := filepath.Join(work, "out") + if err := os.MkdirAll(out, 0755); err != nil { + t.Fatal(err) + } + // A stale slice for the dependency inside out must not count as a staged dependency. + if err := npmcache.Write(out, npmcache.Slice{Name: "ms", Version: "2.1.3", Specifier: "npm:ms@2.1.3"}); err != nil { + t.Fatal(err) + } + tgz := makeNpmTarball(t, work, map[string]string{ + "package.json": `{"name":"debug","version":"4.3.7","dependencies":{"ms":"^2.1.3"}}`, + }) + if err := Run(Options{Archive: tgz, Out: out, Name: "debug", NpmCache: true}); err == nil { + t.Error("the output directory must not satisfy the target's own dependencies") + } +} + +func TestBuildNpmCacheRejectsBadTarballs(t *testing.T) { + work := t.TempDir() + chdir(t, work) + cases := map[string]struct { + files map[string]string + name string + }{ + "no package.json": {map[string]string{"index.js": ""}, ""}, + "no version": {map[string]string{"package.json": `{"name":"x"}`}, ""}, + "wrong name": {map[string]string{"package.json": `{"name":"x","version":"1.0.0"}`}, "y"}, + "invalid json": {map[string]string{"package.json": `{`}, ""}, + } + for n, c := range cases { + tgz := makeNpmTarball(t, work, c.files) + if err := Run(Options{Archive: tgz, Out: filepath.Join(work, "out-"+strings.ReplaceAll(n, " ", "-")), Name: c.name, NpmCache: true}); err == nil { + t.Errorf("%s: expected an error", n) + } + } +} + +func TestSatisfies(t *testing.T) { + for _, tc := range []struct { + version, rng string + want bool + }{ + {"2.1.3", "2.1.3", true}, + {"2.1.3", "^2.1.3", true}, + {"2.4.0", "^2.1.3", true}, + {"2.0.0", "^2.1.3", false}, + {"3.0.0", "^2.1.3", false}, + {"1.2.9", "~1.2.3", true}, + {"1.3.0", "~1.2.3", false}, + {"5.0.0", ">=2.0.0", true}, + {"1.0.0", ">=2.0.0", false}, + {"9.9.9", "*", true}, + {"9.9.9", "", true}, + {"9.9.9", "latest", true}, + // range forms that are not interpreted are accepted: only a forgotten dependency is an error + {"9.9.9", "^1 || ^2", true}, + {"9.9.9", ">=1 <3", true}, + {"9.9.9", "1.x", true}, + } { + if got := satisfies(tc.version, tc.rng); got != tc.want { + t.Errorf("satisfies(%q, %q) = %v, want %v", tc.version, tc.rng, got, tc.want) + } + } +} + +// stageSliceWithPackage stages a dependency slice that holds an extracted package, as the +// output of a ts_npm_module does. +func stageSliceWithPackage(t *testing.T, root, name, version string) { + t.Helper() + stageSlice(t, root, name, version) + pkgDir := filepath.Join(root, "staged", name, "npm", "registry.npmjs.org", name) + if err := os.MkdirAll(filepath.Join(pkgDir, version), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(pkgDir, version, "index.js"), []byte("module.exports = 1;"), 0644); err != nil { + t.Fatal(err) + } + packument := `{"name":"` + name + `","dist-tags":{"latest":"` + version + `"},"versions":{"` + version + `":{"name":"` + name + `","version":"` + version + `"}},"_deno.packumentFormat":"full"}` + if err := os.WriteFile(filepath.Join(pkgDir, "registry.json"), []byte(packument), 0644); err != nil { + t.Fatal(err) + } +} + +func TestBuildNpmCacheBundlesTheDependenciesIntoTheSlice(t *testing.T) { + work := t.TempDir() + chdir(t, work) + stageSliceWithPackage(t, work, "ms", "2.1.3") + tgz := makeNpmTarball(t, work, map[string]string{ + "package.json": `{"name":"debug","version":"4.3.7","dependencies":{"ms":"^2.1.3"}}`, + "index.js": "", + }) + out := filepath.Join(work, "out") + if err := Run(Options{Archive: tgz, Out: out, Name: "debug", NpmCache: true}); err != nil { + t.Fatal(err) + } + + for _, p := range []string{ + "npm/registry.npmjs.org/debug/4.3.7/index.js", + "npm/registry.npmjs.org/ms/2.1.3/index.js", // the dependency, so a consumer only needs debug + "npm/registry.npmjs.org/ms/registry.json", + } { + if _, err := os.Stat(filepath.Join(out, p)); err != nil { + t.Errorf("slice lacks %s: %v", p, err) + } + } + // The slice is still named after its own package: only debug is importable by name. + slice, err := npmcache.Read(out) + if err != nil || slice.Name != "debug" || slice.Version != "4.3.7" { + t.Errorf("slice = %+v, %v", slice, err) + } +} + +func TestBuildNpmCacheSlicesWithDifferentVersionsOfADependencyStayResolvable(t *testing.T) { + work := t.TempDir() + chdir(t, work) + stageSliceWithPackage(t, work, "ms", "2.1.3") + // A second staged dependency that itself carries another version of ms. + other := filepath.Join(work, "staged", "other") + oldPkg := filepath.Join(other, "npm", "registry.npmjs.org", "ms") + if err := os.MkdirAll(filepath.Join(oldPkg, "2.0.0"), 0755); err != nil { + t.Fatal(err) + } + _ = os.WriteFile(filepath.Join(oldPkg, "2.0.0", "index.js"), []byte("x"), 0644) + _ = os.WriteFile(filepath.Join(oldPkg, "registry.json"), []byte(`{"name":"ms","versions":{"2.0.0":{"name":"ms","version":"2.0.0"}}}`), 0644) + if err := npmcache.Write(other, npmcache.Slice{Name: "other", Version: "1.0.0", Specifier: "npm:other@1.0.0"}); err != nil { + t.Fatal(err) + } + + tgz := makeNpmTarball(t, work, map[string]string{ + "package.json": `{"name":"top","version":"1.0.0","dependencies":{"ms":"^2.1.3","other":"1.0.0"}}`, + }) + out := filepath.Join(work, "out") + if err := Run(Options{Archive: tgz, Out: out, Name: "top", NpmCache: true}); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(out, "npm/registry.npmjs.org/ms/registry.json")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), `"2.0.0"`) || !strings.Contains(string(data), `"2.1.3"`) { + t.Errorf("both ms versions must be in the merged packument: %s", data) + } +} diff --git a/tools/please_ts/unpack/unpack.go b/tools/please_ts/unpack/unpack.go index c6aa76b..9782b67 100644 --- a/tools/please_ts/unpack/unpack.go +++ b/tools/please_ts/unpack/unpack.go @@ -28,6 +28,7 @@ type Options struct { Symlink string // optional symlink name for extracted binary ResolveTransitive bool // recursively resolve and unpack transitive dependencies Registry string // npm registry URL (default https://registry.npmjs.org) + NpmCache bool // build a Deno npm cache slice from the tarball, offline } // Validate checks whether the required options are provided. @@ -94,6 +95,9 @@ func Run(opts Options) error { if opts.Binary != "" { return unpackToolchain(opts.ArchivePath(), opts.Out, opts.Binary, opts.Symlink) } + if opts.NpmCache { + return buildNpmCache(opts) + } return unpackModule(opts) } From 970001384b90da0deb6690d764a032a5405297d8 Mon Sep 17 00:00:00 2001 From: Walter Pinto Date: Sun, 4 Oct 2026 08:43:15 +0100 Subject: [PATCH 2/3] fix(ts): do not force the sandbox on ts_npm_module The build step needs no network access, and forcing the sandbox fails on hosts without user namespaces (the CI runner: fopen /proc/self/setgroups: Permission denied). Follow Please's [sandbox] setting like the other rules and say so in the docs. Refs #61 Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 14 +++++++------- build_defs/ts/ts.build_defs | 6 +++--- docs/ts/usage.md | 9 ++++++--- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffd8890..d519fbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,13 +12,13 @@ and this project adheres to ### Added - Experimental `ts_npm_module` rule: provides an npm package to Deno's own - `npm:` resolution, offline. The sha256-pinned tarball is extracted by a - sandboxed build step into a slice of a Deno npm cache that targets merge into - their `DENO_DIR`; no `node_modules`, no lockfile. CommonJS packages, subpath - imports and packages with only an `exports` map work as Deno resolves them. - Dependencies are separate `ts_npm_module` targets and the build fails if one - is missing; targets run with `--cached-only` so a missing module fails instead - of being downloaded. + `npm:` resolution, offline. The sha256-pinned tarball is extracted by a build + step that needs no network access into a slice of a Deno npm cache that + targets merge into their `DENO_DIR`; no `node_modules`, no lockfile. CommonJS + packages, subpath imports and packages with only an `exports` map work as Deno + resolves them. Dependencies are separate `ts_npm_module` targets and the build + fails if one is missing; targets run with `--cached-only` so a missing module + fails instead of being downloaded. - `please_ts unpack --npm-cache` and the `npmcache` package behind it. - Fixtures for `debug` (CommonJS, transitive dependency), `highlight.js` (CommonJS, subpaths) and `@codemirror/legacy-modes` (exports map only). diff --git a/build_defs/ts/ts.build_defs b/build_defs/ts/ts.build_defs index 738af02..ebcc1b7 100644 --- a/build_defs/ts/ts.build_defs +++ b/build_defs/ts/ts.build_defs @@ -230,8 +230,9 @@ def ts_npm_module( labels: list = None): """Provides one npm package to Deno's own npm resolution, hermetically and offline. - The tarball is pinned by `hashes` (sha256, checked by Please). The build step, which - has no network access, extracts it into a slice of a Deno npm cache; targets that + The tarball is pinned by `hashes` (sha256, checked by Please). The build step needs no + network access (it only reads that tarball and the slices of its dependencies; Please's + sandbox, when enabled in `[sandbox]`, enforces this) and extracts it into a slice of a Deno npm cache; targets that depend on it get the slices merged into their DENO_DIR and import the package through an `npm:` specifier. Deno then resolves `exports`, CommonJS and subpaths itself: there is no node_modules directory and no lockfile, and a package that Deno cannot resolve @@ -272,7 +273,6 @@ def ts_npm_module( exported_deps = deps or [], outs = [name], cmd = f'$TOOLS_TOOL unpack --npm-cache --tarball "$SRCS" --out "$OUT" --name "{pkg}"', - sandbox = True, needs_transitive_deps = True, tools = tools, visibility = visibility or ["PUBLIC"], diff --git a/docs/ts/usage.md b/docs/ts/usage.md index faf1c1f..d559293 100644 --- a/docs/ts/usage.md +++ b/docs/ts/usage.md @@ -259,9 +259,12 @@ ts_test( ``` - **Hermetic**: the tarball is pinned by `hashes` (sha256, checked by Please). - The build step runs in the sandbox, without network access, and extracts it - into a slice of a Deno npm cache. There is no `node_modules` directory and no - lockfile; the BUILD files are the lock. + The build step needs no network access: it only reads that tarball and the + modules of its dependencies, and extracts the package into a slice of a Deno + npm cache. Enable Please's `[sandbox]` build setting to have the sandbox + enforce it; the rule does not force it, because it needs user namespaces that + some hosts, including CI runners, do not allow. There is no `node_modules` + directory and no lockfile; the BUILD files are the lock. - **Dependencies are explicit**: nothing is resolved at build time. If a package depends on something that is not in `deps`, the build fails and names it. Each module bundles the packages of its dependencies, so a target lists only the From 6fa91f1f3507bdada76fe50313e3faede7f14477 Mon Sep 17 00:00:00 2001 From: Walter Pinto Date: Sun, 4 Oct 2026 09:07:58 +0100 Subject: [PATCH 3/3] feat(ts): resolve npm dependencies automatically, strictly and reproducibly Add a semver package (ranges, prereleases, x- and hyphen ranges, ||) and an npm registry client (strict resolution, integrity-verified downloads), and rebuild dependency resolution on them for both ts_module and ts_npm_module. Resolution no longer falls back to latest, verifies every download against the registry's integrity data, reads dependencies from the verified tarball, and fails on conflicts and unresolvable dependencies instead of warning. ts_npm_module resolves dependencies by default (resolve_transitive, as ts_module) and keeps several versions side by side; resolve_transitive = False keeps the fully pinned mode. Resolution is reproducible without a lockfile or a date: only versions published by the end of the day the root version was published are considered. Refs #61 Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 41 +- build_defs/ts/ts.build_defs | 60 ++- docs/ts/usage.md | 72 +-- test/ts/npm_cache/BUILD | 64 +++ test/ts/npm_cache/humanize_auto_test.ts | 8 + test/ts/npm_cache/modes_auto_test.ts | 8 + test/ts/npm_cache/modules/BUILD | 12 + tools/please_ts/commands.go | 2 +- tools/please_ts/registry/BUILD | 17 + tools/please_ts/registry/registry.go | 282 ++++++++++++ tools/please_ts/registry/registry_test.go | 242 ++++++++++ tools/please_ts/semver/BUILD | 15 + tools/please_ts/semver/semver.go | 415 +++++++++++++++++ tools/please_ts/semver/semver_test.go | 119 +++++ tools/please_ts/unpack/BUILD | 7 + tools/please_ts/unpack/npmcache.go | 209 +++++---- tools/please_ts/unpack/npmcache_test.go | 62 +-- tools/please_ts/unpack/resolve.go | 219 +++++++++ tools/please_ts/unpack/resolve_test.go | 516 ++++++++++++++++++++++ tools/please_ts/unpack/unpack.go | 257 +---------- tools/please_ts/unpack/unpack_test.go | 87 +--- 21 files changed, 2219 insertions(+), 495 deletions(-) create mode 100644 test/ts/npm_cache/humanize_auto_test.ts create mode 100644 test/ts/npm_cache/modes_auto_test.ts create mode 100644 tools/please_ts/registry/BUILD create mode 100644 tools/please_ts/registry/registry.go create mode 100644 tools/please_ts/registry/registry_test.go create mode 100644 tools/please_ts/semver/BUILD create mode 100644 tools/please_ts/semver/semver.go create mode 100644 tools/please_ts/semver/semver_test.go create mode 100644 tools/please_ts/unpack/resolve.go create mode 100644 tools/please_ts/unpack/resolve_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index d519fbd..40b1d92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,16 +12,39 @@ and this project adheres to ### Added - Experimental `ts_npm_module` rule: provides an npm package to Deno's own - `npm:` resolution, offline. The sha256-pinned tarball is extracted by a build - step that needs no network access into a slice of a Deno npm cache that - targets merge into their `DENO_DIR`; no `node_modules`, no lockfile. CommonJS - packages, subpath imports and packages with only an `exports` map work as Deno - resolves them. Dependencies are separate `ts_npm_module` targets and the build - fails if one is missing; targets run with `--cached-only` so a missing module - fails instead of being downloaded. -- `please_ts unpack --npm-cache` and the `npmcache` package behind it. + `npm:` resolution. The sha256-pinned tarball is extracted by the build into a + slice of a Deno npm cache that targets merge into their `DENO_DIR`; no + `node_modules`, no lockfile. CommonJS packages, subpath imports and packages + with only an `exports` map work as Deno resolves them. Dependencies resolve + automatically (`resolve_transitive`, on by default, strictly and verified + against the registry's integrity data, with several versions kept side by side + when dependents need different ones), or can be declared as separate pinned + `ts_npm_module` targets with `resolve_transitive = False`. Type checks and + tests run with `--cached-only`, so a missing module fails instead of being + downloaded. +- Dependency resolution is reproducible without a lockfile: only versions + published by the end of the day the root version was published are considered, + for both `ts_npm_module` and `ts_module`, so the version pinned by its hash + fixes the result and nothing needs configuring. +- `please_ts unpack --npm-cache`, and the `npmcache`, `registry` and `semver` + packages behind it. - Fixtures for `debug` (CommonJS, transitive dependency), `highlight.js` - (CommonJS, subpaths) and `@codemirror/legacy-modes` (exports map only). + (CommonJS, subpaths) and `@codemirror/legacy-modes` (exports map only), + automatic and fully pinned. + +### Fixed + +- `ts_module`'s automatic dependency resolution (`resolve_transitive`) is now + strict and verified. It used to fall back to the `latest` version when no + version satisfied a range, verify no download against the registry's integrity + data, keep the first version of a package silently when a second dependent + needed an incompatible one, and only print a warning, leaving an incomplete + tree, when resolution failed. A range that no version satisfies, an integrity + mismatch, a tarball that holds another package and conflicting versions are + now build errors; unresolvable peer dependencies stay warnings. Version ranges + are interpreted completely (`||`, hyphen and x-ranges, prereleases) instead of + only `^`, `~` and `>=`. **This can make builds fail that used to pass with an + incomplete or inconsistent tree.** ## [0.6.0] - 2026-10-03 diff --git a/build_defs/ts/ts.build_defs b/build_defs/ts/ts.build_defs index ebcc1b7..04ffa82 100644 --- a/build_defs/ts/ts.build_defs +++ b/build_defs/ts/ts.build_defs @@ -182,7 +182,17 @@ def ts_module( registry: str = "", visibility: list = None, labels: list = None): - """Downloads an npm package tarball, extracts it, resolves transitive dependencies, and creates metadata for downstream targets.""" + """Downloads an npm package tarball, extracts it, resolves transitive dependencies, and creates metadata for downstream targets. + + Only the root tarball is pinned (`hashes`). Dependencies are resolved from the registry at + build time, strictly: a range that no version satisfies, a download that does not match + the registry's integrity data, or two dependents that need incompatible versions of one + package (this layout holds one) fails the build. + + Resolution is reproducible: only dependency versions published by the end of the day the + root version was published are considered, so the pinned root fixes the result and nothing + needs configuring. Bump the root version to move the dependencies forward. + """ pkg = package or name ver = version @@ -226,21 +236,32 @@ def ts_npm_module( hashes: list = None, url: str = "", deps: list = None, + resolve_transitive: bool = True, + registry: str = "", visibility: list = None, labels: list = None): - """Provides one npm package to Deno's own npm resolution, hermetically and offline. - - The tarball is pinned by `hashes` (sha256, checked by Please). The build step needs no - network access (it only reads that tarball and the slices of its dependencies; Please's - sandbox, when enabled in `[sandbox]`, enforces this) and extracts it into a slice of a Deno npm cache; targets that - depend on it get the slices merged into their DENO_DIR and import the package through - an `npm:` specifier. Deno then resolves `exports`, CommonJS and subpaths itself: there - is no node_modules directory and no lockfile, and a package that Deno cannot resolve - from the cache fails the build rather than reaching for the network. - - Unlike ts_module, dependencies are not resolved at build time: every dependency of the - package (and theirs) must be its own ts_npm_module, listed in `deps`. The build fails - with the missing names if one is absent. + """Provides one npm package to Deno's own npm resolution, hermetically and offline once built. + + The tarball is pinned by `hashes` (sha256, checked by Please). The build step extracts it + into a slice of a Deno npm cache; targets that depend on it get the slices merged into + their DENO_DIR and import the package through an `npm:` specifier. Deno then resolves + `exports`, CommonJS and subpaths itself: there is no node_modules directory and no + lockfile, and a package that Deno cannot resolve from the cache fails the build rather + than reaching for the network. + + Dependencies are handled in one of two ways: + + * `resolve_transitive = True` (the default, like ts_module): the build resolves the + dependencies from the registry, each verified against the registry's integrity data, + and keeps several versions of a package when dependents need different ones. This needs + network access during the build (so no sandbox) and trusts the registry for everything + but the root tarball. Resolution is reproducible without a lockfile: only versions + published by the end of the day the root version was published are considered, so the + pinned root fixes the result (bump its version to move the dependencies forward). Dependencies listed in `deps` are used as they are, not + resolved again. + * `resolve_transitive = False`: nothing is resolved. Every dependency (and theirs) must + be its own ts_npm_module, listed in `deps`, each pinned by its own sha256; the build + needs no network access and fails naming any dependency that is missing. Args: name: Name of the rule. @@ -248,7 +269,9 @@ def ts_npm_module( version: Exact package version. hashes: sha256 of the package tarball. url: Tarball URL (defaults to the npm registry). - deps: ts_npm_module targets for the package's dependencies. + deps: ts_npm_module targets that provide dependencies explicitly. + resolve_transitive: Resolve the dependencies from the registry at build time. + registry: npm registry URL used to resolve dependencies. visibility: Visibility of the rule. labels: Labels for the rule. """ @@ -266,13 +289,18 @@ def ts_npm_module( ) tools = {"TOOL": [_ts_tool()]} + resolve_flag = "--resolve-transitive" if resolve_transitive else "" + registry_flag = f'--registry "{registry}"' if registry else "" return build_rule( name = name, srcs = [f":{dl_target}"], deps = deps or [], exported_deps = deps or [], outs = [name], - cmd = f'$TOOLS_TOOL unpack --npm-cache --tarball "$SRCS" --out "$OUT" --name "{pkg}"', + cmd = f'$TOOLS_TOOL unpack --npm-cache --tarball "$SRCS" --out "$OUT" --name "{pkg}" {resolve_flag} {registry_flag}', + # Resolving needs the registry. Without it the step only reads local files and follows + # the [sandbox] setting like every other rule. + sandbox = False if resolve_transitive else None, needs_transitive_deps = True, tools = tools, visibility = visibility or ["PUBLIC"], diff --git a/docs/ts/usage.md b/docs/ts/usage.md index d559293..4c6a586 100644 --- a/docs/ts/usage.md +++ b/docs/ts/usage.md @@ -228,20 +228,14 @@ branch (`BRDA`) records that `coverage.xml` and `coverage.json` do not: cannot express CommonJS packages, subpath imports (`highlight.js/lib/core`) or packages that only have an `exports` map. `ts_npm_module` takes a different route: Deno resolves the package itself through an `npm:` specifier, and the -rule only has to make the package available **offline**: +rule only has to make the package available **offline**. List the package you +import, pinned by the hash of its tarball: ```starlark -ts_npm_module( - name = "ms", - hashes = ["f6616e15e530ed552f9daa2d3ce71963947c6bc7c98c9b64fd3e673fd02622c6"], - version = "2.1.3", -) - ts_npm_module( name = "debug", hashes = ["c803a8ca9b835b7a75f7150ee52f7f640675515bafbe8f5da78fcc0ae12914ac"], version = "4.3.7", - deps = [":ms"], # every dependency is its own ts_npm_module ) ts_library( @@ -258,30 +252,52 @@ ts_test( ) ``` -- **Hermetic**: the tarball is pinned by `hashes` (sha256, checked by Please). - The build step needs no network access: it only reads that tarball and the - modules of its dependencies, and extracts the package into a slice of a Deno - npm cache. Enable Please's `[sandbox]` build setting to have the sandbox - enforce it; the rule does not force it, because it needs user namespaces that - some hosts, including CI runners, do not allow. There is no `node_modules` - directory and no lockfile; the BUILD files are the lock. -- **Dependencies are explicit**: nothing is resolved at build time. If a package - depends on something that is not in `deps`, the build fails and names it. Each - module bundles the packages of its dependencies, so a target lists only the - modules it imports. +```typescript +import createDebug from "debug"; // CommonJS, and it needs the package "ms" +``` + +- **Dependencies resolve automatically** (`resolve_transitive`, on by default, + as for `ts_module`): `debug` needs `ms`, and nothing else has to be declared. + The build resolves dependencies from the registry, strictly: a range that no + version satisfies, a download that does not match the registry's integrity + data, or a tarball that holds another package fails the build. Dependents that + need different versions of one package each get theirs (Deno resolves per + dependent), which `ts_module`'s single `.deps` directory cannot do. Optional + and peer dependencies are not fetched. +- **Reproducible without a lockfile**: only dependency versions published by the + end of the day the root version was published are considered, so the version + you pin by hash fixes the result and nothing needs configuring. Bump the root + version to move the dependencies forward. If the registry does not know the + root version's publish time (a private registry, a tarball from another URL), + the build says so and does not pin. +- **Cost of automatic resolution**: the build needs network access (so no + sandbox for these targets) and trusts the registry for everything except the + root tarball. +- **Fully pinned alternative**: with `resolve_transitive = False` nothing is + resolved. Every dependency, and theirs, is its own `ts_npm_module` in `deps`, + each with its own tarball hash, and the build needs no network access. The + build fails and names any dependency that is missing. Dependencies you do list + in `deps` are always used as they are, never resolved again, so the two modes + can be mixed. +- **Offline afterwards**: everything that runs after the build (type checks, + tests) uses only the extracted cache, with no `node_modules` directory and no + lockfile. Targets run with `--cached-only`, so a module missing from a + target's `deps` fails at once with `npm package not found in cache`; Deno does + not download it. - **Resolution is Deno's**: `exports` maps, CommonJS, subpaths and a package's own types work as they do for `npm:` specifiers. Imports use the plain package - name, for example `import createDebug from "debug"`. -- **Offline is enforced**: targets whose import map has npm modules run Deno - with `--cached-only`. A module missing from a target's `deps` fails at once - with `npm package not found in cache`; Deno does not download it. + name. - **Not wired yet**: `ts_bundle` and `ts_binary`, the Vitest and browser - runners, and a helper that prints the declarations for a package and its - dependencies. The cache layout (`registry.json`, including its - `_deno.packumentFormat` key) is internal to Deno, so it is tied to the Deno - version the plugin pins. + runners, and a helper that prints pinned declarations. The cache layout + (`registry.json`, including its `_deno.packumentFormat` key) is internal to + Deno, so it is tied to the Deno version the plugin pins. + +`ts_module` resolves its dependencies by the same rules (strictly, verified, as +of the root's publish day), with the limit of one version per package: two +dependents that need incompatible versions fail the build and name each other. The fixtures in `test/ts/npm_cache` cover a CommonJS package with a transitive dependency (`debug`), a CommonJS package imported through subpaths (`highlight.js`), and an ES module package that only has an `exports` map and no -`main` (`@codemirror/legacy-modes`, with its eleven dependencies). +`main` (`@codemirror/legacy-modes`), each with automatic resolution and, for the +last two, with every dependency pinned explicitly. diff --git a/test/ts/npm_cache/BUILD b/test/ts/npm_cache/BUILD index bfb44e9..101bc0d 100644 --- a/test/ts/npm_cache/BUILD +++ b/test/ts/npm_cache/BUILD @@ -5,12 +5,14 @@ ts_npm_module( name = "ms", hashes = ["f6616e15e530ed552f9daa2d3ce71963947c6bc7c98c9b64fd3e673fd02622c6"], version = "2.1.3", + resolve_transitive = False, ) ts_npm_module( name = "debug", hashes = ["c803a8ca9b835b7a75f7150ee52f7f640675515bafbe8f5da78fcc0ae12914ac"], version = "4.3.7", + resolve_transitive = False, deps = [":ms"], ) @@ -67,3 +69,65 @@ ts_test( "//test/ts/npm_cache/modules:codemirror_legacy_modes", ], ) + +# Automatic resolution: only the root tarball is pinned. The dependencies are resolved from +# the registry when the module is built, as of the day the root version was published, and +# verified against the registry's integrity data. +ts_npm_module( + name = "debug_auto", + package = "debug", + hashes = ["c803a8ca9b835b7a75f7150ee52f7f640675515bafbe8f5da78fcc0ae12914ac"], + version = "4.3.7", +) + +ts_library( + name = "humanize_auto", + srcs = ["humanize.ts"], + module_name = "@test/humanize_auto", + deps = [":debug_auto"], +) + +ts_test( + name = "humanize_auto_test", + srcs = ["humanize_auto_test.ts"], + deps = [ + ":debug_auto", + ":humanize_auto", + ], +) + +# The same exports-only package as modes_test, with its eleven dependencies resolved +# automatically instead of declared. +ts_npm_module( + name = "legacy_modes_auto", + package = "@codemirror/legacy-modes", + hashes = ["61143cdb9c375dc1521895e4e536d75036baaf6e9df6b4431691f7ea5d273463"], + version = "6.5.1", +) + +ts_npm_module( + name = "codemirror_language_auto", + package = "@codemirror/language", + hashes = ["5e49acf55fde65ce9848068f0f06478be9ec71f818e91de6eca00824e9152226"], + version = "6.12.4", +) + +ts_library( + name = "modes_auto", + srcs = ["modes.ts"], + module_name = "@test/modes_auto", + deps = [ + ":codemirror_language_auto", + ":legacy_modes_auto", + ], +) + +ts_test( + name = "modes_auto_test", + srcs = ["modes_auto_test.ts"], + deps = [ + ":codemirror_language_auto", + ":legacy_modes_auto", + ":modes_auto", + ], +) diff --git a/test/ts/npm_cache/humanize_auto_test.ts b/test/ts/npm_cache/humanize_auto_test.ts new file mode 100644 index 0000000..a199a8f --- /dev/null +++ b/test/ts/npm_cache/humanize_auto_test.ts @@ -0,0 +1,8 @@ +import { humanize } from "@test/humanize_auto"; + +Deno.test("humanize (auto-resolved) formats durations through a CommonJS package and its dependency", () => { + const got = humanize(90061000); + if (got !== "1d") { + throw new Error(`expected 1d, got ${got}`); + } +}); diff --git a/test/ts/npm_cache/modes_auto_test.ts b/test/ts/npm_cache/modes_auto_test.ts new file mode 100644 index 0000000..b355d97 --- /dev/null +++ b/test/ts/npm_cache/modes_auto_test.ts @@ -0,0 +1,8 @@ +import { goLanguage } from "@test/modes_auto"; + +Deno.test("legacy-modes (auto-resolved): an ESM package that only has an exports map", () => { + const language = goLanguage(); + if (!language || typeof language.parser !== "object") { + throw new Error("expected a StreamLanguage with a parser"); + } +}); diff --git a/test/ts/npm_cache/modules/BUILD b/test/ts/npm_cache/modules/BUILD index 3da194b..1f59d54 100644 --- a/test/ts/npm_cache/modules/BUILD +++ b/test/ts/npm_cache/modules/BUILD @@ -10,6 +10,7 @@ ts_npm_module( package = "@codemirror/language", hashes = ["5e49acf55fde65ce9848068f0f06478be9ec71f818e91de6eca00824e9152226"], version = "6.12.4", + resolve_transitive = False, deps = [ ":codemirror_state", ":codemirror_view", @@ -25,6 +26,7 @@ ts_npm_module( package = "@codemirror/legacy-modes", hashes = ["61143cdb9c375dc1521895e4e536d75036baaf6e9df6b4431691f7ea5d273463"], version = "6.5.1", + resolve_transitive = False, deps = [ ":codemirror_language", ], @@ -35,6 +37,7 @@ ts_npm_module( package = "@codemirror/state", hashes = ["91d75acc955ceeaf86af9396c11cda7694fe6222aac8427fa033d89eff5ac839"], version = "6.7.6", + resolve_transitive = False, deps = [ ":marijn_find_cluster_break", ], @@ -45,6 +48,7 @@ ts_npm_module( package = "@codemirror/view", hashes = ["1246c9b6e1ad1d9d870b3e67672d3a8f8c9bc73af3b63c081c1549a94d5850db"], version = "6.43.13", + resolve_transitive = False, deps = [ ":codemirror_state", ":crelt", @@ -58,6 +62,7 @@ ts_npm_module( package = "@lezer/common", hashes = ["a8854639c04adabe5e72c26fa2876475116dc099ae5aa02250a5a938bb5ea4ae"], version = "1.5.3", + resolve_transitive = False, ) ts_npm_module( @@ -65,6 +70,7 @@ ts_npm_module( package = "@lezer/highlight", hashes = ["e349eec1ff4382439c1e6286ae937e0203a8349c5192e2e133863391775ac7ff"], version = "1.2.5", + resolve_transitive = False, deps = [ ":lezer_common", ], @@ -75,6 +81,7 @@ ts_npm_module( package = "@lezer/lr", hashes = ["c71e654cd0d153898e744a613bb82714fcd3c7d8e4df8cef7d193fd5ddfa6015"], version = "1.4.10", + resolve_transitive = False, deps = [ ":lezer_common", ], @@ -85,12 +92,14 @@ ts_npm_module( package = "@marijn/find-cluster-break", hashes = ["5d12b770b64d46632c84fcd18e1d4c09f195ed0f5dbc71f94a41f0af333eaeae"], version = "1.0.4", + resolve_transitive = False, ) ts_npm_module( name = "crelt", hashes = ["3542299c0278fdc26aceb2b13d31bffbcb89d7367706bbc457130e3fd103c03e"], version = "1.0.7", + resolve_transitive = False, ) ts_npm_module( @@ -98,6 +107,7 @@ ts_npm_module( package = "highlight.js", hashes = ["accbfaaab745088609b4eea2bdca2ad62f1f1dd27304e0f8df65cfe0fe042143"], version = "11.12.0", + resolve_transitive = False, ) ts_npm_module( @@ -105,6 +115,7 @@ ts_npm_module( package = "style-mod", hashes = ["769073434ab698eb035cbaa9df5d028231c25d95bc7b8676a46d603fb2e89e39"], version = "4.1.4", + resolve_transitive = False, ) ts_npm_module( @@ -112,4 +123,5 @@ ts_npm_module( package = "w3c-keyname", hashes = ["05d21484026d1f2dd842c843181e391e66d60f1d5dff292d562c3f40302dab36"], version = "2.2.8", + resolve_transitive = False, ) diff --git a/tools/please_ts/commands.go b/tools/please_ts/commands.go index fc7d59e..97845bd 100644 --- a/tools/please_ts/commands.go +++ b/tools/please_ts/commands.go @@ -194,7 +194,7 @@ func handleUnpack(args []string) error { symlink := cmd.String("symlink", "", "Optional symlink name for extracted binary") resolveTransitive := cmd.Bool("resolve-transitive", false, "Recursively resolve and download transitive dependencies") registry := cmd.String("registry", "", "NPM registry base URL") - npmCache := cmd.Bool("npm-cache", false, "Build a Deno npm cache slice from the tarball (offline, with a dependency closure check)") + npmCache := cmd.Bool("npm-cache", false, "Build a Deno npm cache slice from the tarball") if err := cmd.Parse(args); err != nil { return err diff --git a/tools/please_ts/registry/BUILD b/tools/please_ts/registry/BUILD new file mode 100644 index 0000000..6f10a65 --- /dev/null +++ b/tools/please_ts/registry/BUILD @@ -0,0 +1,17 @@ +subinclude("///go//build_defs:go") + +go_library( + name = "registry", + srcs = ["registry.go"], + visibility = ["//tools/please_ts/..."], + deps = ["//tools/please_ts/semver"], +) + +go_test( + name = "registry_test", + srcs = [ + "registry.go", + "registry_test.go", + ], + deps = ["//tools/please_ts/semver"], +) diff --git a/tools/please_ts/registry/registry.go b/tools/please_ts/registry/registry.go new file mode 100644 index 0000000..8d0d598 --- /dev/null +++ b/tools/please_ts/registry/registry.go @@ -0,0 +1,282 @@ +// Package registry is a small npm registry client: it reads packuments, resolves a +// dependency specifier to exactly one version, and downloads tarballs verified against +// the integrity data the registry publishes. +package registry + +import ( + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + "time" + + "tools/please_ts/semver" +) + +// DefaultURL is the public npm registry. +const DefaultURL = "https://registry.npmjs.org" + +// maxTarball bounds a download, so a misbehaving registry cannot exhaust memory. +const maxTarball = 512 << 20 + +// Dist is where a version's tarball is and how to verify it. +type Dist struct { + Tarball string `json:"tarball"` + Integrity string `json:"integrity"` // "sha512-", possibly several, space separated + Shasum string `json:"shasum"` // hex sha1, the older field +} + +// VersionInfo is the part of a version's manifest that resolution needs. Dependencies are +// deliberately not read from here: they come from the package.json inside the verified +// tarball, so that a registry cannot claim different dependencies than the package has. +type VersionInfo struct { + Version string `json:"version"` + Dist Dist `json:"dist"` + Deprecated json.RawMessage `json:"deprecated"` +} + +func (v VersionInfo) deprecated() bool { + d := strings.TrimSpace(string(v.Deprecated)) + return d != "" && d != "null" && d != "false" && d != `""` +} + +// Packument lists the versions of a package. +type Packument struct { + Name string `json:"name"` + DistTags map[string]string `json:"dist-tags"` + Versions map[string]VersionInfo `json:"versions"` + Time map[string]string `json:"time"` +} + +// Client talks to an npm registry. +type Client struct { + BaseURL string + HTTP *http.Client +} + +// New returns a client for the registry at baseURL (the public registry if empty). +func New(baseURL string) *Client { + if baseURL == "" { + baseURL = DefaultURL + } + return &Client{BaseURL: strings.TrimSuffix(baseURL, "/"), HTTP: &http.Client{Timeout: 120 * time.Second}} +} + +// Packument fetches the packument of a package. The full document has publish times, which +// an as-of resolution needs; the abbreviated one is much smaller. +func (c *Client) Packument(name string, full bool) (*Packument, error) { + u := c.BaseURL + "/" + strings.Replace(url.PathEscape(name), "%40", "@", 1) + req, err := http.NewRequest("GET", u, nil) + if err != nil { + return nil, err + } + if full { + req.Header.Set("Accept", "application/json") + } else { + req.Header.Set("Accept", "application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8, */*") + } + resp, err := c.HTTP.Do(req) + if err != nil { + return nil, fmt.Errorf("fetching %s: %w", u, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("registry returned status %d for %s", resp.StatusCode, u) + } + var p Packument + if err := json.NewDecoder(resp.Body).Decode(&p); err != nil { + return nil, fmt.Errorf("decoding the packument of %s: %w", name, err) + } + return &p, nil +} + +// Resolve picks the version of a package that a dependency specifier means: a version +// range or a dist-tag. Nothing is guessed: if no version satisfies the specifier it is an +// error. When asOf is set, versions published after it are ignored, which makes the result +// reproducible (this needs the full packument, with publish times). +// +// Among the versions that satisfy a range the choice follows npm: the "latest" dist-tag +// if it qualifies, otherwise the highest version, preferring releases over prereleases +// and versions that are not deprecated. +func Resolve(p *Packument, spec string, asOf time.Time) (VersionInfo, error) { + spec = strings.TrimSpace(spec) + if err := checkSpecifier(spec); err != nil { + return VersionInfo{}, fmt.Errorf("%s: %w", p.Name, err) + } + + published := func(v string) (bool, error) { + if asOf.IsZero() { + return true, nil + } + raw, ok := p.Time[v] + if !ok { + return false, fmt.Errorf("%s@%s has no publish time in the packument; an as-of date needs the full packument", p.Name, v) + } + t, err := time.Parse(time.RFC3339, raw) + if err != nil { + return false, fmt.Errorf("%s@%s: bad publish time %q", p.Name, v, raw) + } + return !t.After(asOf), nil + } + + rng, rangeErr := semver.ParseRange(spec) + if rangeErr != nil { + // Not a range: it may be a dist-tag such as "latest" or "next". + version, ok := p.DistTags[spec] + if !ok { + return VersionInfo{}, fmt.Errorf("%s: %q is neither a version range nor a dist-tag (%v)", p.Name, spec, rangeErr) + } + info, ok := p.Versions[version] + if !ok { + return VersionInfo{}, fmt.Errorf("%s: dist-tag %q points at %s, which is not in the packument", p.Name, spec, version) + } + if ok, err := published(version); err != nil { + return VersionInfo{}, err + } else if !ok { + return VersionInfo{}, fmt.Errorf("%s: dist-tag %q is %s, published after %s", p.Name, spec, version, asOf.Format("2006-01-02")) + } + return info, nil + } + + type candidate struct { + v semver.Version + info VersionInfo + } + var cands []candidate + for raw, info := range p.Versions { + v, err := semver.Parse(raw) + if err != nil || !rng.Satisfies(v) { + continue + } + ok, err := published(raw) + if err != nil { + return VersionInfo{}, err + } + if ok { + cands = append(cands, candidate{v, info}) + } + } + if len(cands) == 0 { + suffix := "" + if !asOf.IsZero() { + suffix = " published by " + asOf.Format("2006-01-02") + } + return VersionInfo{}, fmt.Errorf("no version of %s satisfies %q%s", p.Name, spec, suffix) + } + sort.Slice(cands, func(i, j int) bool { return semver.Compare(cands[i].v, cands[j].v) > 0 }) + + if asOf.IsZero() { + if latest, ok := p.DistTags["latest"]; ok { + for _, c := range cands { + if c.info.Version == latest && !c.info.deprecated() { + return c.info, nil + } + } + } + } + for _, wantStable := range []bool{true, false} { + for _, c := range cands { + if c.info.deprecated() || (wantStable && len(c.v.Pre) > 0) { + continue + } + return c.info, nil + } + } + return cands[0].info, nil // everything that qualifies is deprecated +} + +// checkSpecifier rejects the dependency forms that are not registry versions. +func checkSpecifier(spec string) error { + for _, prefix := range []string{"git", "file:", "link:", "workspace:", "npm:", "github:", "http:", "https:", "/", "./", "../", "~/"} { + if strings.HasPrefix(spec, prefix) { + return fmt.Errorf("unsupported dependency specifier %q (only registry versions are supported)", spec) + } + } + if strings.Contains(spec, "://") { + return fmt.Errorf("unsupported dependency specifier %q (only registry versions are supported)", spec) + } + return nil +} + +// Download fetches the tarball of a version and verifies it against the registry's integrity +// data. A registry that publishes none is an error: the download would be unverifiable. +func (c *Client) Download(name string, v VersionInfo) ([]byte, error) { + if v.Dist.Integrity == "" && v.Dist.Shasum == "" { + return nil, fmt.Errorf("%s@%s: the registry gives no integrity or shasum to verify the tarball with", name, v.Version) + } + tarball := v.Dist.Tarball + if tarball == "" { + base := name + if i := strings.LastIndex(base, "/"); i >= 0 { + base = base[i+1:] + } + tarball = fmt.Sprintf("%s/%s/-/%s-%s.tgz", c.BaseURL, name, base, v.Version) + } + resp, err := c.HTTP.Get(tarball) + if err != nil { + return nil, fmt.Errorf("downloading %s: %w", tarball, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("downloading %s: status %d", tarball, resp.StatusCode) + } + data, err := io.ReadAll(io.LimitReader(resp.Body, maxTarball+1)) + if err != nil { + return nil, fmt.Errorf("downloading %s: %w", tarball, err) + } + if len(data) > maxTarball { + return nil, fmt.Errorf("%s is larger than %d bytes", tarball, maxTarball) + } + if err := Verify(name, v, data); err != nil { + return nil, err + } + return data, nil +} + +// Verify checks data against the integrity field of v (any of its sha256, sha384 or sha512 +// entries; the strongest is required to match) or, when there is none, its sha1 shasum. +func Verify(name string, v VersionInfo, data []byte) error { + if v.Dist.Integrity != "" { + best, bestRank := "", -1 + for _, entry := range strings.Fields(v.Dist.Integrity) { + algo, _, ok := strings.Cut(entry, "-") + rank := map[string]int{"sha256": 1, "sha384": 2, "sha512": 3}[algo] + if ok && rank > bestRank { + best, bestRank = entry, rank + } + } + if bestRank < 0 { + return fmt.Errorf("%s@%s: unsupported integrity %q", name, v.Version, v.Dist.Integrity) + } + algo, want, _ := strings.Cut(best, "-") + var sum []byte + switch algo { + case "sha256": + s := sha256.Sum256(data) + sum = s[:] + case "sha384": + s := sha512.Sum384(data) + sum = s[:] + default: + s := sha512.Sum512(data) + sum = s[:] + } + if base64.StdEncoding.EncodeToString(sum) != want { + return fmt.Errorf("%s@%s: the tarball does not match the registry's %s integrity", name, v.Version, algo) + } + return nil + } + s := sha1.Sum(data) + if hex.EncodeToString(s[:]) != strings.ToLower(v.Dist.Shasum) { + return fmt.Errorf("%s@%s: the tarball does not match the registry's shasum", name, v.Version) + } + return nil +} diff --git a/tools/please_ts/registry/registry_test.go b/tools/please_ts/registry/registry_test.go new file mode 100644 index 0000000..d2d1b71 --- /dev/null +++ b/tools/please_ts/registry/registry_test.go @@ -0,0 +1,242 @@ +package registry + +import ( + "crypto/sha1" + "crypto/sha512" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func packument(t *testing.T, doc string) *Packument { + t.Helper() + var p Packument + if err := json.Unmarshal([]byte(doc), &p); err != nil { + t.Fatal(err) + } + return &p +} + +const msPackument = `{ + "name": "ms", + "dist-tags": {"latest": "2.1.3", "next": "3.0.0-beta.1"}, + "versions": { + "1.0.0": {"version": "1.0.0"}, + "2.0.0": {"version": "2.0.0"}, + "2.1.2": {"version": "2.1.2"}, + "2.1.3": {"version": "2.1.3"}, + "2.2.0": {"version": "2.2.0"}, + "2.3.0": {"version": "2.3.0", "deprecated": "use 2.2.0"}, + "3.0.0-beta.1": {"version": "3.0.0-beta.1"} + }, + "time": { + "1.0.0": "2015-01-01T00:00:00.000Z", "2.0.0": "2017-01-01T00:00:00.000Z", + "2.1.2": "2019-01-01T00:00:00.000Z", "2.1.3": "2020-01-01T00:00:00.000Z", + "2.2.0": "2023-01-01T00:00:00.000Z", "2.3.0": "2024-01-01T00:00:00.000Z", + "3.0.0-beta.1": "2024-06-01T00:00:00.000Z" + } +}` + +func TestResolve(t *testing.T) { + p := packument(t, msPackument) + date := func(s string) time.Time { + d, err := time.Parse("2006-01-02", s) + if err != nil { + t.Fatal(err) + } + return d.Add(24*time.Hour - time.Second) + } + for _, tc := range []struct { + name, spec string + asOf time.Time + want string + }{ + {"the latest tag when it satisfies", "^2.0.0", time.Time{}, "2.1.3"}, + {"highest when latest does not satisfy", "^1.0.0", time.Time{}, "1.0.0"}, + {"highest satisfying, skipping a deprecated one", "~2.2.0", time.Time{}, "2.2.0"}, + {"exact", "2.1.2", time.Time{}, "2.1.2"}, + {"star prefers latest", "*", time.Time{}, "2.1.3"}, + {"empty prefers latest", "", time.Time{}, "2.1.3"}, + {"a range of alternatives", "1.0.0 || 2.0.0", time.Time{}, "2.0.0"}, + {"a dist-tag", "latest", time.Time{}, "2.1.3"}, + {"another dist-tag", "next", time.Time{}, "3.0.0-beta.1"}, + {"a range that allows a prerelease", ">=3.0.0-beta.0", time.Time{}, "3.0.0-beta.1"}, + {"as of a date ignores the latest tag", "^2.0.0", date("2019-06-01"), "2.1.2"}, + {"as of a date, the newest published by then", "^2.0.0", date("2023-06-01"), "2.2.0"}, + {"as of the day of a release includes it", "^2.0.0", date("2020-01-01"), "2.1.3"}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := Resolve(p, tc.spec, tc.asOf) + if err != nil || got.Version != tc.want { + t.Errorf("Resolve(%q) = %q, %v; want %s", tc.spec, got.Version, err, tc.want) + } + }) + } +} + +func TestResolveNeverGuesses(t *testing.T) { + p := packument(t, msPackument) + for name, tc := range map[string]struct{ spec, want string }{ + "no version satisfies": {"^9.0.0", "no version of ms satisfies"}, + "only a prerelease exists": {"^3.0.0", "no version of ms satisfies"}, + "not a range or a tag": {"banana", "neither a version range nor a dist-tag"}, + "git dependency": {"git+https://github.com/x/y.git", "unsupported dependency specifier"}, + "github shorthand": {"github:x/y", "unsupported dependency specifier"}, + "file dependency": {"file:../x", "unsupported dependency specifier"}, + "npm alias": {"npm:other@^1", "unsupported dependency specifier"}, + "workspace": {"workspace:*", "unsupported dependency specifier"}, + "url": {"https://example.com/x.tgz", "unsupported dependency specifier"}, + "relative path": {"./local", "unsupported dependency specifier"}, + } { + t.Run(name, func(t *testing.T) { + if got, err := Resolve(p, tc.spec, time.Time{}); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("Resolve(%q) = %q, %v; want an error containing %q", tc.spec, got.Version, err, tc.want) + } + }) + } +} + +func TestResolveAsOfErrors(t *testing.T) { + p := packument(t, msPackument) + asOf := time.Date(2016, 1, 1, 0, 0, 0, 0, time.UTC) + if _, err := Resolve(p, "^2.0.0", asOf); err == nil || !strings.Contains(err.Error(), "published by 2016-01-01") { + t.Errorf("nothing was published by then, got %v", err) + } + if _, err := Resolve(p, "latest", asOf); err == nil || !strings.Contains(err.Error(), "published after") { + t.Errorf("a dist-tag newer than the date must be an error, got %v", err) + } + abbreviated := packument(t, `{"name":"ms","versions":{"1.0.0":{"version":"1.0.0"}}}`) + if _, err := Resolve(abbreviated, "^1.0.0", asOf); err == nil || !strings.Contains(err.Error(), "no publish time") { + t.Errorf("an as-of date needs publish times, got %v", err) + } +} + +func TestResolveFallsBackToADeprecatedVersionOnlyWhenNothingElseQualifies(t *testing.T) { + p := packument(t, `{"name":"x","versions":{"1.0.0":{"version":"1.0.0","deprecated":"old"},"1.1.0":{"version":"1.1.0","deprecated":"older"}}}`) + got, err := Resolve(p, "^1.0.0", time.Time{}) + if err != nil || got.Version != "1.1.0" { + t.Errorf("Resolve = %q, %v; want the highest deprecated version", got.Version, err) + } +} + +func integrityOf(data []byte) string { + s := sha512.Sum512(data) + return "sha512-" + base64.StdEncoding.EncodeToString(s[:]) +} + +func TestVerify(t *testing.T) { + data := []byte("tarball bytes") + sha1sum := sha1.Sum(data) + shasum := hex.EncodeToString(sha1sum[:]) + for _, tc := range []struct { + name string + dist Dist + ok bool + }{ + {"sha512 integrity", Dist{Integrity: integrityOf(data)}, true}, + {"integrity wins over a wrong shasum", Dist{Integrity: integrityOf(data), Shasum: "00"}, true}, + {"several integrity entries, the strongest matches", Dist{Integrity: "sha256-AAAA " + integrityOf(data)}, true}, + {"the strongest does not match", Dist{Integrity: "sha256-AAAA sha512-AAAA"}, false}, + {"wrong integrity", Dist{Integrity: "sha512-AAAA", Shasum: shasum}, false}, + {"shasum alone", Dist{Shasum: shasum}, true}, + {"uppercase shasum", Dist{Shasum: strings.ToUpper(shasum)}, true}, + {"wrong shasum", Dist{Shasum: "0000"}, false}, + {"unsupported algorithm", Dist{Integrity: "md5-AAAA"}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + err := Verify("x", VersionInfo{Version: "1.0.0", Dist: tc.dist}, data) + if (err == nil) != tc.ok { + t.Errorf("Verify = %v, want ok=%v", err, tc.ok) + } + }) + } +} + +func newRegistry(t *testing.T, handler func(w http.ResponseWriter, r *http.Request)) *Client { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(handler)) + t.Cleanup(srv.Close) + return New(srv.URL) +} + +func TestPackumentFetchesAndEscapesScopedNames(t *testing.T) { + var gotPath, gotAccept string + c := newRegistry(t, func(w http.ResponseWriter, r *http.Request) { + gotPath, gotAccept = r.URL.EscapedPath(), r.Header.Get("Accept") + fmt.Fprint(w, msPackument) + }) + p, err := c.Packument("@scope/ms", true) + if err != nil || p.Name != "ms" || len(p.Versions) != 7 { + t.Fatalf("Packument = %+v, %v", p, err) + } + if gotPath != "/@scope%2Fms" && gotPath != "/@scope/ms" { + t.Errorf("path = %q", gotPath) + } + if gotAccept != "application/json" { + t.Errorf("a full packument must ask for plain JSON, Accept = %q", gotAccept) + } + if _, err := c.Packument("ms", false); err != nil || !strings.Contains(gotAccept, "install-v1") { + t.Errorf("an abbreviated packument must ask for the install format, Accept = %q, %v", gotAccept, err) + } +} + +func TestPackumentErrors(t *testing.T) { + c := newRegistry(t, func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/bad") { + fmt.Fprint(w, "{") + return + } + http.NotFound(w, r) + }) + if _, err := c.Packument("missing", false); err == nil || !strings.Contains(err.Error(), "404") { + t.Errorf("a 404 must be reported, got %v", err) + } + if _, err := c.Packument("bad", false); err == nil { + t.Error("invalid JSON must be an error") + } +} + +func TestDownloadVerifiesTheTarball(t *testing.T) { + data := []byte("the real tarball") + var served = data + c := newRegistry(t, func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(served) }) + + good := VersionInfo{Version: "1.0.0", Dist: Dist{Tarball: c.BaseURL + "/x/-/x-1.0.0.tgz", Integrity: integrityOf(data)}} + if got, err := c.Download("x", good); err != nil || string(got) != string(data) { + t.Fatalf("Download = %q, %v", got, err) + } + + served = []byte("a different tarball") + if _, err := c.Download("x", good); err == nil || !strings.Contains(err.Error(), "does not match") { + t.Errorf("tampered content must fail verification, got %v", err) + } + + noIntegrity := VersionInfo{Version: "1.0.0", Dist: Dist{Tarball: c.BaseURL + "/x/-/x-1.0.0.tgz"}} + if _, err := c.Download("x", noIntegrity); err == nil || !strings.Contains(err.Error(), "no integrity") { + t.Errorf("a version without integrity data must be refused, got %v", err) + } +} + +func TestDownloadBuildsTheTarballURLWhenTheRegistryOmitsIt(t *testing.T) { + data := []byte("x") + var path string + c := newRegistry(t, func(w http.ResponseWriter, r *http.Request) { path = r.URL.Path; _, _ = w.Write(data) }) + v := VersionInfo{Version: "2.0.0", Dist: Dist{Integrity: integrityOf(data)}} + if _, err := c.Download("@scope/pkg", v); err != nil || path != "/@scope/pkg/-/pkg-2.0.0.tgz" { + t.Errorf("path = %q, err = %v", path, err) + } +} + +func TestDownloadReportsHTTPFailures(t *testing.T) { + c := newRegistry(t, func(w http.ResponseWriter, r *http.Request) { http.Error(w, "boom", http.StatusInternalServerError) }) + v := VersionInfo{Version: "1.0.0", Dist: Dist{Tarball: c.BaseURL + "/x.tgz", Shasum: "00"}} + if _, err := c.Download("x", v); err == nil || !strings.Contains(err.Error(), "500") { + t.Errorf("got %v", err) + } +} diff --git a/tools/please_ts/semver/BUILD b/tools/please_ts/semver/BUILD new file mode 100644 index 0000000..eca299a --- /dev/null +++ b/tools/please_ts/semver/BUILD @@ -0,0 +1,15 @@ +subinclude("///go//build_defs:go") + +go_library( + name = "semver", + srcs = ["semver.go"], + visibility = ["//tools/please_ts/..."], +) + +go_test( + name = "semver_test", + srcs = [ + "semver.go", + "semver_test.go", + ], +) diff --git a/tools/please_ts/semver/semver.go b/tools/please_ts/semver/semver.go new file mode 100644 index 0000000..0aeae84 --- /dev/null +++ b/tools/please_ts/semver/semver.go @@ -0,0 +1,415 @@ +// Package semver implements the parts of npm's semantic versioning that dependency +// resolution needs: versions with prereleases, and ranges with ^, ~, comparators, +// x-ranges, hyphen ranges and ||. +package semver + +import ( + "fmt" + "strconv" + "strings" +) + +// Version is a parsed semantic version. Build metadata is ignored, as it is for +// precedence. +type Version struct { + Major, Minor, Patch uint64 + Pre []string // prerelease identifiers +} + +// Parse parses "1.2.3", "v1.2.3" or "1.2.3-beta.1+build". All three numbers are required. +func Parse(s string) (Version, error) { + orig := s + s = strings.TrimSpace(s) + s = strings.TrimPrefix(strings.TrimPrefix(s, "="), "v") + if i := strings.IndexByte(s, '+'); i >= 0 { + s = s[:i] + } + var pre string + if i := strings.IndexByte(s, '-'); i >= 0 { + s, pre = s[:i], s[i+1:] + if pre == "" { + return Version{}, fmt.Errorf("invalid version %q", orig) + } + } + parts := strings.Split(s, ".") + if len(parts) != 3 { + return Version{}, fmt.Errorf("invalid version %q", orig) + } + var nums [3]uint64 + for i, p := range parts { + n, err := parseNum(p) + if err != nil { + return Version{}, fmt.Errorf("invalid version %q", orig) + } + nums[i] = n + } + v := Version{Major: nums[0], Minor: nums[1], Patch: nums[2]} + if pre != "" { + v.Pre = strings.Split(pre, ".") + for _, id := range v.Pre { + if id == "" { + return Version{}, fmt.Errorf("invalid version %q", orig) + } + } + } + return v, nil +} + +func parseNum(s string) (uint64, error) { + if s == "" || (len(s) > 1 && s[0] == '0') { + return 0, fmt.Errorf("bad number") + } + return strconv.ParseUint(s, 10, 64) +} + +func (v Version) String() string { + s := fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch) + if len(v.Pre) > 0 { + s += "-" + strings.Join(v.Pre, ".") + } + return s +} + +// Compare returns -1, 0 or 1 following semver precedence: a prerelease sorts before its +// release, and prerelease identifiers compare numerically when numeric. +func Compare(a, b Version) int { + for _, d := range [][2]uint64{{a.Major, b.Major}, {a.Minor, b.Minor}, {a.Patch, b.Patch}} { + if d[0] != d[1] { + if d[0] < d[1] { + return -1 + } + return 1 + } + } + switch { + case len(a.Pre) == 0 && len(b.Pre) == 0: + return 0 + case len(a.Pre) == 0: + return 1 + case len(b.Pre) == 0: + return -1 + } + for i := 0; i < len(a.Pre) && i < len(b.Pre); i++ { + if c := comparePre(a.Pre[i], b.Pre[i]); c != 0 { + return c + } + } + switch { + case len(a.Pre) < len(b.Pre): + return -1 + case len(a.Pre) > len(b.Pre): + return 1 + } + return 0 +} + +func comparePre(a, b string) int { + an, aerr := strconv.ParseUint(a, 10, 64) + bn, berr := strconv.ParseUint(b, 10, 64) + switch { + case aerr == nil && berr == nil: + if an < bn { + return -1 + } else if an > bn { + return 1 + } + return 0 + case aerr == nil: // numeric identifiers sort before alphanumeric ones + return -1 + case berr == nil: + return 1 + } + return strings.Compare(a, b) +} + +type comparator struct { + op string // "<", "<=", ">", ">=", "=" + v Version +} + +func (c comparator) test(v Version) bool { + cmp := Compare(v, c.v) + switch c.op { + case "<": + return cmp < 0 + case "<=": + return cmp <= 0 + case ">": + return cmp > 0 + case ">=": + return cmp >= 0 + } + return cmp == 0 +} + +// Range is a parsed npm version range: alternatives (||) of comparator sets. +type Range struct { + sets [][]comparator +} + +// ParseRange parses an npm range such as "^1.2.3", "~1.2", ">=1 <3", "1.x || ^2", "1.2.3 - 2.3.4" +// or "*". The empty string means "*". +func ParseRange(s string) (Range, error) { + var r Range + for _, alt := range strings.Split(s, "||") { + set, err := parseSet(strings.TrimSpace(alt)) + if err != nil { + return Range{}, fmt.Errorf("invalid range %q: %w", s, err) + } + r.sets = append(r.sets, set) + } + return r, nil +} + +// Satisfies reports whether v is in the range. A prerelease version only satisfies a +// comparator set that has a comparator with a prerelease on the same major.minor.patch. +func (r Range) Satisfies(v Version) bool { + for _, set := range r.sets { + if satisfiesSet(set, v) { + return true + } + } + return false +} + +func satisfiesSet(set []comparator, v Version) bool { + for _, c := range set { + if !c.test(v) { + return false + } + } + if len(v.Pre) == 0 { + return true + } + for _, c := range set { + if len(c.v.Pre) > 0 && c.v.Major == v.Major && c.v.Minor == v.Minor && c.v.Patch == v.Patch { + return true + } + } + return false +} + +// partial is a version with optional minor and patch ("1", "1.2", "1.2.x", "1.2.3-beta"). +type partial struct { + major, minor, patch int64 // -1 when missing or a wildcard + pre []string +} + +func parsePartial(s string) (partial, error) { + s = strings.TrimSpace(s) + s = strings.TrimPrefix(strings.TrimPrefix(s, "="), "v") + if s == "" || s == "*" || s == "x" || s == "X" { + return partial{-1, -1, -1, nil}, nil + } + if i := strings.IndexByte(s, '+'); i >= 0 { + s = s[:i] + } + var pre []string + if i := strings.IndexByte(s, '-'); i >= 0 { + pre = strings.Split(s[i+1:], ".") + s = s[:i] + } + parts := strings.Split(s, ".") + if len(parts) > 3 { + return partial{}, fmt.Errorf("bad version %q", s) + } + p := partial{-1, -1, -1, pre} + slots := []*int64{&p.major, &p.minor, &p.patch} + wildcard := false + for i, part := range parts { + if part == "x" || part == "X" || part == "*" { + wildcard = true // a wildcard opens everything after it + continue + } + if wildcard { + return partial{}, fmt.Errorf("bad version %q: a number follows a wildcard", s) + } + n, err := strconv.ParseInt(part, 10, 64) + if err != nil || n < 0 { + return partial{}, fmt.Errorf("bad version %q", s) + } + *slots[i] = n + } + if (p.minor < 0 && p.patch >= 0) || (p.major < 0 && p.minor >= 0) { + return partial{}, fmt.Errorf("bad version %q", s) + } + return p, nil +} + +func (p partial) version(fill uint64) Version { + v := Version{Pre: p.pre} + nums := []*uint64{&v.Major, &v.Minor, &v.Patch} + for i, n := range []int64{p.major, p.minor, p.patch} { + if n >= 0 { + *nums[i] = uint64(n) + } else { + *nums[i] = fill + } + } + return v +} + +// floor is the lowest version the partial denotes ("1.2" is 1.2.0). +func (p partial) floor() Version { return p.version(0) } + +// ceiling returns the exclusive upper bound of the partial ("1.2" is <1.3.0-0). +func (p partial) ceiling() (Version, bool) { + switch { + case p.major < 0: + return Version{}, false + case p.minor < 0: + return Version{Major: uint64(p.major) + 1, Pre: []string{"0"}}, true + case p.patch < 0: + return Version{Major: uint64(p.major), Minor: uint64(p.minor) + 1, Pre: []string{"0"}}, true + } + return Version{}, false +} + +func parseSet(s string) ([]comparator, error) { + if s == "" { + return nil, nil // "*" + } + fields := strings.Fields(s) + // Hyphen range: "A - B". + if len(fields) == 3 && fields[1] == "-" { + lo, err := parsePartial(fields[0]) + if err != nil { + return nil, err + } + hi, err := parsePartial(fields[2]) + if err != nil { + return nil, err + } + var set []comparator + if lo.major >= 0 { + set = append(set, comparator{">=", lo.floor()}) + } + if c, ok := hi.ceiling(); ok { + set = append(set, comparator{"<", c}) + } else if hi.major >= 0 { + set = append(set, comparator{"<=", hi.floor()}) + } + return set, nil + } + + // "> 1.2.3" is written with a space in some manifests: glue a lone operator to the next field. + var glued []string + for i := 0; i < len(fields); i++ { + f := fields[i] + if isOperator(f) && i+1 < len(fields) { + f += fields[i+1] + i++ + } + glued = append(glued, f) + } + + var set []comparator + for _, f := range glued { + cs, err := parseComparator(f) + if err != nil { + return nil, err + } + set = append(set, cs...) + } + return set, nil +} + +func isOperator(s string) bool { + switch s { + case "<", "<=", ">", ">=", "=", "^", "~", "~>": + return true + } + return false +} + +func parseComparator(f string) ([]comparator, error) { + op := "" + for _, o := range []string{"~>", "<=", ">=", "<", ">", "=", "^", "~"} { + if strings.HasPrefix(f, o) { + op = o + f = f[len(o):] + break + } + } + p, err := parsePartial(f) + if err != nil { + return nil, err + } + + switch op { + case "^": + return caret(p), nil + case "~", "~>": + return tilde(p), nil + case ">": + if c, ok := p.ceiling(); ok { + return []comparator{{">=", withoutPre(c)}}, nil + } + if p.major < 0 { + return []comparator{{"<", Version{}}}, nil // ">*" matches nothing + } + return []comparator{{">", p.floor()}}, nil + case ">=": + if p.major < 0 { + return nil, nil + } + return []comparator{{">=", p.floor()}}, nil + case "<": + if p.major < 0 { + return []comparator{{"<", Version{}}}, nil // "<*" matches nothing + } + return []comparator{{"<", p.floor()}}, nil + case "<=": + if c, ok := p.ceiling(); ok { + return []comparator{{"<", c}}, nil + } + if p.major < 0 { + return nil, nil + } + return []comparator{{"<=", p.floor()}}, nil + } + + // No operator or "=": an exact version or an x-range. + if p.major < 0 { + return nil, nil + } + if c, ok := p.ceiling(); ok { + return []comparator{{">=", p.floor()}, {"<", c}}, nil + } + return []comparator{{"=", p.floor()}}, nil +} + +func withoutPre(v Version) Version { v.Pre = nil; return v } + +func caret(p partial) []comparator { + if p.major < 0 { + return nil + } + lo := p.floor() + var hi Version + switch { + case p.major > 0 || (p.minor < 0): + hi = Version{Major: lo.Major + 1} + case p.minor > 0 || p.patch < 0: + hi = Version{Major: 0, Minor: lo.Minor + 1} + default: + hi = Version{Major: 0, Minor: 0, Patch: lo.Patch + 1} + } + hi.Pre = []string{"0"} + return []comparator{{">=", lo}, {"<", hi}} +} + +func tilde(p partial) []comparator { + if p.major < 0 { + return nil + } + lo := p.floor() + var hi Version + if p.minor < 0 { + hi = Version{Major: lo.Major + 1} + } else { + hi = Version{Major: lo.Major, Minor: lo.Minor + 1} + } + hi.Pre = []string{"0"} + return []comparator{{">=", lo}, {"<", hi}} +} diff --git a/tools/please_ts/semver/semver_test.go b/tools/please_ts/semver/semver_test.go new file mode 100644 index 0000000..fed79f3 --- /dev/null +++ b/tools/please_ts/semver/semver_test.go @@ -0,0 +1,119 @@ +package semver + +import "testing" + +func TestParse(t *testing.T) { + for in, want := range map[string]string{ + "1.2.3": "1.2.3", + "v1.2.3": "1.2.3", + "=1.2.3": "1.2.3", + " 1.2.3 ": "1.2.3", + "1.2.3-beta.1": "1.2.3-beta.1", + "1.2.3-beta.1+b.5": "1.2.3-beta.1", + "1.2.3+build": "1.2.3", + "0.0.0": "0.0.0", + } { + v, err := Parse(in) + if err != nil || v.String() != want { + t.Errorf("Parse(%q) = %v, %v; want %s", in, v, err, want) + } + } + for _, in := range []string{"", "1.2", "1.2.3.4", "a.b.c", "01.2.3", "1.2.3-", "1.2.3-a..b", "1.x.3", ">1.2.3", "-1.2.3"} { + if _, err := Parse(in); err == nil { + t.Errorf("Parse(%q): expected an error", in) + } + } +} + +func TestComparePrecedence(t *testing.T) { + // The ordering example of the semver specification, lowest first. + ordered := []string{"1.0.0-alpha", "1.0.0-alpha.1", "1.0.0-alpha.beta", "1.0.0-beta", "1.0.0-beta.2", "1.0.0-beta.11", "1.0.0-rc.1", "1.0.0", "1.0.1", "1.1.0", "2.0.0", "10.0.0"} + for i := range ordered { + for j := range ordered { + a, _ := Parse(ordered[i]) + b, _ := Parse(ordered[j]) + want := 0 + if i < j { + want = -1 + } else if i > j { + want = 1 + } + if got := Compare(a, b); got != want { + t.Errorf("Compare(%s, %s) = %d, want %d", ordered[i], ordered[j], got, want) + } + } + } + a, _ := Parse("1.0.0+a") + b, _ := Parse("1.0.0+b") + if Compare(a, b) != 0 { + t.Error("build metadata must not affect precedence") + } +} + +func TestRangeIncludes(t *testing.T) { + for _, tc := range [][2]string{ + {"1.0.0 - 2.0.0", "1.2.3"}, {"1.0.0 - 2.0.0", "2.0.0"}, {"1.2.3 - 2.3", "2.3.9"}, {"1.2 - 2.3.4", "1.2.0"}, {"1.2.3 - 2", "2.9.9"}, + {"^1.2.3", "1.2.3"}, {"^1.2.3", "1.9.9"}, {"^1.2", "1.9.0"}, {"^1.2.x", "1.2.0"}, {"^1", "1.0.0"}, + {"^0.2.3", "0.2.9"}, {"^0.0.3", "0.0.3"}, {"^0.0", "0.0.9"}, {"^0", "0.9.9"}, {"^0.x", "0.5.0"}, + {"~1.2.3", "1.2.9"}, {"~1.2", "1.2.0"}, {"~1", "1.9.9"}, {"~>1.2.3", "1.2.4"}, + {"1.2.x", "1.2.3"}, {"1.x", "1.9.0"}, {"1.X", "1.0.0"}, {"*", "1.2.3"}, {"x", "9.9.9"}, {"", "1.0.0"}, {"^", "1.0.0"}, + {"1", "1.5.0"}, {"1.2", "1.2.9"}, {"=1.2.3", "1.2.3"}, {"v1.2.3", "1.2.3"}, {"1.2.3", "1.2.3"}, + {">=1.0.0", "1.0.0"}, {">1.0.0", "1.0.1"}, {"<2.0.0", "1.9.9"}, {"<=2.0.0", "2.0.0"}, {"<=2", "2.9.9"}, {"<=1.2", "1.2.9"}, + {">1.2", "1.3.0"}, {">1", "2.0.0"}, {">=1.2", "1.2.0"}, {"<1.2", "1.1.9"}, + {">=1.2.3 <2", "1.9.9"}, {">= 1.2.3", "1.2.3"}, {"> 1.2.3 < 2.0.0", "1.5.0"}, + {"1.2.3 || 2.0.0", "2.0.0"}, {"^1 || ^2", "2.5.0"}, {"<1 || >=3", "3.1.0"}, + {"^1.2.3-beta.2", "1.2.3-beta.4"}, {"^1.2.3-beta.2", "1.3.0"}, {">=1.2.3-beta.2", "1.2.3-beta.4"}, {"1.2.3-beta.2", "1.2.3-beta.2"}, + } { + r, err := ParseRange(tc[0]) + if err != nil { + t.Errorf("ParseRange(%q): %v", tc[0], err) + continue + } + v, _ := Parse(tc[1]) + if !r.Satisfies(v) { + t.Errorf("%q should include %s", tc[0], tc[1]) + } + } +} + +func TestRangeExcludes(t *testing.T) { + for _, tc := range [][2]string{ + {"^1.2.3", "2.0.0"}, {"^1.2.3", "1.2.2"}, {"^0.2.3", "0.3.0"}, {"^0.0.3", "0.0.4"}, {"^0.0", "0.1.0"}, {"^0", "1.0.0"}, + {"~1.2.3", "1.3.0"}, {"~1.2", "1.3.0"}, {"~1", "2.0.0"}, + {"1.2.x", "1.3.0"}, {"1.x", "2.0.0"}, {"1", "2.0.0"}, {"1.2", "1.3.0"}, {"1.2.3", "1.2.4"}, + {">1.0.0", "1.0.0"}, {"<1.0.0", "1.0.0"}, {">1.2", "1.2.9"}, {">1", "1.9.9"}, {"<=1.2", "1.3.0"}, + {"1.2.3 - 2.3.4", "2.3.5"}, {"1.2.3 - 2.3", "2.4.0"}, {"1.2.3 - 2", "3.0.0"}, {"1.0.0 - 2.0.0", "0.9.9"}, + {"1.2.3 || 2.0.0", "1.2.4"}, {">=1 <3", "3.0.0"}, {"<1 || >=3", "2.0.0"}, + // Prereleases only match a comparator that names a prerelease of the same version. + {"^1.2.3", "1.2.4-beta.1"}, {"^1.2.3-beta.2", "1.2.4-beta.1"}, {"*", "1.2.3-beta.1"}, {">=1.0.0", "2.0.0-alpha"}, + {"<2.0.0", "2.0.0-alpha"}, {"^1.2.3-beta.2", "1.2.3-beta.1"}, {"1.x", "1.2.3-beta.1"}, + } { + r, err := ParseRange(tc[0]) + if err != nil { + t.Errorf("ParseRange(%q): %v", tc[0], err) + continue + } + v, _ := Parse(tc[1]) + if r.Satisfies(v) { + t.Errorf("%q should not include %s", tc[0], tc[1]) + } + } +} + +func TestParseRangeErrors(t *testing.T) { + for _, in := range []string{"1.2.3.4", "a", "1.x.3", ">=a", "^a.b", "1.2.3 - ", "||x.y"} { + if _, err := ParseRange(in); err == nil { + t.Errorf("ParseRange(%q): expected an error", in) + } + } +} + +func TestRangeNothingMatchesStarComparators(t *testing.T) { + v, _ := Parse("1.0.0") + for _, in := range []string{">*", "<*"} { + r, err := ParseRange(in) + if err != nil || r.Satisfies(v) { + t.Errorf("%q: err=%v includes=%v; want a range that matches nothing", in, err, r.Satisfies(v)) + } + } +} diff --git a/tools/please_ts/unpack/BUILD b/tools/please_ts/unpack/BUILD index 4aceab7..659a418 100644 --- a/tools/please_ts/unpack/BUILD +++ b/tools/please_ts/unpack/BUILD @@ -4,12 +4,15 @@ go_library( name = "unpack", srcs = [ "npmcache.go", + "resolve.go", "unpack.go", ], visibility = ["//tools/please_ts/..."], deps = [ "//tools/please_ts/importmap", "//tools/please_ts/npmcache", + "//tools/please_ts/registry", + "//tools/please_ts/semver", ], ) @@ -18,11 +21,15 @@ go_test( srcs = [ "npmcache.go", "npmcache_test.go", + "resolve.go", + "resolve_test.go", "unpack.go", "unpack_test.go", ], deps = [ "//tools/please_ts/importmap", "//tools/please_ts/npmcache", + "//tools/please_ts/registry", + "//tools/please_ts/semver", ], ) diff --git a/tools/please_ts/unpack/npmcache.go b/tools/please_ts/unpack/npmcache.go index 05a9f0f..8b61e51 100644 --- a/tools/please_ts/unpack/npmcache.go +++ b/tools/please_ts/unpack/npmcache.go @@ -13,58 +13,51 @@ import ( "tools/please_ts/npmcache" ) -// buildNpmCache turns a hash-pinned npm tarball into a slice of a Deno npm cache, with no -// network access: the package is extracted into npm/registry.npmjs.org/// -// and described by a minimal registry.json, so that Deno resolves npm:@ -// from its cache. Deno does not verify the cache, so the integrity of the package is the -// sha256 Please checked on the tarball. +// buildNpmCache turns an npm tarball into a slice of a Deno npm cache: the package is +// extracted into npm/registry.npmjs.org/// and described by a minimal +// registry.json, so that Deno resolves npm:@ from its cache. Deno does not +// verify the cache, so the integrity of the package is the sha256 Please checked on the +// tarball. // -// Every dependency the package declares must be provided by another slice staged in the -// build directory (a ts_npm_module in deps); otherwise Deno would only notice at run time, -// by trying the network. +// The dependencies of the package come from other slices staged in the build directory (a +// ts_npm_module in deps), whose packages are bundled into this slice. With +// ResolveTransitive, dependencies that no staged slice provides are resolved from the +// registry instead, each verified against the registry's integrity data, and kept as +// separate versions when dependents need different ones. Without it every dependency must +// be provided, and the build fails naming those that are not, since Deno would only notice +// at run time, by trying the network. func buildNpmCache(opts Options) error { - tmp, err := os.MkdirTemp("", "please_ts_npm_*") + data, err := os.ReadFile(opts.ArchivePath()) if err != nil { return err } - defer os.RemoveAll(tmp) - - if err := extractArchive(opts.ArchivePath(), tmp); err != nil { - return fmt.Errorf("failed extracting %s: %w", opts.ArchivePath(), err) - } - pkg, raw, err := readPackageManifest(tmp) + pkg, _, err := inspectTarball(data, opts.Name) if err != nil { - return err - } - if pkg.Name == "" || pkg.Version == "" { - return fmt.Errorf("package.json has no name or version") - } - if opts.Name != "" && opts.Name != pkg.Name { - return fmt.Errorf("tarball holds package %q, expected %q", pkg.Name, opts.Name) + return fmt.Errorf("%s: %w", opts.ArchivePath(), err) } staged, err := stagedSlices(opts.Out) if err != nil { return err } - if err := checkDependencyClosure(pkg, staged); err != nil { - return err - } - pkgDir := filepath.Join(opts.Out, "npm", "registry.npmjs.org", filepath.FromSlash(pkg.Name)) - versionDir := filepath.Join(pkgDir, pkg.Version) - if err := os.MkdirAll(filepath.Dir(versionDir), 0755); err != nil { - return err - } - if err := copyPath(tmp, versionDir); err != nil { - return fmt.Errorf("failed staging the package: %w", err) + if opts.ResolveTransitive && len(pkg.Dependencies) > 0 { + tmp, err := os.MkdirTemp("", "please_ts_resolved_*") + if err != nil { + return err + } + defer os.RemoveAll(tmp) + resolved, err := resolveIntoSlices(pkg, staged, newResolver(opts.RegistryURL()), tmp) + if err != nil { + return fmt.Errorf("failed resolving the dependencies of %s: %w", pkg.Name, err) + } + staged = append(staged, resolved...) } - integrity, err := tarballIntegrity(opts.ArchivePath()) - if err != nil { + if err := checkDependencyClosure(pkg, staged); err != nil { return err } - if err := writePackument(pkgDir, pkg, raw, integrity); err != nil { + if _, err := writeSlice(data, opts.Name, opts.Out); err != nil { return err } @@ -87,31 +80,74 @@ func buildNpmCache(opts Options) error { }) } -// readPackageManifest reads package.json, returning both the typed fields and the raw -// object (to carry dependency fields into registry.json unchanged). -func readPackageManifest(dir string) (PackageJSON, map[string]json.RawMessage, error) { +// inspectTarball extracts a tarball to a scratch directory and reads its package.json, +// returning the typed fields and the raw object (to carry dependency fields into +// registry.json unchanged). wantName, when set, must be the package the tarball holds. +func inspectTarball(data []byte, wantName string) (PackageJSON, map[string]json.RawMessage, error) { var pkg PackageJSON - data, err := os.ReadFile(filepath.Join(dir, "package.json")) + tmp, err := os.MkdirTemp("", "please_ts_npm_*") + if err != nil { + return pkg, nil, err + } + defer os.RemoveAll(tmp) + if err := extractTarballBytes(data, tmp); err != nil { + return pkg, nil, fmt.Errorf("failed extracting the tarball: %w", err) + } + manifest, err := os.ReadFile(filepath.Join(tmp, "package.json")) if err != nil { return pkg, nil, fmt.Errorf("package.json not found in the tarball: %w", err) } - if err := json.Unmarshal(data, &pkg); err != nil { + if err := json.Unmarshal(manifest, &pkg); err != nil { return pkg, nil, fmt.Errorf("invalid package.json: %w", err) } var raw map[string]json.RawMessage - if err := json.Unmarshal(data, &raw); err != nil { + if err := json.Unmarshal(manifest, &raw); err != nil { return pkg, nil, err } + if pkg.Name == "" || pkg.Version == "" { + return pkg, nil, fmt.Errorf("package.json has no name or version") + } + if wantName != "" && wantName != pkg.Name { + return pkg, nil, fmt.Errorf("the tarball holds package %q, expected %q", pkg.Name, wantName) + } return pkg, raw, nil } -func tarballIntegrity(path string) (string, error) { - data, err := os.ReadFile(path) +// writeSlice lays a tarball out as a slice in outDir: the package in the Deno cache layout, +// its registry.json and ts_npm.json. +func writeSlice(tarball []byte, wantName, outDir string) (PackageJSON, error) { + tmp, err := os.MkdirTemp("", "please_ts_npm_*") + if err != nil { + return PackageJSON{}, err + } + defer os.RemoveAll(tmp) + if err := extractTarballBytes(tarball, tmp); err != nil { + return PackageJSON{}, fmt.Errorf("failed extracting the tarball: %w", err) + } + pkg, raw, err := inspectTarball(tarball, wantName) if err != nil { - return "", err + return pkg, err + } + + pkgDir := filepath.Join(outDir, "npm", "registry.npmjs.org", filepath.FromSlash(pkg.Name)) + versionDir := filepath.Join(pkgDir, pkg.Version) + if err := os.MkdirAll(filepath.Dir(versionDir), 0755); err != nil { + return pkg, err + } + if err := copyPath(tmp, versionDir); err != nil { + return pkg, fmt.Errorf("failed staging the package: %w", err) + } + sum := sha512.Sum512(tarball) + integrity := "sha512-" + base64.StdEncoding.EncodeToString(sum[:]) + if err := writePackument(pkgDir, pkg, raw, integrity); err != nil { + return pkg, err } - sum := sha512.Sum512(data) - return "sha512-" + base64.StdEncoding.EncodeToString(sum[:]), nil + return pkg, npmcache.Write(outDir, npmcache.Slice{ + Name: pkg.Name, + Version: pkg.Version, + Specifier: npmcache.Specifier(pkg.Name, pkg.Version), + Dependencies: pkg.Dependencies, + }) } // writePackument writes the registry.json Deno reads from its cache. The @@ -175,6 +211,54 @@ func stagedSlices(outDir string) ([]stagedSlice, error) { return staged, nil } +// resolveIntoSlices resolves the dependencies of pkg that no staged slice provides, and +// theirs in turn, writing each package as a slice under tmpDir. Versions that are already +// available (staged, or resolved for another dependent) are reused when they satisfy the +// specifier; otherwise another version is resolved, so dependents with incompatible needs +// each get theirs. Dependencies come from the package.json inside the verified tarball, not +// from registry metadata. Optional and peer dependencies are not resolved: a peer is for the +// consumer to provide. +func resolveIntoSlices(pkg PackageJSON, staged []stagedSlice, r *resolver, tmpDir string) ([]stagedSlice, error) { + r.pinToRoot(pkg.Name, pkg.Version) + have := map[string][]string{} // package -> versions available + for _, s := range staged { + have[s.Name] = append(have[s.Name], s.Version) + } + + var queue []pendingDep + enqueue := func(deps map[string]string, requiredBy string) { + for _, name := range sortedKeys(deps) { + queue = append(queue, pendingDep{name: name, spec: deps[name], requiredBy: requiredBy}) + } + } + enqueue(pkg.Dependencies, pkg.Name+"@"+pkg.Version) + + var out []stagedSlice + for len(queue) > 0 { + d := queue[0] + queue = queue[1:] + if anySatisfies(have[d.name], d.spec) { + continue + } + info, data, err := r.fetch(d.name, d.spec) + if err != nil { + return nil, fmt.Errorf("%s@%s (required by %s): %w", d.name, d.spec, d.requiredBy, err) + } + dir := filepath.Join(tmpDir, strings.ReplaceAll(d.name, "/", "+")+"@"+info.Version) + resolved, err := writeSlice(data, d.name, dir) + if err != nil { + return nil, fmt.Errorf("%s@%s: %w", d.name, info.Version, err) + } + have[d.name] = append(have[d.name], resolved.Version) + out = append(out, stagedSlice{dir: dir, Slice: npmcache.Slice{ + Name: resolved.Name, Version: resolved.Version, + Specifier: npmcache.Specifier(resolved.Name, resolved.Version), Dependencies: resolved.Dependencies, + }}) + enqueue(resolved.Dependencies, resolved.Name+"@"+resolved.Version) + } + return out, nil +} + // checkDependencyClosure verifies that each required dependency of pkg is provided by a // staged slice. Optional dependencies and peer dependencies, which the consumer provides, // are not required. @@ -202,39 +286,8 @@ func checkDependencyClosure(pkg PackageJSON, staged []stagedSlice) error { } if len(missing) > 0 { return fmt.Errorf("%s@%s depends on %s, which no ts_npm_module in deps provides; "+ - "add a ts_npm_module for each (with its tarball hash) to deps", + "add a ts_npm_module for each (with its tarball hash) to deps, or set resolve_transitive", pkg.Name, pkg.Version, strings.Join(missing, ", ")) } return nil } - -func anySatisfies(versions []string, constraint string) bool { - for _, v := range versions { - if satisfies(v, constraint) { - return true - } - } - return false -} - -// satisfies reports whether version meets an npm range. It understands exact versions, -// "*", "x"-less ranges with ^, ~ and >= prefixes; any other range form (||, spaces, <, -// hyphen ranges, x-ranges) is accepted when a version is staged at all, since the -// closure check only has to catch a dependency that was forgotten, not pick versions. -func satisfies(version, constraint string) bool { - c := strings.TrimSpace(constraint) - if c == "" || c == "*" || c == "latest" || version == c { - return true - } - prefix := "" - switch { - case strings.HasPrefix(c, "^"), strings.HasPrefix(c, "~"): - prefix, c = c[:1], c[1:] - case strings.HasPrefix(c, ">="): - prefix, c = ">=", strings.TrimSpace(c[2:]) - } - if strings.ContainsAny(c, " |<>=xX-") || c == "" { - return true - } - return matchesConstraint(parseSemver(version), parseSemver(c), prefix) -} diff --git a/tools/please_ts/unpack/npmcache_test.go b/tools/please_ts/unpack/npmcache_test.go index bfc9a74..a26fa59 100644 --- a/tools/please_ts/unpack/npmcache_test.go +++ b/tools/please_ts/unpack/npmcache_test.go @@ -213,34 +213,6 @@ func TestBuildNpmCacheRejectsBadTarballs(t *testing.T) { } } -func TestSatisfies(t *testing.T) { - for _, tc := range []struct { - version, rng string - want bool - }{ - {"2.1.3", "2.1.3", true}, - {"2.1.3", "^2.1.3", true}, - {"2.4.0", "^2.1.3", true}, - {"2.0.0", "^2.1.3", false}, - {"3.0.0", "^2.1.3", false}, - {"1.2.9", "~1.2.3", true}, - {"1.3.0", "~1.2.3", false}, - {"5.0.0", ">=2.0.0", true}, - {"1.0.0", ">=2.0.0", false}, - {"9.9.9", "*", true}, - {"9.9.9", "", true}, - {"9.9.9", "latest", true}, - // range forms that are not interpreted are accepted: only a forgotten dependency is an error - {"9.9.9", "^1 || ^2", true}, - {"9.9.9", ">=1 <3", true}, - {"9.9.9", "1.x", true}, - } { - if got := satisfies(tc.version, tc.rng); got != tc.want { - t.Errorf("satisfies(%q, %q) = %v, want %v", tc.version, tc.rng, got, tc.want) - } - } -} - // stageSliceWithPackage stages a dependency slice that holds an extracted package, as the // output of a ts_npm_module does. func stageSliceWithPackage(t *testing.T, root, name, version string) { @@ -319,3 +291,37 @@ func TestBuildNpmCacheSlicesWithDifferentVersionsOfADependencyStayResolvable(t * t.Errorf("both ms versions must be in the merged packument: %s", data) } } + +func TestVersionSatisfies(t *testing.T) { + for _, tc := range []struct { + version, spec string + want bool + }{ + {"2.1.3", "2.1.3", true}, + {"2.1.3", "^2.1.3", true}, + {"2.4.0", "^2.1.3", true}, + {"2.0.0", "^2.1.3", false}, + {"3.0.0", "^2.1.3", false}, + {"1.2.9", "~1.2.3", true}, + {"1.3.0", "~1.2.3", false}, + {"5.0.0", ">=2.0.0", true}, + {"1.0.0", ">=2.0.0", false}, + {"9.9.9", "*", true}, + {"9.9.9", "", true}, + // Ranges are interpreted completely, not approximated. + {"2.5.0", "^1 || ^2", true}, + {"3.5.0", "^1 || ^2", false}, + {"2.0.0", ">=1 <3", true}, + {"3.0.0", ">=1 <3", false}, + {"1.9.0", "1.x", true}, + {"2.0.0", "1.x", false}, + {"1.2.3-beta.1", "^1.0.0", false}, + // Specifiers that are not version ranges cannot be judged and are accepted. + {"9.9.9", "latest", true}, + {"9.9.9", "git+https://example.com/x.git", true}, + } { + if got := versionSatisfies(tc.version, tc.spec); got != tc.want { + t.Errorf("versionSatisfies(%q, %q) = %v, want %v", tc.version, tc.spec, got, tc.want) + } + } +} diff --git a/tools/please_ts/unpack/resolve.go b/tools/please_ts/unpack/resolve.go new file mode 100644 index 0000000..57a3ceb --- /dev/null +++ b/tools/please_ts/unpack/resolve.go @@ -0,0 +1,219 @@ +package unpack + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "time" + + "tools/please_ts/registry" + "tools/please_ts/semver" +) + +// resolver resolves dependency specifiers against an npm registry and downloads the +// tarballs, verified against the integrity data the registry publishes. +// +// Resolution is reproducible: only versions published by the end of the day the root package +// version was published are considered (see pinToRoot), so a range such as ^2.1.3 resolves to +// the same version next year, and the version pinned by its hash is all that fixes the result. +type resolver struct { + client *registry.Client + asOf time.Time + packuments map[string]*registry.Packument +} + +func newResolver(registryURL string) *resolver { + return &resolver{client: registry.New(registryURL), packuments: map[string]*registry.Packument{}} +} + +// pinToRoot sets the as-of date to the end of the day (UTC) the root package version was +// published. The whole day counts so that a dependency released a few minutes after the +// package that needs it is still found. If the registry does not know the root version or +// its publish time (a tarball from another URL, a private registry without times), +// resolution is not pinned, with a warning. +func (r *resolver) pinToRoot(name, version string) { + if !r.asOf.IsZero() { + return + } + p, err := r.client.Packument(name, true) + if err != nil { + fmt.Fprintf(os.Stderr, "Warning: dependencies of %s@%s are not pinned to a date: %v\n", name, version, err) + return + } + raw, ok := p.Time[version] + if !ok { + fmt.Fprintf(os.Stderr, "Warning: dependencies of %s@%s are not pinned to a date: the registry has no publish time for it\n", name, version) + return + } + published, err := time.Parse(time.RFC3339, raw) + if err != nil { + fmt.Fprintf(os.Stderr, "Warning: dependencies of %s@%s are not pinned to a date: bad publish time %q\n", name, version, raw) + return + } + day := time.Date(published.Year(), published.Month(), published.Day(), 0, 0, 0, 0, time.UTC) + r.asOf = day.Add(24*time.Hour - time.Nanosecond) + r.packuments[name] = p // the full packument is reusable +} + +// fetch resolves spec for a package to one version and downloads and verifies its tarball. +// The full packument (with publish times) is only fetched when there is an as-of date. +func (r *resolver) fetch(name, spec string) (registry.VersionInfo, []byte, error) { + p, ok := r.packuments[name] + if !ok { + var err error + if p, err = r.client.Packument(name, !r.asOf.IsZero()); err != nil { + return registry.VersionInfo{}, nil, err + } + r.packuments[name] = p + } + info, err := registry.Resolve(p, spec, r.asOf) + if err != nil { + return registry.VersionInfo{}, nil, err + } + data, err := r.client.Download(name, info) + if err != nil { + return registry.VersionInfo{}, nil, err + } + return info, data, nil +} + +// extractTarballBytes extracts a verified npm tarball into destDir, without the leading +// "package/" directory. +func extractTarballBytes(data []byte, destDir string) error { + f, err := os.CreateTemp("", "please_ts_dep_*.tgz") + if err != nil { + return err + } + defer os.Remove(f.Name()) + if _, err := f.Write(data); err != nil { + f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + return extractTarGz(f.Name(), destDir) +} + +// versionSatisfies reports whether a version meets a dependency specifier. Specifiers that +// are not version ranges (a dist-tag, say) cannot be judged here and count as satisfied. +func versionSatisfies(version, spec string) bool { + v, err := semver.Parse(version) + if err != nil { + return true + } + r, err := semver.ParseRange(spec) + if err != nil { + return true + } + return r.Satisfies(v) +} + +func anySatisfies(versions []string, spec string) bool { + for _, v := range versions { + if versionSatisfies(v, spec) { + return true + } + } + return false +} + +func sortedKeys(m map[string]string) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +// pendingDep is a dependency waiting to be resolved. +type pendingDep struct { + name, spec, requiredBy string + peer bool +} + +// resolveTransitiveDependencies fetches the dependencies of a ts_module into .deps/, +// one directory per package (the layout holds a single version of each). +// +// Resolution is strict: a specifier that no registry version satisfies is an error, and so +// is a download that does not match the registry's integrity data, a tarball that holds a +// different package than asked for, and two dependents that need versions of one package +// that cannot both be the one in .deps (ts_npm_module keeps both). Peer dependencies are +// fetched too, as a default for consumers that do not provide them, but only as a best +// effort: a peer that cannot be resolved or conflicts is a warning. +func resolveTransitiveDependencies(outDir string, rootPkg PackageJSON, r *resolver, depsMap map[string]string) error { + r.pinToRoot(rootPkg.Name, rootPkg.Version) + resolved := map[string]string{} // package -> version in .deps + var queue []pendingDep + for _, name := range sortedKeys(rootPkg.Dependencies) { + queue = append(queue, pendingDep{name, rootPkg.Dependencies[name], rootPkg.Name, false}) + } + for _, name := range sortedKeys(rootPkg.PeerDependencies) { + queue = append(queue, pendingDep{name, rootPkg.PeerDependencies[name], rootPkg.Name, true}) + } + + for len(queue) > 0 { + d := queue[0] + queue = queue[1:] + if d.name == rootPkg.Name { + continue + } + + if have, ok := resolved[d.name]; ok { + if versionSatisfies(have, d.spec) { + continue + } + msg := fmt.Sprintf("%s requires %s@%s, but %s@%s is already in .deps; a ts_module holds one version of a package (use ts_npm_module, which keeps several)", + d.requiredBy, d.name, d.spec, d.name, have) + if d.peer { + fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) + continue + } + return fmt.Errorf("conflicting versions: %s", msg) + } + + destDir := filepath.Join(outDir, ".deps", d.name) + if _, err := os.Stat(filepath.Join(destDir, "package.json")); err == nil { + existing := readPackageJSON(destDir) + resolved[d.name] = existing.Version + if entry := findLocalEntry(destDir); entry != "" { + depsMap[d.name] = cleanRelativePath(filepath.Join(".deps", d.name, entry)) + } + continue + } + + info, data, err := r.fetch(d.name, d.spec) + if err != nil { + if d.peer { + fmt.Fprintf(os.Stderr, "Warning: peer dependency %s@%s of %s was not resolved: %v\n", d.name, d.spec, d.requiredBy, err) + continue + } + return fmt.Errorf("resolving %s@%s (required by %s): %w", d.name, d.spec, d.requiredBy, err) + } + if err := os.MkdirAll(destDir, 0755); err != nil { + return err + } + if err := extractTarballBytes(data, destDir); err != nil { + return fmt.Errorf("extracting %s@%s: %w", d.name, info.Version, err) + } + pkg := readPackageJSON(destDir) + if pkg.Name != "" && pkg.Name != d.name { + return fmt.Errorf("the tarball of %s@%s holds package %q", d.name, info.Version, pkg.Name) + } + if pkg.Version == "" { + pkg.Version = info.Version + } + ensureCommonJSType(destDir) + resolved[d.name] = pkg.Version + + if entry := determineEntry(destDir, pkg); entry != "" { + depsMap[d.name] = cleanRelativePath(filepath.Join(".deps", d.name, entry)) + } + for _, name := range sortedKeys(pkg.Dependencies) { + queue = append(queue, pendingDep{name, pkg.Dependencies[name], d.name + "@" + pkg.Version, false}) + } + } + return nil +} diff --git a/tools/please_ts/unpack/resolve_test.go b/tools/please_ts/unpack/resolve_test.go new file mode 100644 index 0000000..ceb1652 --- /dev/null +++ b/tools/please_ts/unpack/resolve_test.go @@ -0,0 +1,516 @@ +package unpack + +import ( + "crypto/sha512" + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + + "tools/please_ts/semver" +) + +// mockVersion describes one published version. +type mockVersion struct { + deps, peers, optional map[string]string + published string // RFC 3339, for as-of tests +} + +// mockRegistry is an npm registry that serves real tarballs with integrity data. +type mockRegistry struct { + t *testing.T + srv *httptest.Server + mu sync.Mutex + versions map[string]map[string]mockVersion // package -> version -> description + tarballs map[string][]byte // "name@version" -> tarball + hits map[string]int // request path -> count + corrupt map[string]bool // "name@version": serve other bytes than the integrity covers + noIntegrity map[string]bool // "name@version": publish no integrity data + wrongName map[string]string // "name@version": the package.json name to put in the tarball +} + +func newMockRegistry(t *testing.T) *mockRegistry { + m := &mockRegistry{ + t: t, versions: map[string]map[string]mockVersion{}, tarballs: map[string][]byte{}, + hits: map[string]int{}, corrupt: map[string]bool{}, noIntegrity: map[string]bool{}, wrongName: map[string]string{}, + } + m.srv = httptest.NewServer(http.HandlerFunc(m.serve)) + t.Cleanup(m.srv.Close) + return m +} + +func (m *mockRegistry) URL() string { return m.srv.URL } + +func (m *mockRegistry) add(name, version string, v mockVersion) { + if m.versions[name] == nil { + m.versions[name] = map[string]mockVersion{} + } + m.versions[name][version] = v + manifestName := name + if w, ok := m.wrongName[name+"@"+version]; ok { + manifestName = w + } + manifest := map[string]any{"name": manifestName, "version": version, "main": "index.js"} + for k, deps := range map[string]map[string]string{"dependencies": v.deps, "peerDependencies": v.peers, "optionalDependencies": v.optional} { + if len(deps) > 0 { + manifest[k] = deps + } + } + pkgJSON, _ := json.Marshal(manifest) + m.tarballs[name+"@"+version] = createTestTarball(m.t, map[string]string{ + "package.json": string(pkgJSON), "index.js": "module.exports = '" + name + "@" + version + "';", + }) +} + +func (m *mockRegistry) hitsFor(substr string) int { + m.mu.Lock() + defer m.mu.Unlock() + n := 0 + for path, c := range m.hits { + if strings.Contains(path, substr) { + n += c + } + } + return n +} + +func (m *mockRegistry) serve(w http.ResponseWriter, r *http.Request) { + path := strings.TrimPrefix(r.URL.Path, "/") + m.mu.Lock() + m.hits[path]++ + m.mu.Unlock() + + if name, file, ok := strings.Cut(path, "/-/"); ok { + base := name[strings.LastIndex(name, "/")+1:] + version := strings.TrimSuffix(strings.TrimPrefix(file, base+"-"), ".tgz") + data, ok := m.tarballs[name+"@"+version] + if !ok { + http.NotFound(w, r) + return + } + if m.corrupt[name+"@"+version] { + data = append([]byte("corrupt"), data...) + } + _, _ = w.Write(data) + return + } + + versions, ok := m.versions[path] + if !ok { + http.NotFound(w, r) + return + } + all := make([]string, 0, len(versions)) + for v := range versions { + all = append(all, v) + } + sort.Slice(all, func(i, j int) bool { + a, _ := semver.Parse(all[i]) + b, _ := semver.Parse(all[j]) + return semver.Compare(a, b) < 0 + }) + entries := map[string]any{} + times := map[string]string{} + for _, v := range all { + sum := sha512.Sum512(m.tarballs[path+"@"+v]) // integrity of the genuine tarball + dist := map[string]string{"tarball": fmt.Sprintf("%s/%s/-/%s-%s.tgz", m.srv.URL, path, path[strings.LastIndex(path, "/")+1:], v)} + if !m.noIntegrity[path+"@"+v] { + dist["integrity"] = "sha512-" + base64.StdEncoding.EncodeToString(sum[:]) + } + entries[v] = map[string]any{"version": v, "dist": dist} + if versions[v].published != "" { + times[v] = versions[v].published + } + } + doc := map[string]any{"name": path, "dist-tags": map[string]string{"latest": all[len(all)-1]}, "versions": entries, "time": times} + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(doc) +} + +// rootTarball writes a root package's tarball and returns its path. +func rootTarball(t *testing.T, dir, name, version string, v mockVersion) string { + t.Helper() + manifest := map[string]any{"name": name, "version": version, "main": "index.js"} + for k, deps := range map[string]map[string]string{"dependencies": v.deps, "peerDependencies": v.peers, "optionalDependencies": v.optional} { + if len(deps) > 0 { + manifest[k] = deps + } + } + pkgJSON, _ := json.Marshal(manifest) + path := filepath.Join(dir, "root.tgz") + if err := os.WriteFile(path, createTestTarball(t, map[string]string{"package.json": string(pkgJSON), "index.js": ""}), 0644); err != nil { + t.Fatal(err) + } + return path +} + +func deps(kv ...string) map[string]string { + m := map[string]string{} + for i := 0; i+1 < len(kv); i += 2 { + m[kv[i]] = kv[i+1] + } + return m +} + +func versionOf(t *testing.T, dir string) string { + t.Helper() + data, err := os.ReadFile(filepath.Join(dir, "package.json")) + if err != nil { + t.Fatal(err) + } + var p struct{ Version string } + if err := json.Unmarshal(data, &p); err != nil { + t.Fatal(err) + } + return p.Version +} + +// ---- ts_module: the flat .deps layout ---- + +func runModule(t *testing.T, m *mockRegistry, root mockVersion) (string, error) { + t.Helper() + work := t.TempDir() + out := filepath.Join(work, "out") + err := Run(Options{ + Tarball: rootTarball(t, work, "root-pkg", "1.0.0", root), Out: out, Name: "root-pkg", + ResolveTransitive: true, Registry: m.URL(), + }) + return out, err +} + +func TestModuleResolvesAChainOfDependencies(t *testing.T) { + m := newMockRegistry(t) + m.add("a", "1.2.0", mockVersion{deps: deps("b", "^2.0.0")}) + m.add("b", "2.1.0", mockVersion{}) + out, err := runModule(t, m, mockVersion{deps: deps("a", "^1.0.0")}) + if err != nil { + t.Fatal(err) + } + if versionOf(t, filepath.Join(out, ".deps", "a")) != "1.2.0" || versionOf(t, filepath.Join(out, ".deps", "b")) != "2.1.0" { + t.Error("the whole chain should be in .deps at the resolved versions") + } +} + +func TestModuleResolutionIsStrict(t *testing.T) { + m := newMockRegistry(t) + m.add("a", "1.0.0", mockVersion{}) + _, err := runModule(t, m, mockVersion{deps: deps("a", "^9.0.0")}) + if err == nil || !strings.Contains(err.Error(), "no version of a satisfies") { + t.Errorf("a range nothing satisfies must fail, not fall back to another version; got %v", err) + } +} + +func TestModuleResolutionFailureIsAnErrorNotAWarning(t *testing.T) { + m := newMockRegistry(t) // the registry knows no package at all + _, err := runModule(t, m, mockVersion{deps: deps("missing", "^1.0.0")}) + if err == nil || !strings.Contains(err.Error(), "missing@^1.0.0") || !strings.Contains(err.Error(), "root-pkg") { + t.Errorf("an unresolvable dependency must fail and name what needed it, got %v", err) + } +} + +func TestModuleVerifiesIntegrity(t *testing.T) { + m := newMockRegistry(t) + m.add("a", "1.0.0", mockVersion{}) + m.corrupt["a@1.0.0"] = true + if _, err := runModule(t, m, mockVersion{deps: deps("a", "^1.0.0")}); err == nil || !strings.Contains(err.Error(), "does not match") { + t.Errorf("a tarball that does not match the registry's integrity must be refused, got %v", err) + } + + m2 := newMockRegistry(t) + m2.add("a", "1.0.0", mockVersion{}) + m2.noIntegrity["a@1.0.0"] = true + if _, err := runModule(t, m2, mockVersion{deps: deps("a", "^1.0.0")}); err == nil || !strings.Contains(err.Error(), "no integrity") { + t.Errorf("a version without integrity data cannot be verified, got %v", err) + } +} + +func TestModuleRejectsATarballOfAnotherPackage(t *testing.T) { + m := newMockRegistry(t) + m.wrongName["a@1.0.0"] = "evil" + m.add("a", "1.0.0", mockVersion{}) + if _, err := runModule(t, m, mockVersion{deps: deps("a", "^1.0.0")}); err == nil || !strings.Contains(err.Error(), `holds package "evil"`) { + t.Errorf("got %v", err) + } +} + +func TestModuleReportsConflictingVersionsInsteadOfPickingTheFirst(t *testing.T) { + m := newMockRegistry(t) + m.add("a", "1.0.0", mockVersion{deps: deps("c", "^1.0.0")}) + m.add("b", "1.0.0", mockVersion{deps: deps("c", "^2.0.0")}) + m.add("c", "1.4.0", mockVersion{}) + m.add("c", "2.0.0", mockVersion{}) + _, err := runModule(t, m, mockVersion{deps: deps("a", "^1.0.0", "b", "^1.0.0")}) + if err == nil || !strings.Contains(err.Error(), "conflicting versions") || !strings.Contains(err.Error(), "ts_npm_module") { + t.Errorf("incompatible needs for one package must be reported, got %v", err) + } +} + +func TestModuleSharesACompatibleDependency(t *testing.T) { + m := newMockRegistry(t) + m.add("a", "1.0.0", mockVersion{deps: deps("c", "^1.0.0")}) + m.add("b", "1.0.0", mockVersion{deps: deps("c", "^1.2.0")}) + m.add("c", "1.5.0", mockVersion{}) + out, err := runModule(t, m, mockVersion{deps: deps("a", "^1.0.0", "b", "^1.0.0")}) + if err != nil { + t.Fatalf("two compatible ranges are not a conflict: %v", err) + } + if versionOf(t, filepath.Join(out, ".deps", "c")) != "1.5.0" { + t.Error("c should be resolved once, to a version both accept") + } +} + +func TestModulePeerDependenciesAreBestEffort(t *testing.T) { + m := newMockRegistry(t) + m.add("a", "1.0.0", mockVersion{}) + out, err := runModule(t, m, mockVersion{deps: deps("a", "^1.0.0"), peers: deps("not-published", "^1.0.0")}) + if err != nil { + t.Fatalf("a peer that cannot be resolved is a warning, got %v", err) + } + if _, err := os.Stat(filepath.Join(out, ".deps", "a", "package.json")); err != nil { + t.Error("the real dependency must still be resolved") + } +} + +// ---- ts_npm_module: slices ---- + +func runNpmCache(t *testing.T, m *mockRegistry, root mockVersion, resolve bool) (string, error) { + t.Helper() + work := t.TempDir() + chdir(t, work) + out := filepath.Join(work, "out") + err := Run(Options{ + Archive: rootTarball(t, work, "root-pkg", "1.0.0", root), Out: out, Name: "root-pkg", NpmCache: true, + ResolveTransitive: resolve, Registry: m.URL(), + }) + return out, err +} + +func cacheVersions(t *testing.T, out, name string) []string { + t.Helper() + data, err := os.ReadFile(filepath.Join(out, "npm/registry.npmjs.org", name, "registry.json")) + if err != nil { + t.Fatalf("no packument for %s: %v", name, err) + } + var p struct { + Versions map[string]json.RawMessage `json:"versions"` + } + if err := json.Unmarshal(data, &p); err != nil { + t.Fatal(err) + } + var vs []string + for v := range p.Versions { + if _, err := os.Stat(filepath.Join(out, "npm/registry.npmjs.org", name, v, "index.js")); err != nil { + t.Errorf("%s@%s is in the packument but not extracted: %v", name, v, err) + } + vs = append(vs, v) + } + sort.Strings(vs) + return vs +} + +func TestNpmCacheResolvesAndBundlesTransitiveDependencies(t *testing.T) { + m := newMockRegistry(t) + m.add("a", "1.2.0", mockVersion{deps: deps("b", "^2.0.0")}) + m.add("b", "2.1.0", mockVersion{}) + out, err := runNpmCache(t, m, mockVersion{deps: deps("a", "^1.0.0")}, true) + if err != nil { + t.Fatal(err) + } + for name, want := range map[string]string{"root-pkg": "1.0.0", "a": "1.2.0", "b": "2.1.0"} { + if got := cacheVersions(t, out, name); len(got) != 1 || got[0] != want { + t.Errorf("%s versions = %v, want [%s]", name, got, want) + } + } +} + +func TestNpmCacheKeepsConflictingVersionsSideBySide(t *testing.T) { + m := newMockRegistry(t) + m.add("a", "1.0.0", mockVersion{deps: deps("c", "^1.0.0")}) + m.add("b", "1.0.0", mockVersion{deps: deps("c", "^2.0.0")}) + m.add("c", "1.4.0", mockVersion{}) + m.add("c", "2.0.0", mockVersion{}) + out, err := runNpmCache(t, m, mockVersion{deps: deps("a", "^1.0.0", "b", "^1.0.0")}, true) + if err != nil { + t.Fatalf("unlike ts_module, a slice can hold both: %v", err) + } + if got := cacheVersions(t, out, "c"); strings.Join(got, ",") != "1.4.0,2.0.0" { + t.Errorf("c versions = %v, want 1.4.0 and 2.0.0 (Deno picks per dependent)", got) + } +} + +func TestNpmCacheReusesAVersionThatAlreadySatisfies(t *testing.T) { + m := newMockRegistry(t) + m.add("a", "1.0.0", mockVersion{deps: deps("c", "^1.0.0")}) + m.add("b", "1.0.0", mockVersion{deps: deps("c", "^1.2.0")}) + m.add("c", "1.2.0", mockVersion{}) + m.add("c", "1.5.0", mockVersion{}) + out, err := runNpmCache(t, m, mockVersion{deps: deps("a", "^1.0.0", "b", "^1.0.0")}, true) + if err != nil { + t.Fatal(err) + } + if got := cacheVersions(t, out, "c"); len(got) != 1 || got[0] != "1.5.0" { + t.Errorf("c should be resolved once, got %v", got) + } + if n := m.hitsFor("c-1."); n != 1 { + t.Errorf("c was downloaded %d times, want 1", n) + } +} + +func TestNpmCacheUsesStagedSlicesBeforeTheRegistry(t *testing.T) { + m := newMockRegistry(t) + m.add("b", "1.0.0", mockVersion{}) + work := t.TempDir() + chdir(t, work) + stageSliceWithPackage(t, work, "a", "1.0.0") // explicitly provided; the registry has no "a" + out := filepath.Join(work, "out") + err := Run(Options{ + Archive: rootTarball(t, work, "root-pkg", "1.0.0", mockVersion{deps: deps("a", "^1.0.0", "b", "^1.0.0")}), + Out: out, Name: "root-pkg", NpmCache: true, ResolveTransitive: true, Registry: m.URL(), + }) + if err != nil { + t.Fatalf("a declared dependency must not be resolved again: %v", err) + } + if m.hitsFor("a") != 0 { + t.Error("the registry must not be asked for a package a slice provides") + } + if len(cacheVersions(t, out, "a")) != 1 || len(cacheVersions(t, out, "b")) != 1 { + t.Error("both the staged and the resolved dependency must end up in the slice") + } +} + +func TestNpmCacheDoesNotResolveOptionalOrPeerDependencies(t *testing.T) { + m := newMockRegistry(t) // knows nothing: any request would fail + out, err := runNpmCache(t, m, mockVersion{optional: deps("fsevents", "^2.0.0"), peers: deps("react", "^18.0.0")}, true) + if err != nil { + t.Fatalf("optional and peer dependencies are not fetched: %v", err) + } + if got := cacheVersions(t, out, "root-pkg"); len(got) != 1 { + t.Errorf("root-pkg = %v", got) + } + if n := m.hitsFor(""); n != 0 { + t.Errorf("the registry was contacted %d times, want 0", n) + } +} + +func TestNpmCacheResolutionErrors(t *testing.T) { + t.Run("no version satisfies", func(t *testing.T) { + m := newMockRegistry(t) + m.add("a", "1.0.0", mockVersion{}) + _, err := runNpmCache(t, m, mockVersion{deps: deps("a", "^9.0.0")}, true) + if err == nil || !strings.Contains(err.Error(), "no version of a satisfies") || !strings.Contains(err.Error(), "root-pkg@1.0.0") { + t.Errorf("got %v", err) + } + }) + t.Run("integrity mismatch", func(t *testing.T) { + m := newMockRegistry(t) + m.add("a", "1.0.0", mockVersion{}) + m.corrupt["a@1.0.0"] = true + if _, err := runNpmCache(t, m, mockVersion{deps: deps("a", "^1.0.0")}, true); err == nil || !strings.Contains(err.Error(), "does not match") { + t.Errorf("got %v", err) + } + }) + t.Run("a transitive dependency fails", func(t *testing.T) { + m := newMockRegistry(t) + m.add("a", "1.0.0", mockVersion{deps: deps("gone", "^1.0.0")}) + _, err := runNpmCache(t, m, mockVersion{deps: deps("a", "^1.0.0")}, true) + if err == nil || !strings.Contains(err.Error(), "gone@^1.0.0") || !strings.Contains(err.Error(), "a@1.0.0") { + t.Errorf("the error must name the package and who required it, got %v", err) + } + }) +} + +func TestNpmCacheWithoutResolveStillRequiresDeclaredDependencies(t *testing.T) { + m := newMockRegistry(t) + m.add("a", "1.0.0", mockVersion{}) + _, err := runNpmCache(t, m, mockVersion{deps: deps("a", "^1.0.0")}, false) + if err == nil || !strings.Contains(err.Error(), "resolve_transitive") { + t.Errorf("strict mode must fail and point at resolve_transitive, got %v", err) + } + if m.hitsFor("") != 0 { + t.Error("strict mode must not contact the registry") + } +} + +// ---- the as-of date defaults to the day the root version was published ---- + +// publishedAround registers the root and a dependency whose versions straddle the root's +// publish day (2021-03-10). +func publishedAround(t *testing.T) *mockRegistry { + m := newMockRegistry(t) + m.add("root-pkg", "1.0.0", mockVersion{published: "2021-03-10T15:00:00.000Z"}) + m.add("a", "1.0.0", mockVersion{published: "2021-03-01T09:00:00.000Z"}) + m.add("a", "1.1.0", mockVersion{published: "2021-03-10T23:30:00.000Z"}) // later that day: still counts + m.add("a", "1.2.0", mockVersion{published: "2021-03-11T00:10:00.000Z"}) // the next day: too new + return m +} + +func TestAutomaticPinUsesTheDayTheRootWasPublished(t *testing.T) { + root := mockVersion{deps: deps("a", "^1.0.0")} + + m := publishedAround(t) + out, err := runNpmCache(t, m, root, true) + if err != nil { + t.Fatal(err) + } + if got := cacheVersions(t, out, "a"); len(got) != 1 || got[0] != "1.1.0" { + t.Errorf("npm cache: a = %v, want 1.1.0 (the newest published by the end of the root's day)", got) + } + + m = publishedAround(t) + modOut, err := runModule(t, m, root) + if err != nil { + t.Fatal(err) + } + if got := versionOf(t, filepath.Join(modOut, ".deps", "a")); got != "1.1.0" { + t.Errorf("ts_module: a = %s, want 1.1.0", got) + } +} + +func TestPinningIsSkippedWhenTheRegistryDoesNotKnowTheRoot(t *testing.T) { + m := newMockRegistry(t) // root-pkg is not published here, say a private tarball + m.add("a", "1.0.0", mockVersion{published: "2021-03-01T09:00:00.000Z"}) + m.add("a", "1.2.0", mockVersion{published: "2024-03-11T00:10:00.000Z"}) + out, err := runNpmCache(t, m, mockVersion{deps: deps("a", "^1.0.0")}, true) + if err != nil { + t.Fatalf("an unknown root must not fail the build: %v", err) + } + if got := cacheVersions(t, out, "a"); len(got) != 1 || got[0] != "1.2.0" { + t.Errorf("a = %v, want the newest when there is no date to pin to", got) + } +} + +func TestPinningIsSkippedWhenTheRootHasNoPublishTime(t *testing.T) { + m := newMockRegistry(t) + m.add("root-pkg", "1.0.0", mockVersion{}) // no time recorded + m.add("a", "1.0.0", mockVersion{published: "2021-03-01T09:00:00.000Z"}) + m.add("a", "1.2.0", mockVersion{published: "2024-03-11T00:10:00.000Z"}) + out, err := runNpmCache(t, m, mockVersion{deps: deps("a", "^1.0.0")}, true) + if err != nil { + t.Fatal(err) + } + if got := cacheVersions(t, out, "a"); len(got) != 1 || got[0] != "1.2.0" { + t.Errorf("a = %v", got) + } +} + +func TestResolverPinsOnlyOnce(t *testing.T) { + m := publishedAround(t) + r := newResolver(m.URL()) + r.pinToRoot("root-pkg", "1.0.0") + first := r.asOf + if first.IsZero() || first.Format("2006-01-02T15:04:05") != "2021-03-10T23:59:59" { + t.Fatalf("asOf = %v, want the end of 2021-03-10", first) + } + m.add("root-pkg", "2.0.0", mockVersion{published: "2024-01-01T00:00:00.000Z"}) + r.pinToRoot("root-pkg", "2.0.0") + if !r.asOf.Equal(first) { + t.Error("a resolver keeps the date it was pinned to") + } +} diff --git a/tools/please_ts/unpack/unpack.go b/tools/please_ts/unpack/unpack.go index 9782b67..ae55bba 100644 --- a/tools/please_ts/unpack/unpack.go +++ b/tools/please_ts/unpack/unpack.go @@ -7,13 +7,9 @@ import ( "encoding/json" "fmt" "io" - "net/http" - "net/url" "os" "path/filepath" - "strconv" "strings" - "time" "tools/please_ts/importmap" ) @@ -28,7 +24,7 @@ type Options struct { Symlink string // optional symlink name for extracted binary ResolveTransitive bool // recursively resolve and unpack transitive dependencies Registry string // npm registry URL (default https://registry.npmjs.org) - NpmCache bool // build a Deno npm cache slice from the tarball, offline + NpmCache bool // build a Deno npm cache slice from the tarball } // Validate checks whether the required options are provided. @@ -71,22 +67,6 @@ type PackageJSON struct { PeerDependencies map[string]string `json:"peerDependencies"` } -type npmVersionData struct { - Version string `json:"version"` - Dependencies map[string]string `json:"dependencies"` - Dist struct { - Tarball string `json:"tarball"` - } `json:"dist"` -} - -type npmManifest struct { - Name string `json:"name"` - DistTags struct { - Latest string `json:"latest"` - } `json:"dist-tags"` - Versions map[string]npmVersionData `json:"versions"` -} - // Run executes the unpacking operation based on provided Options. func Run(opts Options) error { if err := opts.Validate(); err != nil { @@ -118,8 +98,8 @@ func unpackModule(opts Options) error { peerDepsMap := copyStringMap(pkg.PeerDependencies) if opts.ResolveTransitive && hasDependencies(pkg) { - if err := resolveTransitiveDependencies(opts.Out, pkg, opts.RegistryURL(), depsMap); err != nil { - fmt.Fprintf(os.Stderr, "Warning: transitive dependency resolution failed: %v\n", err) + if err := resolveTransitiveDependencies(opts.Out, pkg, newResolver(opts.RegistryURL()), depsMap); err != nil { + return fmt.Errorf("failed resolving the dependencies of %s: %w", pkg.Name, err) } } @@ -257,152 +237,6 @@ func findLocalEntry(pkgDir string) string { return "" } -func resolveTransitiveDependencies(outDir string, rootPkg PackageJSON, registry string, depsMap map[string]string) error { - client := &http.Client{Timeout: 30 * time.Second} - visited := make(map[string]bool) - visited[rootPkg.Name] = true - - queue := make(map[string]string) - for k, v := range rootPkg.Dependencies { - queue[k] = v - } - for k, v := range rootPkg.PeerDependencies { - queue[k] = v - } - - for len(queue) > 0 { - var currentPkg, currentConstraint string - for k, v := range queue { - currentPkg = k - currentConstraint = v - delete(queue, k) - break - } - - if visited[currentPkg] { - continue - } - visited[currentPkg] = true - - destDir := filepath.Join(outDir, ".deps", currentPkg) - if _, err := os.Stat(filepath.Join(destDir, "package.json")); err == nil { - entry := findLocalEntry(destDir) - if entry != "" { - depsMap[currentPkg] = cleanRelativePath(filepath.Join(".deps", currentPkg, entry)) - } - continue - } - - pkgInfo, err := fetchAndExtractPackage(client, registry, currentPkg, currentConstraint, destDir) - if err != nil { - return fmt.Errorf("failed fetching package %s: %w", currentPkg, err) - } - - entry := determineEntry(destDir, *pkgInfo) - if entry != "" { - depsMap[currentPkg] = cleanRelativePath(filepath.Join(".deps", currentPkg, entry)) - } - - for nextDep, nextVer := range pkgInfo.Dependencies { - if !visited[nextDep] { - queue[nextDep] = nextVer - } - } - } - - return nil -} - -func fetchAndExtractPackage(client *http.Client, registry, pkgName, constraint, destDir string) (*PackageJSON, error) { - manifest, err := fetchPackageManifest(client, registry, pkgName) - if err != nil { - return nil, err - } - - resolvedVer := resolveVersion(manifest.DistTags.Latest, manifest.Versions, constraint) - verData, ok := manifest.Versions[resolvedVer] - if !ok { - return nil, fmt.Errorf("version %s not found in manifest for %s", resolvedVer, pkgName) - } - - tarballURL := verData.Dist.Tarball - if tarballURL == "" { - base := pkgName - if strings.Contains(base, "/") { - base = base[strings.LastIndex(base, "/")+1:] - } - tarballURL = fmt.Sprintf("%s/%s/-/%s-%s.tgz", strings.TrimSuffix(registry, "/"), pkgName, base, resolvedVer) - } - - tarballFile, err := os.CreateTemp("", "please_ts_dep_*.tgz") - if err != nil { - return nil, err - } - defer os.Remove(tarballFile.Name()) - defer tarballFile.Close() - - if err := downloadTarball(client, tarballURL, tarballFile); err != nil { - return nil, err - } - _ = tarballFile.Close() - - if err := os.MkdirAll(destDir, 0755); err != nil { - return nil, err - } - - if err := extractTarGz(tarballFile.Name(), destDir); err != nil { - return nil, fmt.Errorf("failed extracting %s: %w", tarballFile.Name(), err) - } - - pkg := readPackageJSON(destDir) - if pkg.Version == "" { - pkg.Version = resolvedVer - } - ensureCommonJSType(destDir) - - return &pkg, nil -} - -func fetchPackageManifest(client *http.Client, registry, pkgName string) (*npmManifest, error) { - manifestURL := strings.TrimSuffix(registry, "/") + "/" + url.PathEscape(pkgName) - req, err := http.NewRequest("GET", manifestURL, nil) - if err != nil { - return nil, err - } - req.Header.Set("Accept", "application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8, */*") - - resp, err := client.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("registry returned status %d for %s", resp.StatusCode, manifestURL) - } - - var manifest npmManifest - if err := json.NewDecoder(resp.Body).Decode(&manifest); err != nil { - return nil, fmt.Errorf("failed decoding manifest for %s: %w", pkgName, err) - } - return &manifest, nil -} - -func downloadTarball(client *http.Client, tarballURL string, destFile *os.File) error { - dlResp, err := client.Get(tarballURL) - if err != nil { - return fmt.Errorf("failed downloading tarball from %s: %w", tarballURL, err) - } - defer dlResp.Body.Close() - - if dlResp.StatusCode != http.StatusOK { - return fmt.Errorf("failed downloading tarball from %s: status %d", tarballURL, dlResp.StatusCode) - } - - _, err = io.Copy(destFile, dlResp.Body) - return err -} - func ensureCommonJSType(dir string) { pkgJSONPath := filepath.Join(dir, "package.json") data, err := os.ReadFile(pkgJSONPath) @@ -421,91 +255,6 @@ func ensureCommonJSType(dir string) { } } -func resolveVersion(latest string, versions map[string]npmVersionData, constraint string) string { - clean := strings.TrimSpace(constraint) - if clean == "" || clean == "*" || clean == "latest" { - if latest != "" { - return latest - } - } - if _, ok := versions[clean]; ok { - return clean - } - - prefix := "" - if strings.HasPrefix(clean, "^") || strings.HasPrefix(clean, "~") { - prefix = clean[:1] - clean = clean[1:] - } else if strings.HasPrefix(clean, ">=") { - prefix = ">=" - clean = strings.TrimSpace(clean[2:]) - } - - targetParts := parseSemver(clean) - var bestMatch string - var bestParts [3]int - - for ver := range versions { - parts := parseSemver(ver) - if matchesConstraint(parts, targetParts, prefix) { - if bestMatch == "" || compareSemver(parts, bestParts) > 0 { - bestMatch = ver - bestParts = parts - } - } - } - - if bestMatch != "" { - return bestMatch - } - if latest != "" { - return latest - } - for ver := range versions { - return ver - } - return constraint -} - -func matchesConstraint(parts, targetParts [3]int, prefix string) bool { - switch prefix { - case "^": - return parts[0] == targetParts[0] && compareSemver(parts, targetParts) >= 0 - case "~": - return parts[0] == targetParts[0] && parts[1] == targetParts[1] && compareSemver(parts, targetParts) >= 0 - case ">=": - return compareSemver(parts, targetParts) >= 0 - default: - return compareSemver(parts, targetParts) == 0 - } -} - -func parseSemver(v string) [3]int { - v = strings.TrimPrefix(v, "v") - if idx := strings.IndexAny(v, "-+"); idx != -1 { - v = v[:idx] - } - parts := strings.Split(v, ".") - var res [3]int - for i := 0; i < len(parts) && i < 3; i++ { - n, _ := strconv.Atoi(parts[i]) - res[i] = n - } - return res -} - -func compareSemver(a, b [3]int) int { - for i := 0; i < 3; i++ { - if a[i] > b[i] { - return 1 - } - if a[i] < b[i] { - return -1 - } - } - return 0 -} - func extractArchive(archivePath, destDir string) error { if strings.HasSuffix(archivePath, ".zip") { return extractZip(archivePath, destDir) diff --git a/tools/please_ts/unpack/unpack_test.go b/tools/please_ts/unpack/unpack_test.go index 88f7831..641206b 100644 --- a/tools/please_ts/unpack/unpack_test.go +++ b/tools/please_ts/unpack/unpack_test.go @@ -5,6 +5,8 @@ import ( "archive/zip" "bytes" "compress/gzip" + "crypto/sha512" + "encoding/base64" "encoding/json" "fmt" "net/http" @@ -46,85 +48,6 @@ func createTestTarball(t *testing.T, files map[string]string) []byte { return buf.Bytes() } -func TestSemverParseMatrix(t *testing.T) { - tests := []struct { - input string - want [3]int - }{ - {"1.2.3", [3]int{1, 2, 3}}, - {"v2.10.4", [3]int{2, 10, 4}}, - {"0.4.0-alpha.1", [3]int{0, 4, 0}}, - {"1.0.0+build.1", [3]int{1, 0, 0}}, - {"3", [3]int{3, 0, 0}}, - {"1.5", [3]int{1, 5, 0}}, - } - - for _, tt := range tests { - t.Run(tt.input, func(t *testing.T) { - got := parseSemver(tt.input) - if got != tt.want { - t.Errorf("parseSemver(%q) = %v, want %v", tt.input, got, tt.want) - } - }) - } -} - -func TestSemverCompareMatrix(t *testing.T) { - tests := []struct { - a [3]int - b [3]int - want int - }{ - {[3]int{1, 2, 3}, [3]int{1, 2, 3}, 0}, - {[3]int{2, 0, 0}, [3]int{1, 9, 9}, 1}, - {[3]int{1, 2, 0}, [3]int{1, 3, 0}, -1}, - {[3]int{1, 2, 4}, [3]int{1, 2, 3}, 1}, - {[3]int{1, 2, 3}, [3]int{1, 2, 4}, -1}, - } - - for _, tt := range tests { - t.Run(fmt.Sprintf("%v_vs_%v", tt.a, tt.b), func(t *testing.T) { - got := compareSemver(tt.a, tt.b) - if got != tt.want { - t.Errorf("compareSemver(%v, %v) = %d, want %d", tt.a, tt.b, got, tt.want) - } - }) - } -} - -func TestResolveVersionMatrix(t *testing.T) { - versions := map[string]npmVersionData{ - "1.0.0": {Version: "1.0.0"}, - "1.1.0": {Version: "1.1.0"}, - "1.2.3": {Version: "1.2.3"}, - "2.0.0": {Version: "2.0.0"}, - "2.1.0": {Version: "2.1.0"}, - } - - tests := []struct { - constraint string - latest string - want string - }{ - {"^1.0.0", "2.1.0", "1.2.3"}, - {"~1.1.0", "2.1.0", "1.1.0"}, - {">=2.0.0", "2.1.0", "2.1.0"}, - {"1.0.0", "2.1.0", "1.0.0"}, - {"latest", "2.1.0", "2.1.0"}, - {"*", "2.1.0", "2.1.0"}, - {"", "2.1.0", "2.1.0"}, - } - - for _, tt := range tests { - t.Run(tt.constraint, func(t *testing.T) { - got := resolveVersion(tt.latest, versions, tt.constraint) - if got != tt.want { - t.Errorf("resolveVersion(latest=%q, constraint=%q) = %q, want %q", tt.latest, tt.constraint, got, tt.want) - } - }) - } -} - func TestCleanRelativePathMatrix(t *testing.T) { tests := []struct { input string @@ -323,6 +246,8 @@ func TestUnpackWithMockRegistryMatrix(t *testing.T) { "index.js": `module.exports = { helper: true };`, }) + helperSum := sha512.Sum512(helperTarball) + helperIntegrity := "sha512-" + base64.StdEncoding.EncodeToString(helperSum[:]) var server *httptest.Server server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if strings.HasSuffix(r.URL.Path, "/helper-pkg") { @@ -332,10 +257,10 @@ func TestUnpackWithMockRegistryMatrix(t *testing.T) { "versions": { "1.0.0": { "version": "1.0.0", - "dist": { "tarball": "%s/helper-pkg/-/helper-pkg-1.0.0.tgz" } + "dist": { "tarball": "%s/helper-pkg/-/helper-pkg-1.0.0.tgz", "integrity": "%s" } } } - }`, server.URL) + }`, server.URL, helperIntegrity) w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(manifest)) return