diff --git a/CHANGELOG.md b/CHANGELOG.md index 4609941..40b1d92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,45 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- Experimental `ts_npm_module` rule: provides an npm package to Deno's own + `npm:` resolution. The sha256-pinned tarball is extracted by the build into a + slice of a Deno npm cache that targets merge into their `DENO_DIR`; no + `node_modules`, no lockfile. CommonJS packages, subpath imports and packages + with only an `exports` map work as Deno resolves them. Dependencies resolve + automatically (`resolve_transitive`, on by default, strictly and verified + against the registry's integrity data, with several versions kept side by side + when dependents need different ones), or can be declared as separate pinned + `ts_npm_module` targets with `resolve_transitive = False`. Type checks and + tests run with `--cached-only`, so a missing module fails instead of being + downloaded. +- Dependency resolution is reproducible without a lockfile: only versions + published by the end of the day the root version was published are considered, + for both `ts_npm_module` and `ts_module`, so the version pinned by its hash + fixes the result and nothing needs configuring. +- `please_ts unpack --npm-cache`, and the `npmcache`, `registry` and `semver` + packages behind it. +- Fixtures for `debug` (CommonJS, transitive dependency), `highlight.js` + (CommonJS, subpaths) and `@codemirror/legacy-modes` (exports map only), + automatic and fully pinned. + +### Fixed + +- `ts_module`'s automatic dependency resolution (`resolve_transitive`) is now + strict and verified. It used to fall back to the `latest` version when no + version satisfied a range, verify no download against the registry's integrity + data, keep the first version of a package silently when a second dependent + needed an incompatible one, and only print a warning, leaving an incomplete + tree, when resolution failed. A range that no version satisfies, an integrity + mismatch, a tarball that holds another package and conflicting versions are + now build errors; unresolvable peer dependencies stay warnings. Version ranges + are interpreted completely (`||`, hyphen and x-ranges, prereleases) instead of + only `^`, `~` and `>=`. **This can make builds fail that used to pass with an + incomplete or inconsistent tree.** + ## [0.6.0] - 2026-10-03 ### Added diff --git a/build_defs/ts/ts.build_defs b/build_defs/ts/ts.build_defs index 10f4622..04ffa82 100644 --- a/build_defs/ts/ts.build_defs +++ b/build_defs/ts/ts.build_defs @@ -182,7 +182,17 @@ def ts_module( registry: str = "", visibility: list = None, labels: list = None): - """Downloads an npm package tarball, extracts it, resolves transitive dependencies, and creates metadata for downstream targets.""" + """Downloads an npm package tarball, extracts it, resolves transitive dependencies, and creates metadata for downstream targets. + + Only the root tarball is pinned (`hashes`). Dependencies are resolved from the registry at + build time, strictly: a range that no version satisfies, a download that does not match + the registry's integrity data, or two dependents that need incompatible versions of one + package (this layout holds one) fails the build. + + Resolution is reproducible: only dependency versions published by the end of the day the + root version was published are considered, so the pinned root fixes the result and nothing + needs configuring. Bump the root version to move the dependencies forward. + """ pkg = package or name ver = version @@ -219,6 +229,85 @@ def ts_module( building_description = f"Unpacking npm module {pkg}@{ver}...", ) +def ts_npm_module( + name: str, + package: str = "", + version: str = "", + hashes: list = None, + url: str = "", + deps: list = None, + resolve_transitive: bool = True, + registry: str = "", + visibility: list = None, + labels: list = None): + """Provides one npm package to Deno's own npm resolution, hermetically and offline once built. + + The tarball is pinned by `hashes` (sha256, checked by Please). The build step extracts it + into a slice of a Deno npm cache; targets that depend on it get the slices merged into + their DENO_DIR and import the package through an `npm:` specifier. Deno then resolves + `exports`, CommonJS and subpaths itself: there is no node_modules directory and no + lockfile, and a package that Deno cannot resolve from the cache fails the build rather + than reaching for the network. + + Dependencies are handled in one of two ways: + + * `resolve_transitive = True` (the default, like ts_module): the build resolves the + dependencies from the registry, each verified against the registry's integrity data, + and keeps several versions of a package when dependents need different ones. This needs + network access during the build (so no sandbox) and trusts the registry for everything + but the root tarball. Resolution is reproducible without a lockfile: only versions + published by the end of the day the root version was published are considered, so the + pinned root fixes the result (bump its version to move the dependencies forward). Dependencies listed in `deps` are used as they are, not + resolved again. + * `resolve_transitive = False`: nothing is resolved. Every dependency (and theirs) must + be its own ts_npm_module, listed in `deps`, each pinned by its own sha256; the build + needs no network access and fails naming any dependency that is missing. + + Args: + name: Name of the rule. + package: npm package name (defaults to `name`), e.g. "debug" or "@scope/pkg". + version: Exact package version. + hashes: sha256 of the package tarball. + url: Tarball URL (defaults to the npm registry). + deps: ts_npm_module targets that provide dependencies explicitly. + resolve_transitive: Resolve the dependencies from the registry at build time. + registry: npm registry URL used to resolve dependencies. + visibility: Visibility of the rule. + labels: Labels for the rule. + """ + pkg = package or name + if not url: + pkg_base = pkg.split("/")[-1] if "/" in pkg else pkg + url = f"https://registry.npmjs.org/{pkg}/-/{pkg_base}-{version}.tgz" + + dl_target = f"_{name}#download" + remote_file( + name = dl_target, + url = url, + hashes = hashes or [], + out = f"{name}.tgz", + ) + + tools = {"TOOL": [_ts_tool()]} + resolve_flag = "--resolve-transitive" if resolve_transitive else "" + registry_flag = f'--registry "{registry}"' if registry else "" + return build_rule( + name = name, + srcs = [f":{dl_target}"], + deps = deps or [], + exported_deps = deps or [], + outs = [name], + cmd = f'$TOOLS_TOOL unpack --npm-cache --tarball "$SRCS" --out "$OUT" --name "{pkg}" {resolve_flag} {registry_flag}', + # Resolving needs the registry. Without it the step only reads local files and follows + # the [sandbox] setting like every other rule. + sandbox = False if resolve_transitive else None, + needs_transitive_deps = True, + tools = tools, + visibility = visibility or ["PUBLIC"], + labels = labels or ["ts", "module", "npm", "npm_cache"], + building_description = f"Building Deno npm cache slice for {pkg}@{version}...", + ) + def ts_library( name: str, srcs: list, diff --git a/docs/ts/usage.md b/docs/ts/usage.md index 5ea5c70..4c6a586 100644 --- a/docs/ts/usage.md +++ b/docs/ts/usage.md @@ -219,3 +219,85 @@ branch (`BRDA`) records that `coverage.xml` and `coverage.json` do not: `tools/common/lcov` package. Do not merge branch records of the Deno and Vitest runners for the same file: they number the arms of a branch differently. + +--- + +## npm Packages Through Deno's Own Resolution (`ts_npm_module`, experimental) + +`ts_module` unpacks a package and describes it with a single entry file, which +cannot express CommonJS packages, subpath imports (`highlight.js/lib/core`) or +packages that only have an `exports` map. `ts_npm_module` takes a different +route: Deno resolves the package itself through an `npm:` specifier, and the +rule only has to make the package available **offline**. List the package you +import, pinned by the hash of its tarball: + +```starlark +ts_npm_module( + name = "debug", + hashes = ["c803a8ca9b835b7a75f7150ee52f7f640675515bafbe8f5da78fcc0ae12914ac"], + version = "4.3.7", +) + +ts_library( + name = "humanize", + srcs = ["humanize.ts"], + module_name = "@app/humanize", + deps = [":debug"], +) + +ts_test( + name = "humanize_test", + srcs = ["humanize_test.ts"], + deps = [":debug", ":humanize"], # list the npm module itself, as with ts_module +) +``` + +```typescript +import createDebug from "debug"; // CommonJS, and it needs the package "ms" +``` + +- **Dependencies resolve automatically** (`resolve_transitive`, on by default, + as for `ts_module`): `debug` needs `ms`, and nothing else has to be declared. + The build resolves dependencies from the registry, strictly: a range that no + version satisfies, a download that does not match the registry's integrity + data, or a tarball that holds another package fails the build. Dependents that + need different versions of one package each get theirs (Deno resolves per + dependent), which `ts_module`'s single `.deps` directory cannot do. Optional + and peer dependencies are not fetched. +- **Reproducible without a lockfile**: only dependency versions published by the + end of the day the root version was published are considered, so the version + you pin by hash fixes the result and nothing needs configuring. Bump the root + version to move the dependencies forward. If the registry does not know the + root version's publish time (a private registry, a tarball from another URL), + the build says so and does not pin. +- **Cost of automatic resolution**: the build needs network access (so no + sandbox for these targets) and trusts the registry for everything except the + root tarball. +- **Fully pinned alternative**: with `resolve_transitive = False` nothing is + resolved. Every dependency, and theirs, is its own `ts_npm_module` in `deps`, + each with its own tarball hash, and the build needs no network access. The + build fails and names any dependency that is missing. Dependencies you do list + in `deps` are always used as they are, never resolved again, so the two modes + can be mixed. +- **Offline afterwards**: everything that runs after the build (type checks, + tests) uses only the extracted cache, with no `node_modules` directory and no + lockfile. Targets run with `--cached-only`, so a module missing from a + target's `deps` fails at once with `npm package not found in cache`; Deno does + not download it. +- **Resolution is Deno's**: `exports` maps, CommonJS, subpaths and a package's + own types work as they do for `npm:` specifiers. Imports use the plain package + name. +- **Not wired yet**: `ts_bundle` and `ts_binary`, the Vitest and browser + runners, and a helper that prints pinned declarations. The cache layout + (`registry.json`, including its `_deno.packumentFormat` key) is internal to + Deno, so it is tied to the Deno version the plugin pins. + +`ts_module` resolves its dependencies by the same rules (strictly, verified, as +of the root's publish day), with the limit of one version per package: two +dependents that need incompatible versions fail the build and name each other. + +The fixtures in `test/ts/npm_cache` cover a CommonJS package with a transitive +dependency (`debug`), a CommonJS package imported through subpaths +(`highlight.js`), and an ES module package that only has an `exports` map and no +`main` (`@codemirror/legacy-modes`), each with automatic resolution and, for the +last two, with every dependency pinned explicitly. diff --git a/test/ts/npm_cache/BUILD b/test/ts/npm_cache/BUILD new file mode 100644 index 0000000..101bc0d --- /dev/null +++ b/test/ts/npm_cache/BUILD @@ -0,0 +1,133 @@ +subinclude("//build_defs/ts:ts") + +# A CommonJS package and its transitive dependency, each pinned by the sha256 of its tarball. +ts_npm_module( + name = "ms", + hashes = ["f6616e15e530ed552f9daa2d3ce71963947c6bc7c98c9b64fd3e673fd02622c6"], + version = "2.1.3", + resolve_transitive = False, +) + +ts_npm_module( + name = "debug", + hashes = ["c803a8ca9b835b7a75f7150ee52f7f640675515bafbe8f5da78fcc0ae12914ac"], + version = "4.3.7", + resolve_transitive = False, + deps = [":ms"], +) + +ts_library( + name = "humanize", + srcs = ["humanize.ts"], + module_name = "@test/humanize", + deps = [":debug"], +) + +ts_test( + name = "humanize_test", + srcs = ["humanize_test.ts"], + deps = [ + ":debug", + ":humanize", + ], +) + +# highlight.js is CommonJS and is imported through subpaths (highlight.js/lib/core). +ts_library( + name = "highlight", + srcs = ["highlight.ts"], + module_name = "@test/highlight", + deps = ["//test/ts/npm_cache/modules:highlight_js"], +) + +ts_test( + name = "highlight_test", + srcs = ["highlight_test.ts"], + deps = [ + ":highlight", + "//test/ts/npm_cache/modules:highlight_js", + ], +) + +# @codemirror/legacy-modes is ESM with only an exports map (./mode/*) and no main or module. +ts_library( + name = "modes", + srcs = ["modes.ts"], + module_name = "@test/modes", + deps = [ + "//test/ts/npm_cache/modules:codemirror_language", + "//test/ts/npm_cache/modules:codemirror_legacy_modes", + ], +) + +ts_test( + name = "modes_test", + srcs = ["modes_test.ts"], + deps = [ + ":modes", + "//test/ts/npm_cache/modules:codemirror_language", + "//test/ts/npm_cache/modules:codemirror_legacy_modes", + ], +) + +# Automatic resolution: only the root tarball is pinned. The dependencies are resolved from +# the registry when the module is built, as of the day the root version was published, and +# verified against the registry's integrity data. +ts_npm_module( + name = "debug_auto", + package = "debug", + hashes = ["c803a8ca9b835b7a75f7150ee52f7f640675515bafbe8f5da78fcc0ae12914ac"], + version = "4.3.7", +) + +ts_library( + name = "humanize_auto", + srcs = ["humanize.ts"], + module_name = "@test/humanize_auto", + deps = [":debug_auto"], +) + +ts_test( + name = "humanize_auto_test", + srcs = ["humanize_auto_test.ts"], + deps = [ + ":debug_auto", + ":humanize_auto", + ], +) + +# The same exports-only package as modes_test, with its eleven dependencies resolved +# automatically instead of declared. +ts_npm_module( + name = "legacy_modes_auto", + package = "@codemirror/legacy-modes", + hashes = ["61143cdb9c375dc1521895e4e536d75036baaf6e9df6b4431691f7ea5d273463"], + version = "6.5.1", +) + +ts_npm_module( + name = "codemirror_language_auto", + package = "@codemirror/language", + hashes = ["5e49acf55fde65ce9848068f0f06478be9ec71f818e91de6eca00824e9152226"], + version = "6.12.4", +) + +ts_library( + name = "modes_auto", + srcs = ["modes.ts"], + module_name = "@test/modes_auto", + deps = [ + ":codemirror_language_auto", + ":legacy_modes_auto", + ], +) + +ts_test( + name = "modes_auto_test", + srcs = ["modes_auto_test.ts"], + deps = [ + ":codemirror_language_auto", + ":legacy_modes_auto", + ":modes_auto", + ], +) diff --git a/test/ts/npm_cache/highlight.ts b/test/ts/npm_cache/highlight.ts new file mode 100644 index 0000000..496c427 --- /dev/null +++ b/test/ts/npm_cache/highlight.ts @@ -0,0 +1,10 @@ +import core from "highlight.js/lib/core"; +import go from "highlight.js/lib/languages/go"; + +const hljs: any = core; +hljs.registerLanguage("go", go); + +/** Highlights Go source as HTML. */ +export function highlight(source: string): string { + return hljs.highlight(source, { language: "go" }).value; +} diff --git a/test/ts/npm_cache/highlight_test.ts b/test/ts/npm_cache/highlight_test.ts new file mode 100644 index 0000000..00fe786 --- /dev/null +++ b/test/ts/npm_cache/highlight_test.ts @@ -0,0 +1,8 @@ +import { highlight } from "@test/highlight"; + +Deno.test("highlight.js: subpath imports of a CommonJS package", () => { + const html = highlight("package main"); + if (!html.includes("hljs-keyword")) { + throw new Error(`expected a highlighted keyword, got ${html}`); + } +}); diff --git a/test/ts/npm_cache/humanize.ts b/test/ts/npm_cache/humanize.ts new file mode 100644 index 0000000..cbdba6d --- /dev/null +++ b/test/ts/npm_cache/humanize.ts @@ -0,0 +1,6 @@ +import createDebug from "debug"; + +/** Formats a duration in milliseconds with the `ms` package, which `debug` re-exports. */ +export function humanize(milliseconds: number): string { + return (createDebug as any).humanize(milliseconds); +} diff --git a/test/ts/npm_cache/humanize_auto_test.ts b/test/ts/npm_cache/humanize_auto_test.ts new file mode 100644 index 0000000..a199a8f --- /dev/null +++ b/test/ts/npm_cache/humanize_auto_test.ts @@ -0,0 +1,8 @@ +import { humanize } from "@test/humanize_auto"; + +Deno.test("humanize (auto-resolved) formats durations through a CommonJS package and its dependency", () => { + const got = humanize(90061000); + if (got !== "1d") { + throw new Error(`expected 1d, got ${got}`); + } +}); diff --git a/test/ts/npm_cache/humanize_test.ts b/test/ts/npm_cache/humanize_test.ts new file mode 100644 index 0000000..e32cb45 --- /dev/null +++ b/test/ts/npm_cache/humanize_test.ts @@ -0,0 +1,8 @@ +import { humanize } from "@test/humanize"; + +Deno.test("humanize formats durations through a CommonJS package and its dependency", () => { + const got = humanize(90061000); + if (got !== "1d") { + throw new Error(`expected 1d, got ${got}`); + } +}); diff --git a/test/ts/npm_cache/modes.ts b/test/ts/npm_cache/modes.ts new file mode 100644 index 0000000..ca14661 --- /dev/null +++ b/test/ts/npm_cache/modes.ts @@ -0,0 +1,7 @@ +import { go } from "@codemirror/legacy-modes/mode/go"; +import { StreamLanguage } from "@codemirror/language"; + +/** A CodeMirror language for Go, built from a legacy mode. */ +export function goLanguage(): StreamLanguage { + return StreamLanguage.define(go); +} diff --git a/test/ts/npm_cache/modes_auto_test.ts b/test/ts/npm_cache/modes_auto_test.ts new file mode 100644 index 0000000..b355d97 --- /dev/null +++ b/test/ts/npm_cache/modes_auto_test.ts @@ -0,0 +1,8 @@ +import { goLanguage } from "@test/modes_auto"; + +Deno.test("legacy-modes (auto-resolved): an ESM package that only has an exports map", () => { + const language = goLanguage(); + if (!language || typeof language.parser !== "object") { + throw new Error("expected a StreamLanguage with a parser"); + } +}); diff --git a/test/ts/npm_cache/modes_test.ts b/test/ts/npm_cache/modes_test.ts new file mode 100644 index 0000000..6809a59 --- /dev/null +++ b/test/ts/npm_cache/modes_test.ts @@ -0,0 +1,8 @@ +import { goLanguage } from "@test/modes"; + +Deno.test("legacy-modes: an ESM package that only has an exports map", () => { + const language = goLanguage(); + if (!language || typeof language.parser !== "object") { + throw new Error("expected a StreamLanguage with a parser"); + } +}); diff --git a/test/ts/npm_cache/modules/BUILD b/test/ts/npm_cache/modules/BUILD new file mode 100644 index 0000000..1f59d54 --- /dev/null +++ b/test/ts/npm_cache/modules/BUILD @@ -0,0 +1,127 @@ +subinclude("//build_defs/ts:ts") + +# Pinned npm packages for the fixtures: highlight.js (CommonJS, subpath imports) and +# @codemirror/legacy-modes (ESM with only an exports map, no main/module) and its dependencies. +# Every package is its own target with the sha256 of its tarball; dependencies are listed in deps. +# (Generated from an npm lock; the planned helper would print these declarations.) + +ts_npm_module( + name = "codemirror_language", + package = "@codemirror/language", + hashes = ["5e49acf55fde65ce9848068f0f06478be9ec71f818e91de6eca00824e9152226"], + version = "6.12.4", + resolve_transitive = False, + deps = [ + ":codemirror_state", + ":codemirror_view", + ":lezer_common", + ":lezer_highlight", + ":lezer_lr", + ":style_mod", + ], +) + +ts_npm_module( + name = "codemirror_legacy_modes", + package = "@codemirror/legacy-modes", + hashes = ["61143cdb9c375dc1521895e4e536d75036baaf6e9df6b4431691f7ea5d273463"], + version = "6.5.1", + resolve_transitive = False, + deps = [ + ":codemirror_language", + ], +) + +ts_npm_module( + name = "codemirror_state", + package = "@codemirror/state", + hashes = ["91d75acc955ceeaf86af9396c11cda7694fe6222aac8427fa033d89eff5ac839"], + version = "6.7.6", + resolve_transitive = False, + deps = [ + ":marijn_find_cluster_break", + ], +) + +ts_npm_module( + name = "codemirror_view", + package = "@codemirror/view", + hashes = ["1246c9b6e1ad1d9d870b3e67672d3a8f8c9bc73af3b63c081c1549a94d5850db"], + version = "6.43.13", + resolve_transitive = False, + deps = [ + ":codemirror_state", + ":crelt", + ":style_mod", + ":w3c_keyname", + ], +) + +ts_npm_module( + name = "lezer_common", + package = "@lezer/common", + hashes = ["a8854639c04adabe5e72c26fa2876475116dc099ae5aa02250a5a938bb5ea4ae"], + version = "1.5.3", + resolve_transitive = False, +) + +ts_npm_module( + name = "lezer_highlight", + package = "@lezer/highlight", + hashes = ["e349eec1ff4382439c1e6286ae937e0203a8349c5192e2e133863391775ac7ff"], + version = "1.2.5", + resolve_transitive = False, + deps = [ + ":lezer_common", + ], +) + +ts_npm_module( + name = "lezer_lr", + package = "@lezer/lr", + hashes = ["c71e654cd0d153898e744a613bb82714fcd3c7d8e4df8cef7d193fd5ddfa6015"], + version = "1.4.10", + resolve_transitive = False, + deps = [ + ":lezer_common", + ], +) + +ts_npm_module( + name = "marijn_find_cluster_break", + package = "@marijn/find-cluster-break", + hashes = ["5d12b770b64d46632c84fcd18e1d4c09f195ed0f5dbc71f94a41f0af333eaeae"], + version = "1.0.4", + resolve_transitive = False, +) + +ts_npm_module( + name = "crelt", + hashes = ["3542299c0278fdc26aceb2b13d31bffbcb89d7367706bbc457130e3fd103c03e"], + version = "1.0.7", + resolve_transitive = False, +) + +ts_npm_module( + name = "highlight_js", + package = "highlight.js", + hashes = ["accbfaaab745088609b4eea2bdca2ad62f1f1dd27304e0f8df65cfe0fe042143"], + version = "11.12.0", + resolve_transitive = False, +) + +ts_npm_module( + name = "style_mod", + package = "style-mod", + hashes = ["769073434ab698eb035cbaa9df5d028231c25d95bc7b8676a46d603fb2e89e39"], + version = "4.1.4", + resolve_transitive = False, +) + +ts_npm_module( + name = "w3c_keyname", + package = "w3c-keyname", + hashes = ["05d21484026d1f2dd842c843181e391e66d60f1d5dff292d562c3f40302dab36"], + version = "2.2.8", + resolve_transitive = False, +) diff --git a/tools/please_ts/commands.go b/tools/please_ts/commands.go index a4010d1..97845bd 100644 --- a/tools/please_ts/commands.go +++ b/tools/please_ts/commands.go @@ -194,6 +194,7 @@ func handleUnpack(args []string) error { symlink := cmd.String("symlink", "", "Optional symlink name for extracted binary") resolveTransitive := cmd.Bool("resolve-transitive", false, "Recursively resolve and download transitive dependencies") registry := cmd.String("registry", "", "NPM registry base URL") + npmCache := cmd.Bool("npm-cache", false, "Build a Deno npm cache slice from the tarball") if err := cmd.Parse(args); err != nil { return err @@ -213,6 +214,7 @@ func handleUnpack(args []string) error { Symlink: *symlink, ResolveTransitive: *resolveTransitive, Registry: *registry, + NpmCache: *npmCache, } return unpack.Run(opts) } diff --git a/tools/please_ts/compile/BUILD b/tools/please_ts/compile/BUILD index 5309481..cc53324 100644 --- a/tools/please_ts/compile/BUILD +++ b/tools/please_ts/compile/BUILD @@ -4,7 +4,10 @@ go_library( name = "compile", srcs = ["compile.go"], visibility = ["//tools/please_ts/..."], - deps = ["//tools/please_ts/importmap"], + deps = [ + "//tools/please_ts/importmap", + "//tools/please_ts/npmcache", + ], ) go_test( @@ -13,5 +16,8 @@ go_test( "compile.go", "compile_test.go", ], - deps = ["//tools/please_ts/importmap"], + deps = [ + "//tools/please_ts/importmap", + "//tools/please_ts/npmcache", + ], ) diff --git a/tools/please_ts/compile/compile.go b/tools/please_ts/compile/compile.go index 9d6ae75..0f69456 100644 --- a/tools/please_ts/compile/compile.go +++ b/tools/please_ts/compile/compile.go @@ -10,6 +10,7 @@ import ( "strings" "tools/please_ts/importmap" + "tools/please_ts/npmcache" ) // Options holds configuration for TypeScript library compilation. @@ -53,6 +54,14 @@ func Run(opts Options) error { } } + // npm packages provided by ts_npm_module targets are merged into the per-run Deno cache + // and resolved from there, offline. The shared Vitest cache is left as it is. + if opts.VitestDir == "" { + if _, err := npmcache.Prepare(".", denoCacheDir); err != nil { + return fmt.Errorf("failed preparing the npm cache: %w", err) + } + } + // 1. Synthesize target-local import map importMapPath := ".import_map.json" im, err := importmap.Synthesize(opts.ModuleName, opts.Srcs, opts.Deps, ".") @@ -73,6 +82,12 @@ func Run(opts Options) error { "--no-remote", "--import-map", importMapPath, } + // npm packages come from the merged cache only: a ts_npm_module missing from the + // target's deps must fail here, not be downloaded. (The shared Vitest cache is a + // different mechanism and keeps its own behaviour.) + if opts.VitestDir == "" && im.HasNpmSpecifiers() { + args = append(args, "--cached-only") + } args = append(args, opts.Flags...) args = append(args, opts.Srcs...) diff --git a/tools/please_ts/importmap/BUILD b/tools/please_ts/importmap/BUILD index e74ef99..9870036 100644 --- a/tools/please_ts/importmap/BUILD +++ b/tools/please_ts/importmap/BUILD @@ -4,9 +4,11 @@ go_library( name = "importmap", srcs = glob(["*.go"], exclude = ["*_test.go"]), visibility = ["//tools/please_ts/..."], + deps = ["//tools/please_ts/npmcache"], ) go_test( name = "importmap_test", srcs = glob(["*.go"]), + deps = ["//tools/please_ts/npmcache"], ) diff --git a/tools/please_ts/importmap/importmap.go b/tools/please_ts/importmap/importmap.go index 085f751..965b933 100644 --- a/tools/please_ts/importmap/importmap.go +++ b/tools/please_ts/importmap/importmap.go @@ -117,3 +117,16 @@ func (im *ImportMap) WriteToFile(filePath string) error { } return os.WriteFile(filePath, data, 0644) } + +// HasNpmSpecifiers reports whether the import map sends any module to an npm: specifier. +// Those are resolved from the Deno cache that the npm slices are merged into, so a target +// whose import map has them must run with --cached-only: a slice that is missing from the +// target's dependencies then fails at once, instead of Deno quietly downloading the package. +func (im *ImportMap) HasNpmSpecifiers() bool { + for _, target := range im.Imports { + if strings.HasPrefix(target, "npm:") { + return true + } + } + return false +} diff --git a/tools/please_ts/importmap/loader.go b/tools/please_ts/importmap/loader.go index 65ff787..87533eb 100644 --- a/tools/please_ts/importmap/loader.go +++ b/tools/please_ts/importmap/loader.go @@ -6,6 +6,8 @@ import ( "os" "path/filepath" "strings" + + "tools/please_ts/npmcache" ) // discoverDepPaths collects explicit dependencies and auto-discovers module metadata files in workingDir. @@ -15,7 +17,7 @@ func discoverDepPaths(deps []string, workingDir string) []string { _ = filepath.Walk(workingDir, func(path string, info os.FileInfo, err error) error { if err == nil && !info.IsDir() { - if info.Name() == "ts_metadata.json" || info.Name() == "ts_module.json" { + if info.Name() == "ts_metadata.json" || info.Name() == "ts_module.json" || info.Name() == npmcache.MetadataFile { allDeps = append(allDeps, path) } } @@ -51,6 +53,9 @@ func (im *ImportMap) LoadDependencies(depPaths []string, workingDir string) ([]* // LoadDepItem handles a single dependency path: metadata file, directory, or source file. func (im *ImportMap) LoadDepItem(dep string, workingDir string) (*loadedModule, error) { + if filepath.Base(dep) == npmcache.MetadataFile { + return nil, im.registerNpmSlice(filepath.Dir(dep)) + } if isMetadataFile(dep) { mod, err := loadMetadataFile(dep, workingDir) if err != nil { @@ -73,6 +78,10 @@ func (im *ImportMap) LoadDepItem(dep string, workingDir string) (*loadedModule, // LoadDirectoryDep checks for metadata inside a directory or maps it directly. func (im *ImportMap) LoadDirectoryDep(dir, workingDir string) (*loadedModule, error) { + if _, err := os.Stat(filepath.Join(dir, npmcache.MetadataFile)); err == nil { + return nil, im.registerNpmSlice(dir) + } + metaPath := filepath.Join(dir, "ts_module.json") if _, err := os.Stat(metaPath); err == nil { return loadMetadataFile(metaPath, workingDir) @@ -87,6 +96,19 @@ func (im *ImportMap) LoadDirectoryDep(dir, workingDir string) (*loadedModule, er return nil, nil } +// registerNpmSlice maps an npm package provided by a ts_npm_module to npm: specifiers that +// Deno resolves from its cache (see package npmcache): the bare name for the package +// entry, and name/ for subpaths, which Deno resolves with the package's own exports map. +func (im *ImportMap) registerNpmSlice(dir string) error { + slice, err := npmcache.Read(dir) + if err != nil { + return fmt.Errorf("failed loading npm slice %s: %w", dir, err) + } + im.Imports[slice.Name] = slice.Specifier + im.Imports[slice.Name+"/"] = "npm:/" + slice.Name + "@" + slice.Version + "/" + return nil +} + // MapDirectSourceFile maps a single source file to imports. func (im *ImportMap) MapDirectSourceFile(filePath, workingDir string) { relPath, err := relativeTo(workingDir, filePath) diff --git a/tools/please_ts/importmap/npm_slice_test.go b/tools/please_ts/importmap/npm_slice_test.go new file mode 100644 index 0000000..ecc5310 --- /dev/null +++ b/tools/please_ts/importmap/npm_slice_test.go @@ -0,0 +1,106 @@ +package importmap + +import ( + "os" + "path/filepath" + "testing" + + "tools/please_ts/npmcache" +) + +func writeNpmSlice(t *testing.T, root, dir string, s npmcache.Slice) string { + t.Helper() + d := filepath.Join(root, dir) + if err := os.MkdirAll(d, 0755); err != nil { + t.Fatal(err) + } + if err := npmcache.Write(d, s); err != nil { + t.Fatal(err) + } + return d +} + +func TestSynthesizeMapsNpmSlicesToNpmSpecifiers(t *testing.T) { + root := t.TempDir() + writeNpmSlice(t, root, "third_party/debug", npmcache.Slice{Name: "debug", Version: "4.3.7", Specifier: "npm:debug@4.3.7"}) + writeNpmSlice(t, root, "third_party/scoped", npmcache.Slice{Name: "@codemirror/legacy-modes", Version: "6.5.1", Specifier: "npm:@codemirror/legacy-modes@6.5.1"}) + + im, err := Synthesize("", nil, nil, root) + if err != nil { + t.Fatal(err) + } + want := map[string]string{ + "debug": "npm:debug@4.3.7", + "debug/": "npm:/debug@4.3.7/", + "@codemirror/legacy-modes": "npm:@codemirror/legacy-modes@6.5.1", + "@codemirror/legacy-modes/": "npm:/@codemirror/legacy-modes@6.5.1/", + } + for k, v := range want { + if im.Imports[k] != v { + t.Errorf("imports[%q] = %q, want %q", k, im.Imports[k], v) + } + } +} + +func TestSynthesizeMapsExplicitNpmSliceDirectoryAndMetadataFile(t *testing.T) { + root := t.TempDir() + dir := writeNpmSlice(t, root, "third_party/ms", npmcache.Slice{Name: "ms", Version: "2.1.3", Specifier: "npm:ms@2.1.3"}) + + for name, dep := range map[string]string{"directory": dir, "metadata file": filepath.Join(dir, npmcache.MetadataFile)} { + im := New() + if _, err := im.LoadDependencies([]string{dep}, root); err != nil { + t.Fatalf("%s: %v", name, err) + } + if im.Imports["ms"] != "npm:ms@2.1.3" || im.Imports["ms/"] != "npm:/ms@2.1.3/" { + t.Errorf("%s: imports = %v", name, im.Imports) + } + } +} + +func TestSynthesizeReportsAnUnreadableNpmSlice(t *testing.T) { + root := t.TempDir() + d := filepath.Join(root, "third_party", "broken") + if err := os.MkdirAll(d, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(d, npmcache.MetadataFile), []byte("{"), 0644); err != nil { + t.Fatal(err) + } + if _, err := Synthesize("", nil, nil, root); err == nil { + t.Error("expected an error for invalid ts_npm.json") + } +} + +func TestNpmSlicesLeaveFirstPartyAliasesAlone(t *testing.T) { + root := t.TempDir() + writeNpmSlice(t, root, "third_party/ms", npmcache.Slice{Name: "ms", Version: "2.1.3", Specifier: "npm:ms@2.1.3"}) + src := filepath.Join(root, "app", "app.ts") + if err := os.MkdirAll(filepath.Dir(src), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(src, []byte("export {};"), 0644); err != nil { + t.Fatal(err) + } + im, err := Synthesize("@app/app", []string{src}, nil, root) + if err != nil { + t.Fatal(err) + } + if im.Imports["@app/app"] == "" || im.Imports["ms"] != "npm:ms@2.1.3" { + t.Errorf("imports = %v", im.Imports) + } +} + +func TestHasNpmSpecifiers(t *testing.T) { + im := New() + if im.HasNpmSpecifiers() { + t.Error("an empty import map has no npm specifiers") + } + im.Imports["@app/lib"] = "./lib/lib.ts" + if im.HasNpmSpecifiers() { + t.Error("first-party aliases are not npm specifiers") + } + im.Imports["ms/"] = "npm:/ms@2.1.3/" + if !im.HasNpmSpecifiers() { + t.Error("an npm:/ prefix mapping is an npm specifier") + } +} diff --git a/tools/please_ts/npmcache/BUILD b/tools/please_ts/npmcache/BUILD new file mode 100644 index 0000000..a92da92 --- /dev/null +++ b/tools/please_ts/npmcache/BUILD @@ -0,0 +1,15 @@ +subinclude("///go//build_defs:go") + +go_library( + name = "npmcache", + srcs = ["npmcache.go"], + visibility = ["//tools/please_ts/..."], +) + +go_test( + name = "npmcache_test", + srcs = [ + "npmcache.go", + "npmcache_test.go", + ], +) diff --git a/tools/please_ts/npmcache/npmcache.go b/tools/please_ts/npmcache/npmcache.go new file mode 100644 index 0000000..909215a --- /dev/null +++ b/tools/please_ts/npmcache/npmcache.go @@ -0,0 +1,184 @@ +// Package npmcache handles npm packages as slices of a Deno npm cache. +// +// A slice is the output of a ts_npm_module target: a directory holding the package +// extracted into Deno's cache layout (npm/registry.npmjs.org/// plus a +// registry.json), and a ts_npm.json describing it. Slices are built offline from a +// hash-pinned tarball; the runners merge the slices a target depends on into the +// per-run DENO_DIR, so Deno resolves npm: specifiers without network access, without a +// node_modules directory and without a lockfile. +package npmcache + +import ( + "encoding/json" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" +) + +// MetadataFile is the name of the file that marks a directory as a slice. +const MetadataFile = "ts_npm.json" + +// registryDir is the directory under the Deno cache that holds npm packages. +const registryDir = "npm/registry.npmjs.org" + +// Slice describes one npm package of the cache. +type Slice struct { + Name string `json:"name"` + Version string `json:"version"` + Specifier string `json:"specifier"` + Dependencies map[string]string `json:"dependencies,omitempty"` +} + +// Specifier returns the npm: specifier of an exact package version. +func Specifier(name, version string) string { + return "npm:" + name + "@" + version +} + +// Read loads the slice described by dir/ts_npm.json. +func Read(dir string) (*Slice, error) { + data, err := os.ReadFile(filepath.Join(dir, MetadataFile)) + if err != nil { + return nil, err + } + var s Slice + if err := json.Unmarshal(data, &s); err != nil { + return nil, fmt.Errorf("%s: %w", filepath.Join(dir, MetadataFile), err) + } + if s.Name == "" || s.Version == "" { + return nil, fmt.Errorf("%s: name and version are required", filepath.Join(dir, MetadataFile)) + } + return &s, nil +} + +// Write stores the slice metadata in dir/ts_npm.json. +func Write(dir string, s Slice) error { + data, err := json.MarshalIndent(s, "", " ") + if err != nil { + return err + } + return os.WriteFile(filepath.Join(dir, MetadataFile), data, 0644) +} + +// Discover returns the slice directories below root, sorted. +func Discover(root string) []string { + var dirs []string + _ = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err == nil && !d.IsDir() && d.Name() == MetadataFile { + dirs = append(dirs, filepath.Dir(path)) + } + return nil + }) + sort.Strings(dirs) + return dirs +} + +// Merge copies the npm cache content of every slice into denoDir (a DENO_DIR). Files +// that already exist are kept, except registry.json, whose versions are unioned so that +// two slices holding different versions of one package both stay resolvable. +func Merge(slices []string, denoDir string) error { + for _, slice := range slices { + src := filepath.Join(slice, "npm") + if _, err := os.Stat(src); err != nil { + continue + } + err := filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(slice, path) + if err != nil { + return err + } + dst := filepath.Join(denoDir, rel) + if d.IsDir() { + return os.MkdirAll(dst, 0755) + } + if _, err := os.Stat(dst); err == nil { + if d.Name() == "registry.json" && strings.Contains(filepath.ToSlash(dst), registryDir) { + return mergePackument(path, dst) + } + return nil + } + return copyFile(path, dst) + }) + if err != nil { + return fmt.Errorf("merging npm slice %s: %w", slice, err) + } + } + return nil +} + +// Prepare merges the slices found below root into denoDir and reports whether there were +// any. Callers add --cached-only to the Deno command when it returns true, so a package +// missing from the cache fails at once instead of reaching for the network. +func Prepare(root, denoDir string) (bool, error) { + slices := Discover(root) + if len(slices) == 0 { + return false, nil + } + if err := os.MkdirAll(denoDir, 0755); err != nil { + return false, err + } + return true, Merge(slices, denoDir) +} + +func copyFile(src, dst string) error { + info, err := os.Stat(src) + if err != nil { + return err + } + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, info.Mode().Perm()|0200) + if err != nil { + return err + } + if _, err := io.Copy(out, in); err != nil { + out.Close() + return err + } + return out.Close() +} + +// mergePackument adds the versions of the registry.json at src to the one at dst. +func mergePackument(src, dst string) error { + var a, b map[string]json.RawMessage + for path, target := range map[string]*map[string]json.RawMessage{dst: &a, src: &b} { + data, err := os.ReadFile(path) + if err != nil { + return err + } + if err := json.Unmarshal(data, target); err != nil { + return fmt.Errorf("%s: %w", path, err) + } + } + var va, vb map[string]json.RawMessage + if err := json.Unmarshal(a["versions"], &va); err != nil { + return err + } + if err := json.Unmarshal(b["versions"], &vb); err != nil { + return err + } + for v, raw := range vb { + if _, ok := va[v]; !ok { + va[v] = raw + } + } + merged, err := json.Marshal(va) + if err != nil { + return err + } + a["versions"] = merged + out, err := json.Marshal(a) + if err != nil { + return err + } + return os.WriteFile(dst, out, 0644) +} diff --git a/tools/please_ts/npmcache/npmcache_test.go b/tools/please_ts/npmcache/npmcache_test.go new file mode 100644 index 0000000..6a07336 --- /dev/null +++ b/tools/please_ts/npmcache/npmcache_test.go @@ -0,0 +1,176 @@ +package npmcache + +import ( + "encoding/json" + "os" + "path/filepath" + "reflect" + "testing" +) + +func makeSlice(t *testing.T, root, name, version string, deps map[string]string) string { + t.Helper() + dir := filepath.Join(root, "slices", name+"-"+version) + pkgDir := filepath.Join(dir, "npm", "registry.npmjs.org", name, version) + if err := os.MkdirAll(pkgDir, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(pkgDir, "index.js"), []byte("module.exports = '"+version+"';"), 0644); err != nil { + t.Fatal(err) + } + packument := map[string]any{ + "name": name, + "versions": map[string]any{version: map[string]any{"name": name, "version": version}}, + "dist-tags": map[string]string{"latest": version}, + } + data, _ := json.Marshal(packument) + if err := os.WriteFile(filepath.Join(dir, "npm", "registry.npmjs.org", name, "registry.json"), data, 0644); err != nil { + t.Fatal(err) + } + if err := Write(dir, Slice{Name: name, Version: version, Specifier: Specifier(name, version), Dependencies: deps}); err != nil { + t.Fatal(err) + } + return dir +} + +func TestWriteReadRoundTrip(t *testing.T) { + dir := t.TempDir() + want := Slice{Name: "@scope/pkg", Version: "1.2.3", Specifier: "npm:@scope/pkg@1.2.3", Dependencies: map[string]string{"ms": "^2.1.3"}} + if err := Write(dir, want); err != nil { + t.Fatal(err) + } + got, err := Read(dir) + if err != nil || !reflect.DeepEqual(*got, want) { + t.Fatalf("Read = %+v, %v; want %+v", got, err, want) + } +} + +func TestReadRejectsIncompleteMetadata(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, MetadataFile), []byte(`{"name":"x"}`), 0644); err != nil { + t.Fatal(err) + } + if _, err := Read(dir); err == nil { + t.Error("expected an error for metadata without a version") + } + if _, err := Read(t.TempDir()); err == nil { + t.Error("expected an error for a missing file") + } +} + +func TestDiscoverFindsSlicesSorted(t *testing.T) { + root := t.TempDir() + b := makeSlice(t, root, "b", "1.0.0", nil) + a := makeSlice(t, root, "a", "1.0.0", nil) + got := Discover(root) + if !reflect.DeepEqual(got, []string{a, b}) { + t.Errorf("Discover = %v, want [%s %s]", got, a, b) + } + if len(Discover(t.TempDir())) != 0 { + t.Error("an empty tree has no slices") + } +} + +func TestMergeCopiesPackagesIntoDenoDir(t *testing.T) { + root := t.TempDir() + debug := makeSlice(t, root, "debug", "4.3.7", map[string]string{"ms": "^2.1.3"}) + ms := makeSlice(t, root, "ms", "2.1.3", nil) + denoDir := filepath.Join(t.TempDir(), "deno") + + if err := Merge([]string{debug, ms}, denoDir); err != nil { + t.Fatal(err) + } + for _, p := range []string{ + "npm/registry.npmjs.org/debug/4.3.7/index.js", + "npm/registry.npmjs.org/debug/registry.json", + "npm/registry.npmjs.org/ms/2.1.3/index.js", + "npm/registry.npmjs.org/ms/registry.json", + } { + if _, err := os.Stat(filepath.Join(denoDir, p)); err != nil { + t.Errorf("missing %s: %v", p, err) + } + } + if _, err := os.Stat(filepath.Join(denoDir, MetadataFile)); err == nil { + t.Error("slice metadata must not be copied into the cache") + } +} + +func TestMergeUnionsVersionsOfTheSamePackage(t *testing.T) { + root := t.TempDir() + old := makeSlice(t, root, "ms", "2.0.0", nil) + cur := makeSlice(t, root, "ms", "2.1.3", nil) + denoDir := t.TempDir() + + if err := Merge([]string{old, cur}, denoDir); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(denoDir, "npm/registry.npmjs.org/ms/registry.json")) + if err != nil { + t.Fatal(err) + } + var packument struct { + Versions map[string]json.RawMessage `json:"versions"` + } + if err := json.Unmarshal(data, &packument); err != nil { + t.Fatal(err) + } + if len(packument.Versions) != 2 { + t.Errorf("versions = %v, want both 2.0.0 and 2.1.3", packument.Versions) + } + for _, v := range []string{"2.0.0", "2.1.3"} { + if _, err := os.Stat(filepath.Join(denoDir, "npm/registry.npmjs.org/ms", v, "index.js")); err != nil { + t.Errorf("version %s missing: %v", v, err) + } + } +} + +func TestMergeIsIdempotentAndKeepsExistingFiles(t *testing.T) { + root := t.TempDir() + s := makeSlice(t, root, "ms", "2.1.3", nil) + denoDir := t.TempDir() + for i := 0; i < 2; i++ { + if err := Merge([]string{s}, denoDir); err != nil { + t.Fatal(err) + } + } + target := filepath.Join(denoDir, "npm/registry.npmjs.org/ms/2.1.3/index.js") + if err := os.WriteFile(target, []byte("kept"), 0644); err != nil { + t.Fatal(err) + } + if err := Merge([]string{s}, denoDir); err != nil { + t.Fatal(err) + } + if got, _ := os.ReadFile(target); string(got) != "kept" { + t.Errorf("an existing file was overwritten: %q", got) + } +} + +func TestMergeSkipsSlicesWithoutCacheContent(t *testing.T) { + dir := t.TempDir() + if err := Merge([]string{dir}, t.TempDir()); err != nil { + t.Errorf("a directory without npm/ must be skipped, got %v", err) + } +} + +func TestPrepareMergesAndReportsWhetherThereWereSlices(t *testing.T) { + root := t.TempDir() + makeSlice(t, root, "ms", "2.1.3", nil) + denoDir := filepath.Join(t.TempDir(), "deno") + + found, err := Prepare(root, denoDir) + if err != nil || !found { + t.Fatalf("Prepare = %v, %v; want true", found, err) + } + if _, err := os.Stat(filepath.Join(denoDir, "npm/registry.npmjs.org/ms/2.1.3/index.js")); err != nil { + t.Errorf("package not merged: %v", err) + } + + empty := filepath.Join(t.TempDir(), "deno") + found, err = Prepare(t.TempDir(), empty) + if err != nil || found { + t.Fatalf("Prepare without slices = %v, %v; want false", found, err) + } + if _, err := os.Stat(empty); err == nil { + t.Error("the Deno directory must be left alone when there is nothing to merge") + } +} diff --git a/tools/please_ts/registry/BUILD b/tools/please_ts/registry/BUILD new file mode 100644 index 0000000..6f10a65 --- /dev/null +++ b/tools/please_ts/registry/BUILD @@ -0,0 +1,17 @@ +subinclude("///go//build_defs:go") + +go_library( + name = "registry", + srcs = ["registry.go"], + visibility = ["//tools/please_ts/..."], + deps = ["//tools/please_ts/semver"], +) + +go_test( + name = "registry_test", + srcs = [ + "registry.go", + "registry_test.go", + ], + deps = ["//tools/please_ts/semver"], +) diff --git a/tools/please_ts/registry/registry.go b/tools/please_ts/registry/registry.go new file mode 100644 index 0000000..8d0d598 --- /dev/null +++ b/tools/please_ts/registry/registry.go @@ -0,0 +1,282 @@ +// Package registry is a small npm registry client: it reads packuments, resolves a +// dependency specifier to exactly one version, and downloads tarballs verified against +// the integrity data the registry publishes. +package registry + +import ( + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + "time" + + "tools/please_ts/semver" +) + +// DefaultURL is the public npm registry. +const DefaultURL = "https://registry.npmjs.org" + +// maxTarball bounds a download, so a misbehaving registry cannot exhaust memory. +const maxTarball = 512 << 20 + +// Dist is where a version's tarball is and how to verify it. +type Dist struct { + Tarball string `json:"tarball"` + Integrity string `json:"integrity"` // "sha512-", possibly several, space separated + Shasum string `json:"shasum"` // hex sha1, the older field +} + +// VersionInfo is the part of a version's manifest that resolution needs. Dependencies are +// deliberately not read from here: they come from the package.json inside the verified +// tarball, so that a registry cannot claim different dependencies than the package has. +type VersionInfo struct { + Version string `json:"version"` + Dist Dist `json:"dist"` + Deprecated json.RawMessage `json:"deprecated"` +} + +func (v VersionInfo) deprecated() bool { + d := strings.TrimSpace(string(v.Deprecated)) + return d != "" && d != "null" && d != "false" && d != `""` +} + +// Packument lists the versions of a package. +type Packument struct { + Name string `json:"name"` + DistTags map[string]string `json:"dist-tags"` + Versions map[string]VersionInfo `json:"versions"` + Time map[string]string `json:"time"` +} + +// Client talks to an npm registry. +type Client struct { + BaseURL string + HTTP *http.Client +} + +// New returns a client for the registry at baseURL (the public registry if empty). +func New(baseURL string) *Client { + if baseURL == "" { + baseURL = DefaultURL + } + return &Client{BaseURL: strings.TrimSuffix(baseURL, "/"), HTTP: &http.Client{Timeout: 120 * time.Second}} +} + +// Packument fetches the packument of a package. The full document has publish times, which +// an as-of resolution needs; the abbreviated one is much smaller. +func (c *Client) Packument(name string, full bool) (*Packument, error) { + u := c.BaseURL + "/" + strings.Replace(url.PathEscape(name), "%40", "@", 1) + req, err := http.NewRequest("GET", u, nil) + if err != nil { + return nil, err + } + if full { + req.Header.Set("Accept", "application/json") + } else { + req.Header.Set("Accept", "application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8, */*") + } + resp, err := c.HTTP.Do(req) + if err != nil { + return nil, fmt.Errorf("fetching %s: %w", u, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("registry returned status %d for %s", resp.StatusCode, u) + } + var p Packument + if err := json.NewDecoder(resp.Body).Decode(&p); err != nil { + return nil, fmt.Errorf("decoding the packument of %s: %w", name, err) + } + return &p, nil +} + +// Resolve picks the version of a package that a dependency specifier means: a version +// range or a dist-tag. Nothing is guessed: if no version satisfies the specifier it is an +// error. When asOf is set, versions published after it are ignored, which makes the result +// reproducible (this needs the full packument, with publish times). +// +// Among the versions that satisfy a range the choice follows npm: the "latest" dist-tag +// if it qualifies, otherwise the highest version, preferring releases over prereleases +// and versions that are not deprecated. +func Resolve(p *Packument, spec string, asOf time.Time) (VersionInfo, error) { + spec = strings.TrimSpace(spec) + if err := checkSpecifier(spec); err != nil { + return VersionInfo{}, fmt.Errorf("%s: %w", p.Name, err) + } + + published := func(v string) (bool, error) { + if asOf.IsZero() { + return true, nil + } + raw, ok := p.Time[v] + if !ok { + return false, fmt.Errorf("%s@%s has no publish time in the packument; an as-of date needs the full packument", p.Name, v) + } + t, err := time.Parse(time.RFC3339, raw) + if err != nil { + return false, fmt.Errorf("%s@%s: bad publish time %q", p.Name, v, raw) + } + return !t.After(asOf), nil + } + + rng, rangeErr := semver.ParseRange(spec) + if rangeErr != nil { + // Not a range: it may be a dist-tag such as "latest" or "next". + version, ok := p.DistTags[spec] + if !ok { + return VersionInfo{}, fmt.Errorf("%s: %q is neither a version range nor a dist-tag (%v)", p.Name, spec, rangeErr) + } + info, ok := p.Versions[version] + if !ok { + return VersionInfo{}, fmt.Errorf("%s: dist-tag %q points at %s, which is not in the packument", p.Name, spec, version) + } + if ok, err := published(version); err != nil { + return VersionInfo{}, err + } else if !ok { + return VersionInfo{}, fmt.Errorf("%s: dist-tag %q is %s, published after %s", p.Name, spec, version, asOf.Format("2006-01-02")) + } + return info, nil + } + + type candidate struct { + v semver.Version + info VersionInfo + } + var cands []candidate + for raw, info := range p.Versions { + v, err := semver.Parse(raw) + if err != nil || !rng.Satisfies(v) { + continue + } + ok, err := published(raw) + if err != nil { + return VersionInfo{}, err + } + if ok { + cands = append(cands, candidate{v, info}) + } + } + if len(cands) == 0 { + suffix := "" + if !asOf.IsZero() { + suffix = " published by " + asOf.Format("2006-01-02") + } + return VersionInfo{}, fmt.Errorf("no version of %s satisfies %q%s", p.Name, spec, suffix) + } + sort.Slice(cands, func(i, j int) bool { return semver.Compare(cands[i].v, cands[j].v) > 0 }) + + if asOf.IsZero() { + if latest, ok := p.DistTags["latest"]; ok { + for _, c := range cands { + if c.info.Version == latest && !c.info.deprecated() { + return c.info, nil + } + } + } + } + for _, wantStable := range []bool{true, false} { + for _, c := range cands { + if c.info.deprecated() || (wantStable && len(c.v.Pre) > 0) { + continue + } + return c.info, nil + } + } + return cands[0].info, nil // everything that qualifies is deprecated +} + +// checkSpecifier rejects the dependency forms that are not registry versions. +func checkSpecifier(spec string) error { + for _, prefix := range []string{"git", "file:", "link:", "workspace:", "npm:", "github:", "http:", "https:", "/", "./", "../", "~/"} { + if strings.HasPrefix(spec, prefix) { + return fmt.Errorf("unsupported dependency specifier %q (only registry versions are supported)", spec) + } + } + if strings.Contains(spec, "://") { + return fmt.Errorf("unsupported dependency specifier %q (only registry versions are supported)", spec) + } + return nil +} + +// Download fetches the tarball of a version and verifies it against the registry's integrity +// data. A registry that publishes none is an error: the download would be unverifiable. +func (c *Client) Download(name string, v VersionInfo) ([]byte, error) { + if v.Dist.Integrity == "" && v.Dist.Shasum == "" { + return nil, fmt.Errorf("%s@%s: the registry gives no integrity or shasum to verify the tarball with", name, v.Version) + } + tarball := v.Dist.Tarball + if tarball == "" { + base := name + if i := strings.LastIndex(base, "/"); i >= 0 { + base = base[i+1:] + } + tarball = fmt.Sprintf("%s/%s/-/%s-%s.tgz", c.BaseURL, name, base, v.Version) + } + resp, err := c.HTTP.Get(tarball) + if err != nil { + return nil, fmt.Errorf("downloading %s: %w", tarball, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("downloading %s: status %d", tarball, resp.StatusCode) + } + data, err := io.ReadAll(io.LimitReader(resp.Body, maxTarball+1)) + if err != nil { + return nil, fmt.Errorf("downloading %s: %w", tarball, err) + } + if len(data) > maxTarball { + return nil, fmt.Errorf("%s is larger than %d bytes", tarball, maxTarball) + } + if err := Verify(name, v, data); err != nil { + return nil, err + } + return data, nil +} + +// Verify checks data against the integrity field of v (any of its sha256, sha384 or sha512 +// entries; the strongest is required to match) or, when there is none, its sha1 shasum. +func Verify(name string, v VersionInfo, data []byte) error { + if v.Dist.Integrity != "" { + best, bestRank := "", -1 + for _, entry := range strings.Fields(v.Dist.Integrity) { + algo, _, ok := strings.Cut(entry, "-") + rank := map[string]int{"sha256": 1, "sha384": 2, "sha512": 3}[algo] + if ok && rank > bestRank { + best, bestRank = entry, rank + } + } + if bestRank < 0 { + return fmt.Errorf("%s@%s: unsupported integrity %q", name, v.Version, v.Dist.Integrity) + } + algo, want, _ := strings.Cut(best, "-") + var sum []byte + switch algo { + case "sha256": + s := sha256.Sum256(data) + sum = s[:] + case "sha384": + s := sha512.Sum384(data) + sum = s[:] + default: + s := sha512.Sum512(data) + sum = s[:] + } + if base64.StdEncoding.EncodeToString(sum) != want { + return fmt.Errorf("%s@%s: the tarball does not match the registry's %s integrity", name, v.Version, algo) + } + return nil + } + s := sha1.Sum(data) + if hex.EncodeToString(s[:]) != strings.ToLower(v.Dist.Shasum) { + return fmt.Errorf("%s@%s: the tarball does not match the registry's shasum", name, v.Version) + } + return nil +} diff --git a/tools/please_ts/registry/registry_test.go b/tools/please_ts/registry/registry_test.go new file mode 100644 index 0000000..d2d1b71 --- /dev/null +++ b/tools/please_ts/registry/registry_test.go @@ -0,0 +1,242 @@ +package registry + +import ( + "crypto/sha1" + "crypto/sha512" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func packument(t *testing.T, doc string) *Packument { + t.Helper() + var p Packument + if err := json.Unmarshal([]byte(doc), &p); err != nil { + t.Fatal(err) + } + return &p +} + +const msPackument = `{ + "name": "ms", + "dist-tags": {"latest": "2.1.3", "next": "3.0.0-beta.1"}, + "versions": { + "1.0.0": {"version": "1.0.0"}, + "2.0.0": {"version": "2.0.0"}, + "2.1.2": {"version": "2.1.2"}, + "2.1.3": {"version": "2.1.3"}, + "2.2.0": {"version": "2.2.0"}, + "2.3.0": {"version": "2.3.0", "deprecated": "use 2.2.0"}, + "3.0.0-beta.1": {"version": "3.0.0-beta.1"} + }, + "time": { + "1.0.0": "2015-01-01T00:00:00.000Z", "2.0.0": "2017-01-01T00:00:00.000Z", + "2.1.2": "2019-01-01T00:00:00.000Z", "2.1.3": "2020-01-01T00:00:00.000Z", + "2.2.0": "2023-01-01T00:00:00.000Z", "2.3.0": "2024-01-01T00:00:00.000Z", + "3.0.0-beta.1": "2024-06-01T00:00:00.000Z" + } +}` + +func TestResolve(t *testing.T) { + p := packument(t, msPackument) + date := func(s string) time.Time { + d, err := time.Parse("2006-01-02", s) + if err != nil { + t.Fatal(err) + } + return d.Add(24*time.Hour - time.Second) + } + for _, tc := range []struct { + name, spec string + asOf time.Time + want string + }{ + {"the latest tag when it satisfies", "^2.0.0", time.Time{}, "2.1.3"}, + {"highest when latest does not satisfy", "^1.0.0", time.Time{}, "1.0.0"}, + {"highest satisfying, skipping a deprecated one", "~2.2.0", time.Time{}, "2.2.0"}, + {"exact", "2.1.2", time.Time{}, "2.1.2"}, + {"star prefers latest", "*", time.Time{}, "2.1.3"}, + {"empty prefers latest", "", time.Time{}, "2.1.3"}, + {"a range of alternatives", "1.0.0 || 2.0.0", time.Time{}, "2.0.0"}, + {"a dist-tag", "latest", time.Time{}, "2.1.3"}, + {"another dist-tag", "next", time.Time{}, "3.0.0-beta.1"}, + {"a range that allows a prerelease", ">=3.0.0-beta.0", time.Time{}, "3.0.0-beta.1"}, + {"as of a date ignores the latest tag", "^2.0.0", date("2019-06-01"), "2.1.2"}, + {"as of a date, the newest published by then", "^2.0.0", date("2023-06-01"), "2.2.0"}, + {"as of the day of a release includes it", "^2.0.0", date("2020-01-01"), "2.1.3"}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := Resolve(p, tc.spec, tc.asOf) + if err != nil || got.Version != tc.want { + t.Errorf("Resolve(%q) = %q, %v; want %s", tc.spec, got.Version, err, tc.want) + } + }) + } +} + +func TestResolveNeverGuesses(t *testing.T) { + p := packument(t, msPackument) + for name, tc := range map[string]struct{ spec, want string }{ + "no version satisfies": {"^9.0.0", "no version of ms satisfies"}, + "only a prerelease exists": {"^3.0.0", "no version of ms satisfies"}, + "not a range or a tag": {"banana", "neither a version range nor a dist-tag"}, + "git dependency": {"git+https://github.com/x/y.git", "unsupported dependency specifier"}, + "github shorthand": {"github:x/y", "unsupported dependency specifier"}, + "file dependency": {"file:../x", "unsupported dependency specifier"}, + "npm alias": {"npm:other@^1", "unsupported dependency specifier"}, + "workspace": {"workspace:*", "unsupported dependency specifier"}, + "url": {"https://example.com/x.tgz", "unsupported dependency specifier"}, + "relative path": {"./local", "unsupported dependency specifier"}, + } { + t.Run(name, func(t *testing.T) { + if got, err := Resolve(p, tc.spec, time.Time{}); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("Resolve(%q) = %q, %v; want an error containing %q", tc.spec, got.Version, err, tc.want) + } + }) + } +} + +func TestResolveAsOfErrors(t *testing.T) { + p := packument(t, msPackument) + asOf := time.Date(2016, 1, 1, 0, 0, 0, 0, time.UTC) + if _, err := Resolve(p, "^2.0.0", asOf); err == nil || !strings.Contains(err.Error(), "published by 2016-01-01") { + t.Errorf("nothing was published by then, got %v", err) + } + if _, err := Resolve(p, "latest", asOf); err == nil || !strings.Contains(err.Error(), "published after") { + t.Errorf("a dist-tag newer than the date must be an error, got %v", err) + } + abbreviated := packument(t, `{"name":"ms","versions":{"1.0.0":{"version":"1.0.0"}}}`) + if _, err := Resolve(abbreviated, "^1.0.0", asOf); err == nil || !strings.Contains(err.Error(), "no publish time") { + t.Errorf("an as-of date needs publish times, got %v", err) + } +} + +func TestResolveFallsBackToADeprecatedVersionOnlyWhenNothingElseQualifies(t *testing.T) { + p := packument(t, `{"name":"x","versions":{"1.0.0":{"version":"1.0.0","deprecated":"old"},"1.1.0":{"version":"1.1.0","deprecated":"older"}}}`) + got, err := Resolve(p, "^1.0.0", time.Time{}) + if err != nil || got.Version != "1.1.0" { + t.Errorf("Resolve = %q, %v; want the highest deprecated version", got.Version, err) + } +} + +func integrityOf(data []byte) string { + s := sha512.Sum512(data) + return "sha512-" + base64.StdEncoding.EncodeToString(s[:]) +} + +func TestVerify(t *testing.T) { + data := []byte("tarball bytes") + sha1sum := sha1.Sum(data) + shasum := hex.EncodeToString(sha1sum[:]) + for _, tc := range []struct { + name string + dist Dist + ok bool + }{ + {"sha512 integrity", Dist{Integrity: integrityOf(data)}, true}, + {"integrity wins over a wrong shasum", Dist{Integrity: integrityOf(data), Shasum: "00"}, true}, + {"several integrity entries, the strongest matches", Dist{Integrity: "sha256-AAAA " + integrityOf(data)}, true}, + {"the strongest does not match", Dist{Integrity: "sha256-AAAA sha512-AAAA"}, false}, + {"wrong integrity", Dist{Integrity: "sha512-AAAA", Shasum: shasum}, false}, + {"shasum alone", Dist{Shasum: shasum}, true}, + {"uppercase shasum", Dist{Shasum: strings.ToUpper(shasum)}, true}, + {"wrong shasum", Dist{Shasum: "0000"}, false}, + {"unsupported algorithm", Dist{Integrity: "md5-AAAA"}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + err := Verify("x", VersionInfo{Version: "1.0.0", Dist: tc.dist}, data) + if (err == nil) != tc.ok { + t.Errorf("Verify = %v, want ok=%v", err, tc.ok) + } + }) + } +} + +func newRegistry(t *testing.T, handler func(w http.ResponseWriter, r *http.Request)) *Client { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(handler)) + t.Cleanup(srv.Close) + return New(srv.URL) +} + +func TestPackumentFetchesAndEscapesScopedNames(t *testing.T) { + var gotPath, gotAccept string + c := newRegistry(t, func(w http.ResponseWriter, r *http.Request) { + gotPath, gotAccept = r.URL.EscapedPath(), r.Header.Get("Accept") + fmt.Fprint(w, msPackument) + }) + p, err := c.Packument("@scope/ms", true) + if err != nil || p.Name != "ms" || len(p.Versions) != 7 { + t.Fatalf("Packument = %+v, %v", p, err) + } + if gotPath != "/@scope%2Fms" && gotPath != "/@scope/ms" { + t.Errorf("path = %q", gotPath) + } + if gotAccept != "application/json" { + t.Errorf("a full packument must ask for plain JSON, Accept = %q", gotAccept) + } + if _, err := c.Packument("ms", false); err != nil || !strings.Contains(gotAccept, "install-v1") { + t.Errorf("an abbreviated packument must ask for the install format, Accept = %q, %v", gotAccept, err) + } +} + +func TestPackumentErrors(t *testing.T) { + c := newRegistry(t, func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/bad") { + fmt.Fprint(w, "{") + return + } + http.NotFound(w, r) + }) + if _, err := c.Packument("missing", false); err == nil || !strings.Contains(err.Error(), "404") { + t.Errorf("a 404 must be reported, got %v", err) + } + if _, err := c.Packument("bad", false); err == nil { + t.Error("invalid JSON must be an error") + } +} + +func TestDownloadVerifiesTheTarball(t *testing.T) { + data := []byte("the real tarball") + var served = data + c := newRegistry(t, func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(served) }) + + good := VersionInfo{Version: "1.0.0", Dist: Dist{Tarball: c.BaseURL + "/x/-/x-1.0.0.tgz", Integrity: integrityOf(data)}} + if got, err := c.Download("x", good); err != nil || string(got) != string(data) { + t.Fatalf("Download = %q, %v", got, err) + } + + served = []byte("a different tarball") + if _, err := c.Download("x", good); err == nil || !strings.Contains(err.Error(), "does not match") { + t.Errorf("tampered content must fail verification, got %v", err) + } + + noIntegrity := VersionInfo{Version: "1.0.0", Dist: Dist{Tarball: c.BaseURL + "/x/-/x-1.0.0.tgz"}} + if _, err := c.Download("x", noIntegrity); err == nil || !strings.Contains(err.Error(), "no integrity") { + t.Errorf("a version without integrity data must be refused, got %v", err) + } +} + +func TestDownloadBuildsTheTarballURLWhenTheRegistryOmitsIt(t *testing.T) { + data := []byte("x") + var path string + c := newRegistry(t, func(w http.ResponseWriter, r *http.Request) { path = r.URL.Path; _, _ = w.Write(data) }) + v := VersionInfo{Version: "2.0.0", Dist: Dist{Integrity: integrityOf(data)}} + if _, err := c.Download("@scope/pkg", v); err != nil || path != "/@scope/pkg/-/pkg-2.0.0.tgz" { + t.Errorf("path = %q, err = %v", path, err) + } +} + +func TestDownloadReportsHTTPFailures(t *testing.T) { + c := newRegistry(t, func(w http.ResponseWriter, r *http.Request) { http.Error(w, "boom", http.StatusInternalServerError) }) + v := VersionInfo{Version: "1.0.0", Dist: Dist{Tarball: c.BaseURL + "/x.tgz", Shasum: "00"}} + if _, err := c.Download("x", v); err == nil || !strings.Contains(err.Error(), "500") { + t.Errorf("got %v", err) + } +} diff --git a/tools/please_ts/semver/BUILD b/tools/please_ts/semver/BUILD new file mode 100644 index 0000000..eca299a --- /dev/null +++ b/tools/please_ts/semver/BUILD @@ -0,0 +1,15 @@ +subinclude("///go//build_defs:go") + +go_library( + name = "semver", + srcs = ["semver.go"], + visibility = ["//tools/please_ts/..."], +) + +go_test( + name = "semver_test", + srcs = [ + "semver.go", + "semver_test.go", + ], +) diff --git a/tools/please_ts/semver/semver.go b/tools/please_ts/semver/semver.go new file mode 100644 index 0000000..0aeae84 --- /dev/null +++ b/tools/please_ts/semver/semver.go @@ -0,0 +1,415 @@ +// Package semver implements the parts of npm's semantic versioning that dependency +// resolution needs: versions with prereleases, and ranges with ^, ~, comparators, +// x-ranges, hyphen ranges and ||. +package semver + +import ( + "fmt" + "strconv" + "strings" +) + +// Version is a parsed semantic version. Build metadata is ignored, as it is for +// precedence. +type Version struct { + Major, Minor, Patch uint64 + Pre []string // prerelease identifiers +} + +// Parse parses "1.2.3", "v1.2.3" or "1.2.3-beta.1+build". All three numbers are required. +func Parse(s string) (Version, error) { + orig := s + s = strings.TrimSpace(s) + s = strings.TrimPrefix(strings.TrimPrefix(s, "="), "v") + if i := strings.IndexByte(s, '+'); i >= 0 { + s = s[:i] + } + var pre string + if i := strings.IndexByte(s, '-'); i >= 0 { + s, pre = s[:i], s[i+1:] + if pre == "" { + return Version{}, fmt.Errorf("invalid version %q", orig) + } + } + parts := strings.Split(s, ".") + if len(parts) != 3 { + return Version{}, fmt.Errorf("invalid version %q", orig) + } + var nums [3]uint64 + for i, p := range parts { + n, err := parseNum(p) + if err != nil { + return Version{}, fmt.Errorf("invalid version %q", orig) + } + nums[i] = n + } + v := Version{Major: nums[0], Minor: nums[1], Patch: nums[2]} + if pre != "" { + v.Pre = strings.Split(pre, ".") + for _, id := range v.Pre { + if id == "" { + return Version{}, fmt.Errorf("invalid version %q", orig) + } + } + } + return v, nil +} + +func parseNum(s string) (uint64, error) { + if s == "" || (len(s) > 1 && s[0] == '0') { + return 0, fmt.Errorf("bad number") + } + return strconv.ParseUint(s, 10, 64) +} + +func (v Version) String() string { + s := fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch) + if len(v.Pre) > 0 { + s += "-" + strings.Join(v.Pre, ".") + } + return s +} + +// Compare returns -1, 0 or 1 following semver precedence: a prerelease sorts before its +// release, and prerelease identifiers compare numerically when numeric. +func Compare(a, b Version) int { + for _, d := range [][2]uint64{{a.Major, b.Major}, {a.Minor, b.Minor}, {a.Patch, b.Patch}} { + if d[0] != d[1] { + if d[0] < d[1] { + return -1 + } + return 1 + } + } + switch { + case len(a.Pre) == 0 && len(b.Pre) == 0: + return 0 + case len(a.Pre) == 0: + return 1 + case len(b.Pre) == 0: + return -1 + } + for i := 0; i < len(a.Pre) && i < len(b.Pre); i++ { + if c := comparePre(a.Pre[i], b.Pre[i]); c != 0 { + return c + } + } + switch { + case len(a.Pre) < len(b.Pre): + return -1 + case len(a.Pre) > len(b.Pre): + return 1 + } + return 0 +} + +func comparePre(a, b string) int { + an, aerr := strconv.ParseUint(a, 10, 64) + bn, berr := strconv.ParseUint(b, 10, 64) + switch { + case aerr == nil && berr == nil: + if an < bn { + return -1 + } else if an > bn { + return 1 + } + return 0 + case aerr == nil: // numeric identifiers sort before alphanumeric ones + return -1 + case berr == nil: + return 1 + } + return strings.Compare(a, b) +} + +type comparator struct { + op string // "<", "<=", ">", ">=", "=" + v Version +} + +func (c comparator) test(v Version) bool { + cmp := Compare(v, c.v) + switch c.op { + case "<": + return cmp < 0 + case "<=": + return cmp <= 0 + case ">": + return cmp > 0 + case ">=": + return cmp >= 0 + } + return cmp == 0 +} + +// Range is a parsed npm version range: alternatives (||) of comparator sets. +type Range struct { + sets [][]comparator +} + +// ParseRange parses an npm range such as "^1.2.3", "~1.2", ">=1 <3", "1.x || ^2", "1.2.3 - 2.3.4" +// or "*". The empty string means "*". +func ParseRange(s string) (Range, error) { + var r Range + for _, alt := range strings.Split(s, "||") { + set, err := parseSet(strings.TrimSpace(alt)) + if err != nil { + return Range{}, fmt.Errorf("invalid range %q: %w", s, err) + } + r.sets = append(r.sets, set) + } + return r, nil +} + +// Satisfies reports whether v is in the range. A prerelease version only satisfies a +// comparator set that has a comparator with a prerelease on the same major.minor.patch. +func (r Range) Satisfies(v Version) bool { + for _, set := range r.sets { + if satisfiesSet(set, v) { + return true + } + } + return false +} + +func satisfiesSet(set []comparator, v Version) bool { + for _, c := range set { + if !c.test(v) { + return false + } + } + if len(v.Pre) == 0 { + return true + } + for _, c := range set { + if len(c.v.Pre) > 0 && c.v.Major == v.Major && c.v.Minor == v.Minor && c.v.Patch == v.Patch { + return true + } + } + return false +} + +// partial is a version with optional minor and patch ("1", "1.2", "1.2.x", "1.2.3-beta"). +type partial struct { + major, minor, patch int64 // -1 when missing or a wildcard + pre []string +} + +func parsePartial(s string) (partial, error) { + s = strings.TrimSpace(s) + s = strings.TrimPrefix(strings.TrimPrefix(s, "="), "v") + if s == "" || s == "*" || s == "x" || s == "X" { + return partial{-1, -1, -1, nil}, nil + } + if i := strings.IndexByte(s, '+'); i >= 0 { + s = s[:i] + } + var pre []string + if i := strings.IndexByte(s, '-'); i >= 0 { + pre = strings.Split(s[i+1:], ".") + s = s[:i] + } + parts := strings.Split(s, ".") + if len(parts) > 3 { + return partial{}, fmt.Errorf("bad version %q", s) + } + p := partial{-1, -1, -1, pre} + slots := []*int64{&p.major, &p.minor, &p.patch} + wildcard := false + for i, part := range parts { + if part == "x" || part == "X" || part == "*" { + wildcard = true // a wildcard opens everything after it + continue + } + if wildcard { + return partial{}, fmt.Errorf("bad version %q: a number follows a wildcard", s) + } + n, err := strconv.ParseInt(part, 10, 64) + if err != nil || n < 0 { + return partial{}, fmt.Errorf("bad version %q", s) + } + *slots[i] = n + } + if (p.minor < 0 && p.patch >= 0) || (p.major < 0 && p.minor >= 0) { + return partial{}, fmt.Errorf("bad version %q", s) + } + return p, nil +} + +func (p partial) version(fill uint64) Version { + v := Version{Pre: p.pre} + nums := []*uint64{&v.Major, &v.Minor, &v.Patch} + for i, n := range []int64{p.major, p.minor, p.patch} { + if n >= 0 { + *nums[i] = uint64(n) + } else { + *nums[i] = fill + } + } + return v +} + +// floor is the lowest version the partial denotes ("1.2" is 1.2.0). +func (p partial) floor() Version { return p.version(0) } + +// ceiling returns the exclusive upper bound of the partial ("1.2" is <1.3.0-0). +func (p partial) ceiling() (Version, bool) { + switch { + case p.major < 0: + return Version{}, false + case p.minor < 0: + return Version{Major: uint64(p.major) + 1, Pre: []string{"0"}}, true + case p.patch < 0: + return Version{Major: uint64(p.major), Minor: uint64(p.minor) + 1, Pre: []string{"0"}}, true + } + return Version{}, false +} + +func parseSet(s string) ([]comparator, error) { + if s == "" { + return nil, nil // "*" + } + fields := strings.Fields(s) + // Hyphen range: "A - B". + if len(fields) == 3 && fields[1] == "-" { + lo, err := parsePartial(fields[0]) + if err != nil { + return nil, err + } + hi, err := parsePartial(fields[2]) + if err != nil { + return nil, err + } + var set []comparator + if lo.major >= 0 { + set = append(set, comparator{">=", lo.floor()}) + } + if c, ok := hi.ceiling(); ok { + set = append(set, comparator{"<", c}) + } else if hi.major >= 0 { + set = append(set, comparator{"<=", hi.floor()}) + } + return set, nil + } + + // "> 1.2.3" is written with a space in some manifests: glue a lone operator to the next field. + var glued []string + for i := 0; i < len(fields); i++ { + f := fields[i] + if isOperator(f) && i+1 < len(fields) { + f += fields[i+1] + i++ + } + glued = append(glued, f) + } + + var set []comparator + for _, f := range glued { + cs, err := parseComparator(f) + if err != nil { + return nil, err + } + set = append(set, cs...) + } + return set, nil +} + +func isOperator(s string) bool { + switch s { + case "<", "<=", ">", ">=", "=", "^", "~", "~>": + return true + } + return false +} + +func parseComparator(f string) ([]comparator, error) { + op := "" + for _, o := range []string{"~>", "<=", ">=", "<", ">", "=", "^", "~"} { + if strings.HasPrefix(f, o) { + op = o + f = f[len(o):] + break + } + } + p, err := parsePartial(f) + if err != nil { + return nil, err + } + + switch op { + case "^": + return caret(p), nil + case "~", "~>": + return tilde(p), nil + case ">": + if c, ok := p.ceiling(); ok { + return []comparator{{">=", withoutPre(c)}}, nil + } + if p.major < 0 { + return []comparator{{"<", Version{}}}, nil // ">*" matches nothing + } + return []comparator{{">", p.floor()}}, nil + case ">=": + if p.major < 0 { + return nil, nil + } + return []comparator{{">=", p.floor()}}, nil + case "<": + if p.major < 0 { + return []comparator{{"<", Version{}}}, nil // "<*" matches nothing + } + return []comparator{{"<", p.floor()}}, nil + case "<=": + if c, ok := p.ceiling(); ok { + return []comparator{{"<", c}}, nil + } + if p.major < 0 { + return nil, nil + } + return []comparator{{"<=", p.floor()}}, nil + } + + // No operator or "=": an exact version or an x-range. + if p.major < 0 { + return nil, nil + } + if c, ok := p.ceiling(); ok { + return []comparator{{">=", p.floor()}, {"<", c}}, nil + } + return []comparator{{"=", p.floor()}}, nil +} + +func withoutPre(v Version) Version { v.Pre = nil; return v } + +func caret(p partial) []comparator { + if p.major < 0 { + return nil + } + lo := p.floor() + var hi Version + switch { + case p.major > 0 || (p.minor < 0): + hi = Version{Major: lo.Major + 1} + case p.minor > 0 || p.patch < 0: + hi = Version{Major: 0, Minor: lo.Minor + 1} + default: + hi = Version{Major: 0, Minor: 0, Patch: lo.Patch + 1} + } + hi.Pre = []string{"0"} + return []comparator{{">=", lo}, {"<", hi}} +} + +func tilde(p partial) []comparator { + if p.major < 0 { + return nil + } + lo := p.floor() + var hi Version + if p.minor < 0 { + hi = Version{Major: lo.Major + 1} + } else { + hi = Version{Major: lo.Major, Minor: lo.Minor + 1} + } + hi.Pre = []string{"0"} + return []comparator{{">=", lo}, {"<", hi}} +} diff --git a/tools/please_ts/semver/semver_test.go b/tools/please_ts/semver/semver_test.go new file mode 100644 index 0000000..fed79f3 --- /dev/null +++ b/tools/please_ts/semver/semver_test.go @@ -0,0 +1,119 @@ +package semver + +import "testing" + +func TestParse(t *testing.T) { + for in, want := range map[string]string{ + "1.2.3": "1.2.3", + "v1.2.3": "1.2.3", + "=1.2.3": "1.2.3", + " 1.2.3 ": "1.2.3", + "1.2.3-beta.1": "1.2.3-beta.1", + "1.2.3-beta.1+b.5": "1.2.3-beta.1", + "1.2.3+build": "1.2.3", + "0.0.0": "0.0.0", + } { + v, err := Parse(in) + if err != nil || v.String() != want { + t.Errorf("Parse(%q) = %v, %v; want %s", in, v, err, want) + } + } + for _, in := range []string{"", "1.2", "1.2.3.4", "a.b.c", "01.2.3", "1.2.3-", "1.2.3-a..b", "1.x.3", ">1.2.3", "-1.2.3"} { + if _, err := Parse(in); err == nil { + t.Errorf("Parse(%q): expected an error", in) + } + } +} + +func TestComparePrecedence(t *testing.T) { + // The ordering example of the semver specification, lowest first. + ordered := []string{"1.0.0-alpha", "1.0.0-alpha.1", "1.0.0-alpha.beta", "1.0.0-beta", "1.0.0-beta.2", "1.0.0-beta.11", "1.0.0-rc.1", "1.0.0", "1.0.1", "1.1.0", "2.0.0", "10.0.0"} + for i := range ordered { + for j := range ordered { + a, _ := Parse(ordered[i]) + b, _ := Parse(ordered[j]) + want := 0 + if i < j { + want = -1 + } else if i > j { + want = 1 + } + if got := Compare(a, b); got != want { + t.Errorf("Compare(%s, %s) = %d, want %d", ordered[i], ordered[j], got, want) + } + } + } + a, _ := Parse("1.0.0+a") + b, _ := Parse("1.0.0+b") + if Compare(a, b) != 0 { + t.Error("build metadata must not affect precedence") + } +} + +func TestRangeIncludes(t *testing.T) { + for _, tc := range [][2]string{ + {"1.0.0 - 2.0.0", "1.2.3"}, {"1.0.0 - 2.0.0", "2.0.0"}, {"1.2.3 - 2.3", "2.3.9"}, {"1.2 - 2.3.4", "1.2.0"}, {"1.2.3 - 2", "2.9.9"}, + {"^1.2.3", "1.2.3"}, {"^1.2.3", "1.9.9"}, {"^1.2", "1.9.0"}, {"^1.2.x", "1.2.0"}, {"^1", "1.0.0"}, + {"^0.2.3", "0.2.9"}, {"^0.0.3", "0.0.3"}, {"^0.0", "0.0.9"}, {"^0", "0.9.9"}, {"^0.x", "0.5.0"}, + {"~1.2.3", "1.2.9"}, {"~1.2", "1.2.0"}, {"~1", "1.9.9"}, {"~>1.2.3", "1.2.4"}, + {"1.2.x", "1.2.3"}, {"1.x", "1.9.0"}, {"1.X", "1.0.0"}, {"*", "1.2.3"}, {"x", "9.9.9"}, {"", "1.0.0"}, {"^", "1.0.0"}, + {"1", "1.5.0"}, {"1.2", "1.2.9"}, {"=1.2.3", "1.2.3"}, {"v1.2.3", "1.2.3"}, {"1.2.3", "1.2.3"}, + {">=1.0.0", "1.0.0"}, {">1.0.0", "1.0.1"}, {"<2.0.0", "1.9.9"}, {"<=2.0.0", "2.0.0"}, {"<=2", "2.9.9"}, {"<=1.2", "1.2.9"}, + {">1.2", "1.3.0"}, {">1", "2.0.0"}, {">=1.2", "1.2.0"}, {"<1.2", "1.1.9"}, + {">=1.2.3 <2", "1.9.9"}, {">= 1.2.3", "1.2.3"}, {"> 1.2.3 < 2.0.0", "1.5.0"}, + {"1.2.3 || 2.0.0", "2.0.0"}, {"^1 || ^2", "2.5.0"}, {"<1 || >=3", "3.1.0"}, + {"^1.2.3-beta.2", "1.2.3-beta.4"}, {"^1.2.3-beta.2", "1.3.0"}, {">=1.2.3-beta.2", "1.2.3-beta.4"}, {"1.2.3-beta.2", "1.2.3-beta.2"}, + } { + r, err := ParseRange(tc[0]) + if err != nil { + t.Errorf("ParseRange(%q): %v", tc[0], err) + continue + } + v, _ := Parse(tc[1]) + if !r.Satisfies(v) { + t.Errorf("%q should include %s", tc[0], tc[1]) + } + } +} + +func TestRangeExcludes(t *testing.T) { + for _, tc := range [][2]string{ + {"^1.2.3", "2.0.0"}, {"^1.2.3", "1.2.2"}, {"^0.2.3", "0.3.0"}, {"^0.0.3", "0.0.4"}, {"^0.0", "0.1.0"}, {"^0", "1.0.0"}, + {"~1.2.3", "1.3.0"}, {"~1.2", "1.3.0"}, {"~1", "2.0.0"}, + {"1.2.x", "1.3.0"}, {"1.x", "2.0.0"}, {"1", "2.0.0"}, {"1.2", "1.3.0"}, {"1.2.3", "1.2.4"}, + {">1.0.0", "1.0.0"}, {"<1.0.0", "1.0.0"}, {">1.2", "1.2.9"}, {">1", "1.9.9"}, {"<=1.2", "1.3.0"}, + {"1.2.3 - 2.3.4", "2.3.5"}, {"1.2.3 - 2.3", "2.4.0"}, {"1.2.3 - 2", "3.0.0"}, {"1.0.0 - 2.0.0", "0.9.9"}, + {"1.2.3 || 2.0.0", "1.2.4"}, {">=1 <3", "3.0.0"}, {"<1 || >=3", "2.0.0"}, + // Prereleases only match a comparator that names a prerelease of the same version. + {"^1.2.3", "1.2.4-beta.1"}, {"^1.2.3-beta.2", "1.2.4-beta.1"}, {"*", "1.2.3-beta.1"}, {">=1.0.0", "2.0.0-alpha"}, + {"<2.0.0", "2.0.0-alpha"}, {"^1.2.3-beta.2", "1.2.3-beta.1"}, {"1.x", "1.2.3-beta.1"}, + } { + r, err := ParseRange(tc[0]) + if err != nil { + t.Errorf("ParseRange(%q): %v", tc[0], err) + continue + } + v, _ := Parse(tc[1]) + if r.Satisfies(v) { + t.Errorf("%q should not include %s", tc[0], tc[1]) + } + } +} + +func TestParseRangeErrors(t *testing.T) { + for _, in := range []string{"1.2.3.4", "a", "1.x.3", ">=a", "^a.b", "1.2.3 - ", "||x.y"} { + if _, err := ParseRange(in); err == nil { + t.Errorf("ParseRange(%q): expected an error", in) + } + } +} + +func TestRangeNothingMatchesStarComparators(t *testing.T) { + v, _ := Parse("1.0.0") + for _, in := range []string{">*", "<*"} { + r, err := ParseRange(in) + if err != nil || r.Satisfies(v) { + t.Errorf("%q: err=%v includes=%v; want a range that matches nothing", in, err, r.Satisfies(v)) + } + } +} diff --git a/tools/please_ts/testrunner/BUILD b/tools/please_ts/testrunner/BUILD index b443715..4cc5484 100644 --- a/tools/please_ts/testrunner/BUILD +++ b/tools/please_ts/testrunner/BUILD @@ -8,6 +8,7 @@ go_library( "//tools/common/lcov", "//tools/please_ts/bundle", "//tools/please_ts/importmap", + "//tools/please_ts/npmcache", ], ) @@ -18,5 +19,6 @@ go_test( "//tools/common/lcov", "//tools/please_ts/bundle", "//tools/please_ts/importmap", + "//tools/please_ts/npmcache", ], ) diff --git a/tools/please_ts/testrunner/deno.go b/tools/please_ts/testrunner/deno.go index acf863c..5983368 100644 --- a/tools/please_ts/testrunner/deno.go +++ b/tools/please_ts/testrunner/deno.go @@ -8,6 +8,7 @@ import ( "strings" "tools/please_ts/importmap" + "tools/please_ts/npmcache" ) func (opts RunOptions) runDeno(resultsFile string) error { @@ -29,6 +30,12 @@ func (opts RunOptions) runDeno(resultsFile string) error { return fmt.Errorf("failed creating DENO_DIR: %w", err) } + // npm packages provided by ts_npm_module targets are merged into the per-run Deno cache + // and resolved from there, offline. + if _, err := npmcache.Prepare(".", denoCacheDir); err != nil { + return fmt.Errorf("failed preparing the npm cache: %w", err) + } + // 1. Synthesize target-local import map importMapPath := ".import_map.json" im, err := importmap.Synthesize(opts.ModuleName, opts.Srcs, opts.Deps, ".") @@ -52,6 +59,11 @@ func (opts RunOptions) runDeno(resultsFile string) error { "--import-map", importMapPath, "--junit-path", resultsFile, } + // npm packages come from the merged cache only: a ts_npm_module missing from the + // target's deps must fail here, not be downloaded. + if im.HasNpmSpecifiers() { + args = append(args, "--cached-only") + } covDir := "" if coverageActive { diff --git a/tools/please_ts/unpack/BUILD b/tools/please_ts/unpack/BUILD index 80f2af6..659a418 100644 --- a/tools/please_ts/unpack/BUILD +++ b/tools/please_ts/unpack/BUILD @@ -2,16 +2,34 @@ subinclude("///go//build_defs:go") go_library( name = "unpack", - srcs = ["unpack.go"], + srcs = [ + "npmcache.go", + "resolve.go", + "unpack.go", + ], visibility = ["//tools/please_ts/..."], - deps = ["//tools/please_ts/importmap"], + deps = [ + "//tools/please_ts/importmap", + "//tools/please_ts/npmcache", + "//tools/please_ts/registry", + "//tools/please_ts/semver", + ], ) go_test( name = "unpack_test", srcs = [ + "npmcache.go", + "npmcache_test.go", + "resolve.go", + "resolve_test.go", "unpack.go", "unpack_test.go", ], - deps = ["//tools/please_ts/importmap"], + deps = [ + "//tools/please_ts/importmap", + "//tools/please_ts/npmcache", + "//tools/please_ts/registry", + "//tools/please_ts/semver", + ], ) diff --git a/tools/please_ts/unpack/npmcache.go b/tools/please_ts/unpack/npmcache.go new file mode 100644 index 0000000..8b61e51 --- /dev/null +++ b/tools/please_ts/unpack/npmcache.go @@ -0,0 +1,293 @@ +package unpack + +import ( + "crypto/sha512" + "encoding/base64" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "tools/please_ts/npmcache" +) + +// buildNpmCache turns an npm tarball into a slice of a Deno npm cache: the package is +// extracted into npm/registry.npmjs.org/// and described by a minimal +// registry.json, so that Deno resolves npm:@ from its cache. Deno does not +// verify the cache, so the integrity of the package is the sha256 Please checked on the +// tarball. +// +// The dependencies of the package come from other slices staged in the build directory (a +// ts_npm_module in deps), whose packages are bundled into this slice. With +// ResolveTransitive, dependencies that no staged slice provides are resolved from the +// registry instead, each verified against the registry's integrity data, and kept as +// separate versions when dependents need different ones. Without it every dependency must +// be provided, and the build fails naming those that are not, since Deno would only notice +// at run time, by trying the network. +func buildNpmCache(opts Options) error { + data, err := os.ReadFile(opts.ArchivePath()) + if err != nil { + return err + } + pkg, _, err := inspectTarball(data, opts.Name) + if err != nil { + return fmt.Errorf("%s: %w", opts.ArchivePath(), err) + } + + staged, err := stagedSlices(opts.Out) + if err != nil { + return err + } + + if opts.ResolveTransitive && len(pkg.Dependencies) > 0 { + tmp, err := os.MkdirTemp("", "please_ts_resolved_*") + if err != nil { + return err + } + defer os.RemoveAll(tmp) + resolved, err := resolveIntoSlices(pkg, staged, newResolver(opts.RegistryURL()), tmp) + if err != nil { + return fmt.Errorf("failed resolving the dependencies of %s: %w", pkg.Name, err) + } + staged = append(staged, resolved...) + } + + if err := checkDependencyClosure(pkg, staged); err != nil { + return err + } + if _, err := writeSlice(data, opts.Name, opts.Out); err != nil { + return err + } + + // Bundle the dependencies' packages into this slice, so that a target only needs to list + // this module, like ts_module: Please stages the direct dependencies of a target, not + // the transitive ones. Each staged slice already contains its own dependencies. + dirs := make([]string, 0, len(staged)) + for _, s := range staged { + dirs = append(dirs, s.dir) + } + if err := npmcache.Merge(dirs, opts.Out); err != nil { + return fmt.Errorf("failed bundling dependencies: %w", err) + } + + return npmcache.Write(opts.Out, npmcache.Slice{ + Name: pkg.Name, + Version: pkg.Version, + Specifier: npmcache.Specifier(pkg.Name, pkg.Version), + Dependencies: pkg.Dependencies, + }) +} + +// inspectTarball extracts a tarball to a scratch directory and reads its package.json, +// returning the typed fields and the raw object (to carry dependency fields into +// registry.json unchanged). wantName, when set, must be the package the tarball holds. +func inspectTarball(data []byte, wantName string) (PackageJSON, map[string]json.RawMessage, error) { + var pkg PackageJSON + tmp, err := os.MkdirTemp("", "please_ts_npm_*") + if err != nil { + return pkg, nil, err + } + defer os.RemoveAll(tmp) + if err := extractTarballBytes(data, tmp); err != nil { + return pkg, nil, fmt.Errorf("failed extracting the tarball: %w", err) + } + manifest, err := os.ReadFile(filepath.Join(tmp, "package.json")) + if err != nil { + return pkg, nil, fmt.Errorf("package.json not found in the tarball: %w", err) + } + if err := json.Unmarshal(manifest, &pkg); err != nil { + return pkg, nil, fmt.Errorf("invalid package.json: %w", err) + } + var raw map[string]json.RawMessage + if err := json.Unmarshal(manifest, &raw); err != nil { + return pkg, nil, err + } + if pkg.Name == "" || pkg.Version == "" { + return pkg, nil, fmt.Errorf("package.json has no name or version") + } + if wantName != "" && wantName != pkg.Name { + return pkg, nil, fmt.Errorf("the tarball holds package %q, expected %q", pkg.Name, wantName) + } + return pkg, raw, nil +} + +// writeSlice lays a tarball out as a slice in outDir: the package in the Deno cache layout, +// its registry.json and ts_npm.json. +func writeSlice(tarball []byte, wantName, outDir string) (PackageJSON, error) { + tmp, err := os.MkdirTemp("", "please_ts_npm_*") + if err != nil { + return PackageJSON{}, err + } + defer os.RemoveAll(tmp) + if err := extractTarballBytes(tarball, tmp); err != nil { + return PackageJSON{}, fmt.Errorf("failed extracting the tarball: %w", err) + } + pkg, raw, err := inspectTarball(tarball, wantName) + if err != nil { + return pkg, err + } + + pkgDir := filepath.Join(outDir, "npm", "registry.npmjs.org", filepath.FromSlash(pkg.Name)) + versionDir := filepath.Join(pkgDir, pkg.Version) + if err := os.MkdirAll(filepath.Dir(versionDir), 0755); err != nil { + return pkg, err + } + if err := copyPath(tmp, versionDir); err != nil { + return pkg, fmt.Errorf("failed staging the package: %w", err) + } + sum := sha512.Sum512(tarball) + integrity := "sha512-" + base64.StdEncoding.EncodeToString(sum[:]) + if err := writePackument(pkgDir, pkg, raw, integrity); err != nil { + return pkg, err + } + return pkg, npmcache.Write(outDir, npmcache.Slice{ + Name: pkg.Name, + Version: pkg.Version, + Specifier: npmcache.Specifier(pkg.Name, pkg.Version), + Dependencies: pkg.Dependencies, + }) +} + +// writePackument writes the registry.json Deno reads from its cache. The +// _deno.packumentFormat key tells Deno that the packument is complete; without it Deno +// tries to download the real one. +func writePackument(pkgDir string, pkg PackageJSON, raw map[string]json.RawMessage, integrity string) error { + version := map[string]any{ + "name": pkg.Name, + "version": pkg.Version, + "dist": map[string]string{ + "tarball": fmt.Sprintf("https://registry.npmjs.org/%s/-/%s-%s.tgz", pkg.Name, tarballBase(pkg.Name), pkg.Version), + "integrity": integrity, + }, + } + for _, key := range []string{"dependencies", "peerDependencies", "peerDependenciesMeta", "optionalDependencies", "bin"} { + if v, ok := raw[key]; ok { + version[key] = v + } + } + packument := map[string]any{ + "name": pkg.Name, + "dist-tags": map[string]string{"latest": pkg.Version}, + "versions": map[string]any{pkg.Version: version}, + "_deno.packumentFormat": "full", + } + data, err := json.Marshal(packument) + if err != nil { + return err + } + return os.WriteFile(filepath.Join(pkgDir, "registry.json"), data, 0644) +} + +func tarballBase(name string) string { + if i := strings.LastIndex(name, "/"); i >= 0 { + return name[i+1:] + } + return name +} + +// stagedSlice is a slice found in the build directory. +type stagedSlice struct { + dir string + npmcache.Slice +} + +// stagedSlices returns the slices staged under the working directory, which are the +// ts_npm_module targets in deps; outDir itself is skipped. +func stagedSlices(outDir string) ([]stagedSlice, error) { + absOut, _ := filepath.Abs(outDir) + var staged []stagedSlice + for _, dir := range npmcache.Discover(".") { + if abs, _ := filepath.Abs(dir); abs == absOut { + continue + } + s, err := npmcache.Read(dir) + if err != nil { + return nil, err + } + staged = append(staged, stagedSlice{dir: dir, Slice: *s}) + } + return staged, nil +} + +// resolveIntoSlices resolves the dependencies of pkg that no staged slice provides, and +// theirs in turn, writing each package as a slice under tmpDir. Versions that are already +// available (staged, or resolved for another dependent) are reused when they satisfy the +// specifier; otherwise another version is resolved, so dependents with incompatible needs +// each get theirs. Dependencies come from the package.json inside the verified tarball, not +// from registry metadata. Optional and peer dependencies are not resolved: a peer is for the +// consumer to provide. +func resolveIntoSlices(pkg PackageJSON, staged []stagedSlice, r *resolver, tmpDir string) ([]stagedSlice, error) { + r.pinToRoot(pkg.Name, pkg.Version) + have := map[string][]string{} // package -> versions available + for _, s := range staged { + have[s.Name] = append(have[s.Name], s.Version) + } + + var queue []pendingDep + enqueue := func(deps map[string]string, requiredBy string) { + for _, name := range sortedKeys(deps) { + queue = append(queue, pendingDep{name: name, spec: deps[name], requiredBy: requiredBy}) + } + } + enqueue(pkg.Dependencies, pkg.Name+"@"+pkg.Version) + + var out []stagedSlice + for len(queue) > 0 { + d := queue[0] + queue = queue[1:] + if anySatisfies(have[d.name], d.spec) { + continue + } + info, data, err := r.fetch(d.name, d.spec) + if err != nil { + return nil, fmt.Errorf("%s@%s (required by %s): %w", d.name, d.spec, d.requiredBy, err) + } + dir := filepath.Join(tmpDir, strings.ReplaceAll(d.name, "/", "+")+"@"+info.Version) + resolved, err := writeSlice(data, d.name, dir) + if err != nil { + return nil, fmt.Errorf("%s@%s: %w", d.name, info.Version, err) + } + have[d.name] = append(have[d.name], resolved.Version) + out = append(out, stagedSlice{dir: dir, Slice: npmcache.Slice{ + Name: resolved.Name, Version: resolved.Version, + Specifier: npmcache.Specifier(resolved.Name, resolved.Version), Dependencies: resolved.Dependencies, + }}) + enqueue(resolved.Dependencies, resolved.Name+"@"+resolved.Version) + } + return out, nil +} + +// checkDependencyClosure verifies that each required dependency of pkg is provided by a +// staged slice. Optional dependencies and peer dependencies, which the consumer provides, +// are not required. +func checkDependencyClosure(pkg PackageJSON, staged []stagedSlice) error { + if len(pkg.Dependencies) == 0 { + return nil + } + versions := map[string][]string{} + for _, s := range staged { + versions[s.Name] = append(versions[s.Name], s.Version) + } + + names := make([]string, 0, len(pkg.Dependencies)) + for name := range pkg.Dependencies { + names = append(names, name) + } + sort.Strings(names) + + var missing []string + for _, name := range names { + rng := pkg.Dependencies[name] + if !anySatisfies(versions[name], rng) { + missing = append(missing, fmt.Sprintf("%s@%s", name, rng)) + } + } + if len(missing) > 0 { + return fmt.Errorf("%s@%s depends on %s, which no ts_npm_module in deps provides; "+ + "add a ts_npm_module for each (with its tarball hash) to deps, or set resolve_transitive", + pkg.Name, pkg.Version, strings.Join(missing, ", ")) + } + return nil +} diff --git a/tools/please_ts/unpack/npmcache_test.go b/tools/please_ts/unpack/npmcache_test.go new file mode 100644 index 0000000..a26fa59 --- /dev/null +++ b/tools/please_ts/unpack/npmcache_test.go @@ -0,0 +1,327 @@ +package unpack + +import ( + "archive/tar" + "compress/gzip" + "crypto/sha512" + "encoding/base64" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "tools/please_ts/npmcache" +) + +// makeNpmTarball writes a gzipped tarball laid out like an npm package ("package/" prefix). +func makeNpmTarball(t *testing.T, dir string, files map[string]string) string { + t.Helper() + path := filepath.Join(dir, "pkg.tgz") + f, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + gz := gzip.NewWriter(f) + tw := tar.NewWriter(gz) + for name, content := range files { + if err := tw.WriteHeader(&tar.Header{Name: "package/" + name, Mode: 0644, Size: int64(len(content)), Typeflag: tar.TypeReg}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte(content)); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + return path +} + +func chdir(t *testing.T, dir string) { + t.Helper() + old, _ := os.Getwd() + if err := os.Chdir(dir); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chdir(old) }) +} + +func stageSlice(t *testing.T, root, name, version string) { + t.Helper() + dir := filepath.Join(root, "staged", name) + if err := os.MkdirAll(dir, 0755); err != nil { + t.Fatal(err) + } + if err := npmcache.Write(dir, npmcache.Slice{Name: name, Version: version, Specifier: npmcache.Specifier(name, version)}); err != nil { + t.Fatal(err) + } +} + +func TestBuildNpmCacheLaysOutTheDenoCache(t *testing.T) { + work := t.TempDir() + chdir(t, work) + tgz := makeNpmTarball(t, work, map[string]string{ + "package.json": `{"name":"ms","version":"2.1.3"}`, + "index.js": "module.exports = 1;", + "lib/util.js": "exports.x = 1;", + }) + out := filepath.Join(work, "out") + + if err := Run(Options{Archive: tgz, Out: out, Name: "ms", NpmCache: true}); err != nil { + t.Fatal(err) + } + + for _, p := range []string{"index.js", "package.json", "lib/util.js"} { + if _, err := os.Stat(filepath.Join(out, "npm/registry.npmjs.org/ms/2.1.3", p)); err != nil { + t.Errorf("package file %s missing: %v", p, err) + } + } + slice, err := npmcache.Read(out) + if err != nil || slice.Specifier != "npm:ms@2.1.3" { + t.Fatalf("slice = %+v, %v", slice, err) + } + + data, err := os.ReadFile(filepath.Join(out, "npm/registry.npmjs.org/ms/registry.json")) + if err != nil { + t.Fatal(err) + } + var packument struct { + Name string `json:"name"` + Format string `json:"_deno.packumentFormat"` + DistTags map[string]string + Versions map[string]struct { + Dist struct { + Integrity string `json:"integrity"` + Tarball string `json:"tarball"` + } `json:"dist"` + } `json:"versions"` + } + if err := json.Unmarshal(data, &packument); err != nil { + t.Fatal(err) + } + raw, _ := os.ReadFile(tgz) + sum := sha512.Sum512(raw) + wantIntegrity := "sha512-" + base64.StdEncoding.EncodeToString(sum[:]) + v := packument.Versions["2.1.3"] + if packument.Name != "ms" || packument.Format != "full" || v.Dist.Integrity != wantIntegrity || + v.Dist.Tarball != "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz" { + t.Errorf("packument = %s", data) + } +} + +func TestBuildNpmCacheScopedPackageAndDependencyFields(t *testing.T) { + work := t.TempDir() + chdir(t, work) + stageSlice(t, work, "lib", "1.0.0") + tgz := makeNpmTarball(t, work, map[string]string{ + "package.json": `{"name":"@scope/pkg","version":"1.0.0","dependencies":{"lib":"1.0.0"}, + "peerDependencies":{"react":"^18"},"peerDependenciesMeta":{"react":{"optional":true}}}`, + "index.js": "", + }) + out := filepath.Join(work, "out") + if err := Run(Options{Archive: tgz, Out: out, NpmCache: true}); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(out, "npm/registry.npmjs.org/@scope/pkg/registry.json")) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{`"dependencies":{"lib":"1.0.0"}`, `"peerDependencies"`, `"peerDependenciesMeta"`, "pkg-1.0.0.tgz"} { + if !strings.Contains(string(data), want) { + t.Errorf("registry.json lacks %s:\n%s", want, data) + } + } +} + +func TestBuildNpmCacheDependencyClosure(t *testing.T) { + depPkg := map[string]string{ + "package.json": `{"name":"debug","version":"4.3.7","dependencies":{"ms":"^2.1.3"}}`, + "index.js": "", + } + for name, tc := range map[string]struct { + staged map[string]string // name -> version + wantErr string + }{ + "missing": {nil, `ms@^2.1.3`}, + "unsatisfied": {map[string]string{"ms": "1.0.0"}, `ms@^2.1.3`}, + "different major": {map[string]string{"ms": "3.0.0"}, `ms@^2.1.3`}, + "satisfied": {map[string]string{"ms": "2.1.3"}, ""}, + "newer minor": {map[string]string{"ms": "2.4.0"}, ""}, + } { + t.Run(name, func(t *testing.T) { + work := t.TempDir() + chdir(t, work) + for n, v := range tc.staged { + stageSlice(t, work, n, v) + } + tgz := makeNpmTarball(t, work, depPkg) + err := Run(Options{Archive: tgz, Out: filepath.Join(work, "out"), Name: "debug", NpmCache: true}) + if tc.wantErr == "" { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tc.wantErr) || !strings.Contains(err.Error(), "ts_npm_module") { + t.Fatalf("error = %v, want one naming %s and ts_npm_module", err, tc.wantErr) + } + }) + } +} + +func TestBuildNpmCacheIgnoresItsOwnOutputInTheClosureCheck(t *testing.T) { + work := t.TempDir() + chdir(t, work) + out := filepath.Join(work, "out") + if err := os.MkdirAll(out, 0755); err != nil { + t.Fatal(err) + } + // A stale slice for the dependency inside out must not count as a staged dependency. + if err := npmcache.Write(out, npmcache.Slice{Name: "ms", Version: "2.1.3", Specifier: "npm:ms@2.1.3"}); err != nil { + t.Fatal(err) + } + tgz := makeNpmTarball(t, work, map[string]string{ + "package.json": `{"name":"debug","version":"4.3.7","dependencies":{"ms":"^2.1.3"}}`, + }) + if err := Run(Options{Archive: tgz, Out: out, Name: "debug", NpmCache: true}); err == nil { + t.Error("the output directory must not satisfy the target's own dependencies") + } +} + +func TestBuildNpmCacheRejectsBadTarballs(t *testing.T) { + work := t.TempDir() + chdir(t, work) + cases := map[string]struct { + files map[string]string + name string + }{ + "no package.json": {map[string]string{"index.js": ""}, ""}, + "no version": {map[string]string{"package.json": `{"name":"x"}`}, ""}, + "wrong name": {map[string]string{"package.json": `{"name":"x","version":"1.0.0"}`}, "y"}, + "invalid json": {map[string]string{"package.json": `{`}, ""}, + } + for n, c := range cases { + tgz := makeNpmTarball(t, work, c.files) + if err := Run(Options{Archive: tgz, Out: filepath.Join(work, "out-"+strings.ReplaceAll(n, " ", "-")), Name: c.name, NpmCache: true}); err == nil { + t.Errorf("%s: expected an error", n) + } + } +} + +// stageSliceWithPackage stages a dependency slice that holds an extracted package, as the +// output of a ts_npm_module does. +func stageSliceWithPackage(t *testing.T, root, name, version string) { + t.Helper() + stageSlice(t, root, name, version) + pkgDir := filepath.Join(root, "staged", name, "npm", "registry.npmjs.org", name) + if err := os.MkdirAll(filepath.Join(pkgDir, version), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(pkgDir, version, "index.js"), []byte("module.exports = 1;"), 0644); err != nil { + t.Fatal(err) + } + packument := `{"name":"` + name + `","dist-tags":{"latest":"` + version + `"},"versions":{"` + version + `":{"name":"` + name + `","version":"` + version + `"}},"_deno.packumentFormat":"full"}` + if err := os.WriteFile(filepath.Join(pkgDir, "registry.json"), []byte(packument), 0644); err != nil { + t.Fatal(err) + } +} + +func TestBuildNpmCacheBundlesTheDependenciesIntoTheSlice(t *testing.T) { + work := t.TempDir() + chdir(t, work) + stageSliceWithPackage(t, work, "ms", "2.1.3") + tgz := makeNpmTarball(t, work, map[string]string{ + "package.json": `{"name":"debug","version":"4.3.7","dependencies":{"ms":"^2.1.3"}}`, + "index.js": "", + }) + out := filepath.Join(work, "out") + if err := Run(Options{Archive: tgz, Out: out, Name: "debug", NpmCache: true}); err != nil { + t.Fatal(err) + } + + for _, p := range []string{ + "npm/registry.npmjs.org/debug/4.3.7/index.js", + "npm/registry.npmjs.org/ms/2.1.3/index.js", // the dependency, so a consumer only needs debug + "npm/registry.npmjs.org/ms/registry.json", + } { + if _, err := os.Stat(filepath.Join(out, p)); err != nil { + t.Errorf("slice lacks %s: %v", p, err) + } + } + // The slice is still named after its own package: only debug is importable by name. + slice, err := npmcache.Read(out) + if err != nil || slice.Name != "debug" || slice.Version != "4.3.7" { + t.Errorf("slice = %+v, %v", slice, err) + } +} + +func TestBuildNpmCacheSlicesWithDifferentVersionsOfADependencyStayResolvable(t *testing.T) { + work := t.TempDir() + chdir(t, work) + stageSliceWithPackage(t, work, "ms", "2.1.3") + // A second staged dependency that itself carries another version of ms. + other := filepath.Join(work, "staged", "other") + oldPkg := filepath.Join(other, "npm", "registry.npmjs.org", "ms") + if err := os.MkdirAll(filepath.Join(oldPkg, "2.0.0"), 0755); err != nil { + t.Fatal(err) + } + _ = os.WriteFile(filepath.Join(oldPkg, "2.0.0", "index.js"), []byte("x"), 0644) + _ = os.WriteFile(filepath.Join(oldPkg, "registry.json"), []byte(`{"name":"ms","versions":{"2.0.0":{"name":"ms","version":"2.0.0"}}}`), 0644) + if err := npmcache.Write(other, npmcache.Slice{Name: "other", Version: "1.0.0", Specifier: "npm:other@1.0.0"}); err != nil { + t.Fatal(err) + } + + tgz := makeNpmTarball(t, work, map[string]string{ + "package.json": `{"name":"top","version":"1.0.0","dependencies":{"ms":"^2.1.3","other":"1.0.0"}}`, + }) + out := filepath.Join(work, "out") + if err := Run(Options{Archive: tgz, Out: out, Name: "top", NpmCache: true}); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(out, "npm/registry.npmjs.org/ms/registry.json")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), `"2.0.0"`) || !strings.Contains(string(data), `"2.1.3"`) { + t.Errorf("both ms versions must be in the merged packument: %s", data) + } +} + +func TestVersionSatisfies(t *testing.T) { + for _, tc := range []struct { + version, spec string + want bool + }{ + {"2.1.3", "2.1.3", true}, + {"2.1.3", "^2.1.3", true}, + {"2.4.0", "^2.1.3", true}, + {"2.0.0", "^2.1.3", false}, + {"3.0.0", "^2.1.3", false}, + {"1.2.9", "~1.2.3", true}, + {"1.3.0", "~1.2.3", false}, + {"5.0.0", ">=2.0.0", true}, + {"1.0.0", ">=2.0.0", false}, + {"9.9.9", "*", true}, + {"9.9.9", "", true}, + // Ranges are interpreted completely, not approximated. + {"2.5.0", "^1 || ^2", true}, + {"3.5.0", "^1 || ^2", false}, + {"2.0.0", ">=1 <3", true}, + {"3.0.0", ">=1 <3", false}, + {"1.9.0", "1.x", true}, + {"2.0.0", "1.x", false}, + {"1.2.3-beta.1", "^1.0.0", false}, + // Specifiers that are not version ranges cannot be judged and are accepted. + {"9.9.9", "latest", true}, + {"9.9.9", "git+https://example.com/x.git", true}, + } { + if got := versionSatisfies(tc.version, tc.spec); got != tc.want { + t.Errorf("versionSatisfies(%q, %q) = %v, want %v", tc.version, tc.spec, got, tc.want) + } + } +} diff --git a/tools/please_ts/unpack/resolve.go b/tools/please_ts/unpack/resolve.go new file mode 100644 index 0000000..57a3ceb --- /dev/null +++ b/tools/please_ts/unpack/resolve.go @@ -0,0 +1,219 @@ +package unpack + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "time" + + "tools/please_ts/registry" + "tools/please_ts/semver" +) + +// resolver resolves dependency specifiers against an npm registry and downloads the +// tarballs, verified against the integrity data the registry publishes. +// +// Resolution is reproducible: only versions published by the end of the day the root package +// version was published are considered (see pinToRoot), so a range such as ^2.1.3 resolves to +// the same version next year, and the version pinned by its hash is all that fixes the result. +type resolver struct { + client *registry.Client + asOf time.Time + packuments map[string]*registry.Packument +} + +func newResolver(registryURL string) *resolver { + return &resolver{client: registry.New(registryURL), packuments: map[string]*registry.Packument{}} +} + +// pinToRoot sets the as-of date to the end of the day (UTC) the root package version was +// published. The whole day counts so that a dependency released a few minutes after the +// package that needs it is still found. If the registry does not know the root version or +// its publish time (a tarball from another URL, a private registry without times), +// resolution is not pinned, with a warning. +func (r *resolver) pinToRoot(name, version string) { + if !r.asOf.IsZero() { + return + } + p, err := r.client.Packument(name, true) + if err != nil { + fmt.Fprintf(os.Stderr, "Warning: dependencies of %s@%s are not pinned to a date: %v\n", name, version, err) + return + } + raw, ok := p.Time[version] + if !ok { + fmt.Fprintf(os.Stderr, "Warning: dependencies of %s@%s are not pinned to a date: the registry has no publish time for it\n", name, version) + return + } + published, err := time.Parse(time.RFC3339, raw) + if err != nil { + fmt.Fprintf(os.Stderr, "Warning: dependencies of %s@%s are not pinned to a date: bad publish time %q\n", name, version, raw) + return + } + day := time.Date(published.Year(), published.Month(), published.Day(), 0, 0, 0, 0, time.UTC) + r.asOf = day.Add(24*time.Hour - time.Nanosecond) + r.packuments[name] = p // the full packument is reusable +} + +// fetch resolves spec for a package to one version and downloads and verifies its tarball. +// The full packument (with publish times) is only fetched when there is an as-of date. +func (r *resolver) fetch(name, spec string) (registry.VersionInfo, []byte, error) { + p, ok := r.packuments[name] + if !ok { + var err error + if p, err = r.client.Packument(name, !r.asOf.IsZero()); err != nil { + return registry.VersionInfo{}, nil, err + } + r.packuments[name] = p + } + info, err := registry.Resolve(p, spec, r.asOf) + if err != nil { + return registry.VersionInfo{}, nil, err + } + data, err := r.client.Download(name, info) + if err != nil { + return registry.VersionInfo{}, nil, err + } + return info, data, nil +} + +// extractTarballBytes extracts a verified npm tarball into destDir, without the leading +// "package/" directory. +func extractTarballBytes(data []byte, destDir string) error { + f, err := os.CreateTemp("", "please_ts_dep_*.tgz") + if err != nil { + return err + } + defer os.Remove(f.Name()) + if _, err := f.Write(data); err != nil { + f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + return extractTarGz(f.Name(), destDir) +} + +// versionSatisfies reports whether a version meets a dependency specifier. Specifiers that +// are not version ranges (a dist-tag, say) cannot be judged here and count as satisfied. +func versionSatisfies(version, spec string) bool { + v, err := semver.Parse(version) + if err != nil { + return true + } + r, err := semver.ParseRange(spec) + if err != nil { + return true + } + return r.Satisfies(v) +} + +func anySatisfies(versions []string, spec string) bool { + for _, v := range versions { + if versionSatisfies(v, spec) { + return true + } + } + return false +} + +func sortedKeys(m map[string]string) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +// pendingDep is a dependency waiting to be resolved. +type pendingDep struct { + name, spec, requiredBy string + peer bool +} + +// resolveTransitiveDependencies fetches the dependencies of a ts_module into .deps/, +// one directory per package (the layout holds a single version of each). +// +// Resolution is strict: a specifier that no registry version satisfies is an error, and so +// is a download that does not match the registry's integrity data, a tarball that holds a +// different package than asked for, and two dependents that need versions of one package +// that cannot both be the one in .deps (ts_npm_module keeps both). Peer dependencies are +// fetched too, as a default for consumers that do not provide them, but only as a best +// effort: a peer that cannot be resolved or conflicts is a warning. +func resolveTransitiveDependencies(outDir string, rootPkg PackageJSON, r *resolver, depsMap map[string]string) error { + r.pinToRoot(rootPkg.Name, rootPkg.Version) + resolved := map[string]string{} // package -> version in .deps + var queue []pendingDep + for _, name := range sortedKeys(rootPkg.Dependencies) { + queue = append(queue, pendingDep{name, rootPkg.Dependencies[name], rootPkg.Name, false}) + } + for _, name := range sortedKeys(rootPkg.PeerDependencies) { + queue = append(queue, pendingDep{name, rootPkg.PeerDependencies[name], rootPkg.Name, true}) + } + + for len(queue) > 0 { + d := queue[0] + queue = queue[1:] + if d.name == rootPkg.Name { + continue + } + + if have, ok := resolved[d.name]; ok { + if versionSatisfies(have, d.spec) { + continue + } + msg := fmt.Sprintf("%s requires %s@%s, but %s@%s is already in .deps; a ts_module holds one version of a package (use ts_npm_module, which keeps several)", + d.requiredBy, d.name, d.spec, d.name, have) + if d.peer { + fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) + continue + } + return fmt.Errorf("conflicting versions: %s", msg) + } + + destDir := filepath.Join(outDir, ".deps", d.name) + if _, err := os.Stat(filepath.Join(destDir, "package.json")); err == nil { + existing := readPackageJSON(destDir) + resolved[d.name] = existing.Version + if entry := findLocalEntry(destDir); entry != "" { + depsMap[d.name] = cleanRelativePath(filepath.Join(".deps", d.name, entry)) + } + continue + } + + info, data, err := r.fetch(d.name, d.spec) + if err != nil { + if d.peer { + fmt.Fprintf(os.Stderr, "Warning: peer dependency %s@%s of %s was not resolved: %v\n", d.name, d.spec, d.requiredBy, err) + continue + } + return fmt.Errorf("resolving %s@%s (required by %s): %w", d.name, d.spec, d.requiredBy, err) + } + if err := os.MkdirAll(destDir, 0755); err != nil { + return err + } + if err := extractTarballBytes(data, destDir); err != nil { + return fmt.Errorf("extracting %s@%s: %w", d.name, info.Version, err) + } + pkg := readPackageJSON(destDir) + if pkg.Name != "" && pkg.Name != d.name { + return fmt.Errorf("the tarball of %s@%s holds package %q", d.name, info.Version, pkg.Name) + } + if pkg.Version == "" { + pkg.Version = info.Version + } + ensureCommonJSType(destDir) + resolved[d.name] = pkg.Version + + if entry := determineEntry(destDir, pkg); entry != "" { + depsMap[d.name] = cleanRelativePath(filepath.Join(".deps", d.name, entry)) + } + for _, name := range sortedKeys(pkg.Dependencies) { + queue = append(queue, pendingDep{name, pkg.Dependencies[name], d.name + "@" + pkg.Version, false}) + } + } + return nil +} diff --git a/tools/please_ts/unpack/resolve_test.go b/tools/please_ts/unpack/resolve_test.go new file mode 100644 index 0000000..ceb1652 --- /dev/null +++ b/tools/please_ts/unpack/resolve_test.go @@ -0,0 +1,516 @@ +package unpack + +import ( + "crypto/sha512" + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + + "tools/please_ts/semver" +) + +// mockVersion describes one published version. +type mockVersion struct { + deps, peers, optional map[string]string + published string // RFC 3339, for as-of tests +} + +// mockRegistry is an npm registry that serves real tarballs with integrity data. +type mockRegistry struct { + t *testing.T + srv *httptest.Server + mu sync.Mutex + versions map[string]map[string]mockVersion // package -> version -> description + tarballs map[string][]byte // "name@version" -> tarball + hits map[string]int // request path -> count + corrupt map[string]bool // "name@version": serve other bytes than the integrity covers + noIntegrity map[string]bool // "name@version": publish no integrity data + wrongName map[string]string // "name@version": the package.json name to put in the tarball +} + +func newMockRegistry(t *testing.T) *mockRegistry { + m := &mockRegistry{ + t: t, versions: map[string]map[string]mockVersion{}, tarballs: map[string][]byte{}, + hits: map[string]int{}, corrupt: map[string]bool{}, noIntegrity: map[string]bool{}, wrongName: map[string]string{}, + } + m.srv = httptest.NewServer(http.HandlerFunc(m.serve)) + t.Cleanup(m.srv.Close) + return m +} + +func (m *mockRegistry) URL() string { return m.srv.URL } + +func (m *mockRegistry) add(name, version string, v mockVersion) { + if m.versions[name] == nil { + m.versions[name] = map[string]mockVersion{} + } + m.versions[name][version] = v + manifestName := name + if w, ok := m.wrongName[name+"@"+version]; ok { + manifestName = w + } + manifest := map[string]any{"name": manifestName, "version": version, "main": "index.js"} + for k, deps := range map[string]map[string]string{"dependencies": v.deps, "peerDependencies": v.peers, "optionalDependencies": v.optional} { + if len(deps) > 0 { + manifest[k] = deps + } + } + pkgJSON, _ := json.Marshal(manifest) + m.tarballs[name+"@"+version] = createTestTarball(m.t, map[string]string{ + "package.json": string(pkgJSON), "index.js": "module.exports = '" + name + "@" + version + "';", + }) +} + +func (m *mockRegistry) hitsFor(substr string) int { + m.mu.Lock() + defer m.mu.Unlock() + n := 0 + for path, c := range m.hits { + if strings.Contains(path, substr) { + n += c + } + } + return n +} + +func (m *mockRegistry) serve(w http.ResponseWriter, r *http.Request) { + path := strings.TrimPrefix(r.URL.Path, "/") + m.mu.Lock() + m.hits[path]++ + m.mu.Unlock() + + if name, file, ok := strings.Cut(path, "/-/"); ok { + base := name[strings.LastIndex(name, "/")+1:] + version := strings.TrimSuffix(strings.TrimPrefix(file, base+"-"), ".tgz") + data, ok := m.tarballs[name+"@"+version] + if !ok { + http.NotFound(w, r) + return + } + if m.corrupt[name+"@"+version] { + data = append([]byte("corrupt"), data...) + } + _, _ = w.Write(data) + return + } + + versions, ok := m.versions[path] + if !ok { + http.NotFound(w, r) + return + } + all := make([]string, 0, len(versions)) + for v := range versions { + all = append(all, v) + } + sort.Slice(all, func(i, j int) bool { + a, _ := semver.Parse(all[i]) + b, _ := semver.Parse(all[j]) + return semver.Compare(a, b) < 0 + }) + entries := map[string]any{} + times := map[string]string{} + for _, v := range all { + sum := sha512.Sum512(m.tarballs[path+"@"+v]) // integrity of the genuine tarball + dist := map[string]string{"tarball": fmt.Sprintf("%s/%s/-/%s-%s.tgz", m.srv.URL, path, path[strings.LastIndex(path, "/")+1:], v)} + if !m.noIntegrity[path+"@"+v] { + dist["integrity"] = "sha512-" + base64.StdEncoding.EncodeToString(sum[:]) + } + entries[v] = map[string]any{"version": v, "dist": dist} + if versions[v].published != "" { + times[v] = versions[v].published + } + } + doc := map[string]any{"name": path, "dist-tags": map[string]string{"latest": all[len(all)-1]}, "versions": entries, "time": times} + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(doc) +} + +// rootTarball writes a root package's tarball and returns its path. +func rootTarball(t *testing.T, dir, name, version string, v mockVersion) string { + t.Helper() + manifest := map[string]any{"name": name, "version": version, "main": "index.js"} + for k, deps := range map[string]map[string]string{"dependencies": v.deps, "peerDependencies": v.peers, "optionalDependencies": v.optional} { + if len(deps) > 0 { + manifest[k] = deps + } + } + pkgJSON, _ := json.Marshal(manifest) + path := filepath.Join(dir, "root.tgz") + if err := os.WriteFile(path, createTestTarball(t, map[string]string{"package.json": string(pkgJSON), "index.js": ""}), 0644); err != nil { + t.Fatal(err) + } + return path +} + +func deps(kv ...string) map[string]string { + m := map[string]string{} + for i := 0; i+1 < len(kv); i += 2 { + m[kv[i]] = kv[i+1] + } + return m +} + +func versionOf(t *testing.T, dir string) string { + t.Helper() + data, err := os.ReadFile(filepath.Join(dir, "package.json")) + if err != nil { + t.Fatal(err) + } + var p struct{ Version string } + if err := json.Unmarshal(data, &p); err != nil { + t.Fatal(err) + } + return p.Version +} + +// ---- ts_module: the flat .deps layout ---- + +func runModule(t *testing.T, m *mockRegistry, root mockVersion) (string, error) { + t.Helper() + work := t.TempDir() + out := filepath.Join(work, "out") + err := Run(Options{ + Tarball: rootTarball(t, work, "root-pkg", "1.0.0", root), Out: out, Name: "root-pkg", + ResolveTransitive: true, Registry: m.URL(), + }) + return out, err +} + +func TestModuleResolvesAChainOfDependencies(t *testing.T) { + m := newMockRegistry(t) + m.add("a", "1.2.0", mockVersion{deps: deps("b", "^2.0.0")}) + m.add("b", "2.1.0", mockVersion{}) + out, err := runModule(t, m, mockVersion{deps: deps("a", "^1.0.0")}) + if err != nil { + t.Fatal(err) + } + if versionOf(t, filepath.Join(out, ".deps", "a")) != "1.2.0" || versionOf(t, filepath.Join(out, ".deps", "b")) != "2.1.0" { + t.Error("the whole chain should be in .deps at the resolved versions") + } +} + +func TestModuleResolutionIsStrict(t *testing.T) { + m := newMockRegistry(t) + m.add("a", "1.0.0", mockVersion{}) + _, err := runModule(t, m, mockVersion{deps: deps("a", "^9.0.0")}) + if err == nil || !strings.Contains(err.Error(), "no version of a satisfies") { + t.Errorf("a range nothing satisfies must fail, not fall back to another version; got %v", err) + } +} + +func TestModuleResolutionFailureIsAnErrorNotAWarning(t *testing.T) { + m := newMockRegistry(t) // the registry knows no package at all + _, err := runModule(t, m, mockVersion{deps: deps("missing", "^1.0.0")}) + if err == nil || !strings.Contains(err.Error(), "missing@^1.0.0") || !strings.Contains(err.Error(), "root-pkg") { + t.Errorf("an unresolvable dependency must fail and name what needed it, got %v", err) + } +} + +func TestModuleVerifiesIntegrity(t *testing.T) { + m := newMockRegistry(t) + m.add("a", "1.0.0", mockVersion{}) + m.corrupt["a@1.0.0"] = true + if _, err := runModule(t, m, mockVersion{deps: deps("a", "^1.0.0")}); err == nil || !strings.Contains(err.Error(), "does not match") { + t.Errorf("a tarball that does not match the registry's integrity must be refused, got %v", err) + } + + m2 := newMockRegistry(t) + m2.add("a", "1.0.0", mockVersion{}) + m2.noIntegrity["a@1.0.0"] = true + if _, err := runModule(t, m2, mockVersion{deps: deps("a", "^1.0.0")}); err == nil || !strings.Contains(err.Error(), "no integrity") { + t.Errorf("a version without integrity data cannot be verified, got %v", err) + } +} + +func TestModuleRejectsATarballOfAnotherPackage(t *testing.T) { + m := newMockRegistry(t) + m.wrongName["a@1.0.0"] = "evil" + m.add("a", "1.0.0", mockVersion{}) + if _, err := runModule(t, m, mockVersion{deps: deps("a", "^1.0.0")}); err == nil || !strings.Contains(err.Error(), `holds package "evil"`) { + t.Errorf("got %v", err) + } +} + +func TestModuleReportsConflictingVersionsInsteadOfPickingTheFirst(t *testing.T) { + m := newMockRegistry(t) + m.add("a", "1.0.0", mockVersion{deps: deps("c", "^1.0.0")}) + m.add("b", "1.0.0", mockVersion{deps: deps("c", "^2.0.0")}) + m.add("c", "1.4.0", mockVersion{}) + m.add("c", "2.0.0", mockVersion{}) + _, err := runModule(t, m, mockVersion{deps: deps("a", "^1.0.0", "b", "^1.0.0")}) + if err == nil || !strings.Contains(err.Error(), "conflicting versions") || !strings.Contains(err.Error(), "ts_npm_module") { + t.Errorf("incompatible needs for one package must be reported, got %v", err) + } +} + +func TestModuleSharesACompatibleDependency(t *testing.T) { + m := newMockRegistry(t) + m.add("a", "1.0.0", mockVersion{deps: deps("c", "^1.0.0")}) + m.add("b", "1.0.0", mockVersion{deps: deps("c", "^1.2.0")}) + m.add("c", "1.5.0", mockVersion{}) + out, err := runModule(t, m, mockVersion{deps: deps("a", "^1.0.0", "b", "^1.0.0")}) + if err != nil { + t.Fatalf("two compatible ranges are not a conflict: %v", err) + } + if versionOf(t, filepath.Join(out, ".deps", "c")) != "1.5.0" { + t.Error("c should be resolved once, to a version both accept") + } +} + +func TestModulePeerDependenciesAreBestEffort(t *testing.T) { + m := newMockRegistry(t) + m.add("a", "1.0.0", mockVersion{}) + out, err := runModule(t, m, mockVersion{deps: deps("a", "^1.0.0"), peers: deps("not-published", "^1.0.0")}) + if err != nil { + t.Fatalf("a peer that cannot be resolved is a warning, got %v", err) + } + if _, err := os.Stat(filepath.Join(out, ".deps", "a", "package.json")); err != nil { + t.Error("the real dependency must still be resolved") + } +} + +// ---- ts_npm_module: slices ---- + +func runNpmCache(t *testing.T, m *mockRegistry, root mockVersion, resolve bool) (string, error) { + t.Helper() + work := t.TempDir() + chdir(t, work) + out := filepath.Join(work, "out") + err := Run(Options{ + Archive: rootTarball(t, work, "root-pkg", "1.0.0", root), Out: out, Name: "root-pkg", NpmCache: true, + ResolveTransitive: resolve, Registry: m.URL(), + }) + return out, err +} + +func cacheVersions(t *testing.T, out, name string) []string { + t.Helper() + data, err := os.ReadFile(filepath.Join(out, "npm/registry.npmjs.org", name, "registry.json")) + if err != nil { + t.Fatalf("no packument for %s: %v", name, err) + } + var p struct { + Versions map[string]json.RawMessage `json:"versions"` + } + if err := json.Unmarshal(data, &p); err != nil { + t.Fatal(err) + } + var vs []string + for v := range p.Versions { + if _, err := os.Stat(filepath.Join(out, "npm/registry.npmjs.org", name, v, "index.js")); err != nil { + t.Errorf("%s@%s is in the packument but not extracted: %v", name, v, err) + } + vs = append(vs, v) + } + sort.Strings(vs) + return vs +} + +func TestNpmCacheResolvesAndBundlesTransitiveDependencies(t *testing.T) { + m := newMockRegistry(t) + m.add("a", "1.2.0", mockVersion{deps: deps("b", "^2.0.0")}) + m.add("b", "2.1.0", mockVersion{}) + out, err := runNpmCache(t, m, mockVersion{deps: deps("a", "^1.0.0")}, true) + if err != nil { + t.Fatal(err) + } + for name, want := range map[string]string{"root-pkg": "1.0.0", "a": "1.2.0", "b": "2.1.0"} { + if got := cacheVersions(t, out, name); len(got) != 1 || got[0] != want { + t.Errorf("%s versions = %v, want [%s]", name, got, want) + } + } +} + +func TestNpmCacheKeepsConflictingVersionsSideBySide(t *testing.T) { + m := newMockRegistry(t) + m.add("a", "1.0.0", mockVersion{deps: deps("c", "^1.0.0")}) + m.add("b", "1.0.0", mockVersion{deps: deps("c", "^2.0.0")}) + m.add("c", "1.4.0", mockVersion{}) + m.add("c", "2.0.0", mockVersion{}) + out, err := runNpmCache(t, m, mockVersion{deps: deps("a", "^1.0.0", "b", "^1.0.0")}, true) + if err != nil { + t.Fatalf("unlike ts_module, a slice can hold both: %v", err) + } + if got := cacheVersions(t, out, "c"); strings.Join(got, ",") != "1.4.0,2.0.0" { + t.Errorf("c versions = %v, want 1.4.0 and 2.0.0 (Deno picks per dependent)", got) + } +} + +func TestNpmCacheReusesAVersionThatAlreadySatisfies(t *testing.T) { + m := newMockRegistry(t) + m.add("a", "1.0.0", mockVersion{deps: deps("c", "^1.0.0")}) + m.add("b", "1.0.0", mockVersion{deps: deps("c", "^1.2.0")}) + m.add("c", "1.2.0", mockVersion{}) + m.add("c", "1.5.0", mockVersion{}) + out, err := runNpmCache(t, m, mockVersion{deps: deps("a", "^1.0.0", "b", "^1.0.0")}, true) + if err != nil { + t.Fatal(err) + } + if got := cacheVersions(t, out, "c"); len(got) != 1 || got[0] != "1.5.0" { + t.Errorf("c should be resolved once, got %v", got) + } + if n := m.hitsFor("c-1."); n != 1 { + t.Errorf("c was downloaded %d times, want 1", n) + } +} + +func TestNpmCacheUsesStagedSlicesBeforeTheRegistry(t *testing.T) { + m := newMockRegistry(t) + m.add("b", "1.0.0", mockVersion{}) + work := t.TempDir() + chdir(t, work) + stageSliceWithPackage(t, work, "a", "1.0.0") // explicitly provided; the registry has no "a" + out := filepath.Join(work, "out") + err := Run(Options{ + Archive: rootTarball(t, work, "root-pkg", "1.0.0", mockVersion{deps: deps("a", "^1.0.0", "b", "^1.0.0")}), + Out: out, Name: "root-pkg", NpmCache: true, ResolveTransitive: true, Registry: m.URL(), + }) + if err != nil { + t.Fatalf("a declared dependency must not be resolved again: %v", err) + } + if m.hitsFor("a") != 0 { + t.Error("the registry must not be asked for a package a slice provides") + } + if len(cacheVersions(t, out, "a")) != 1 || len(cacheVersions(t, out, "b")) != 1 { + t.Error("both the staged and the resolved dependency must end up in the slice") + } +} + +func TestNpmCacheDoesNotResolveOptionalOrPeerDependencies(t *testing.T) { + m := newMockRegistry(t) // knows nothing: any request would fail + out, err := runNpmCache(t, m, mockVersion{optional: deps("fsevents", "^2.0.0"), peers: deps("react", "^18.0.0")}, true) + if err != nil { + t.Fatalf("optional and peer dependencies are not fetched: %v", err) + } + if got := cacheVersions(t, out, "root-pkg"); len(got) != 1 { + t.Errorf("root-pkg = %v", got) + } + if n := m.hitsFor(""); n != 0 { + t.Errorf("the registry was contacted %d times, want 0", n) + } +} + +func TestNpmCacheResolutionErrors(t *testing.T) { + t.Run("no version satisfies", func(t *testing.T) { + m := newMockRegistry(t) + m.add("a", "1.0.0", mockVersion{}) + _, err := runNpmCache(t, m, mockVersion{deps: deps("a", "^9.0.0")}, true) + if err == nil || !strings.Contains(err.Error(), "no version of a satisfies") || !strings.Contains(err.Error(), "root-pkg@1.0.0") { + t.Errorf("got %v", err) + } + }) + t.Run("integrity mismatch", func(t *testing.T) { + m := newMockRegistry(t) + m.add("a", "1.0.0", mockVersion{}) + m.corrupt["a@1.0.0"] = true + if _, err := runNpmCache(t, m, mockVersion{deps: deps("a", "^1.0.0")}, true); err == nil || !strings.Contains(err.Error(), "does not match") { + t.Errorf("got %v", err) + } + }) + t.Run("a transitive dependency fails", func(t *testing.T) { + m := newMockRegistry(t) + m.add("a", "1.0.0", mockVersion{deps: deps("gone", "^1.0.0")}) + _, err := runNpmCache(t, m, mockVersion{deps: deps("a", "^1.0.0")}, true) + if err == nil || !strings.Contains(err.Error(), "gone@^1.0.0") || !strings.Contains(err.Error(), "a@1.0.0") { + t.Errorf("the error must name the package and who required it, got %v", err) + } + }) +} + +func TestNpmCacheWithoutResolveStillRequiresDeclaredDependencies(t *testing.T) { + m := newMockRegistry(t) + m.add("a", "1.0.0", mockVersion{}) + _, err := runNpmCache(t, m, mockVersion{deps: deps("a", "^1.0.0")}, false) + if err == nil || !strings.Contains(err.Error(), "resolve_transitive") { + t.Errorf("strict mode must fail and point at resolve_transitive, got %v", err) + } + if m.hitsFor("") != 0 { + t.Error("strict mode must not contact the registry") + } +} + +// ---- the as-of date defaults to the day the root version was published ---- + +// publishedAround registers the root and a dependency whose versions straddle the root's +// publish day (2021-03-10). +func publishedAround(t *testing.T) *mockRegistry { + m := newMockRegistry(t) + m.add("root-pkg", "1.0.0", mockVersion{published: "2021-03-10T15:00:00.000Z"}) + m.add("a", "1.0.0", mockVersion{published: "2021-03-01T09:00:00.000Z"}) + m.add("a", "1.1.0", mockVersion{published: "2021-03-10T23:30:00.000Z"}) // later that day: still counts + m.add("a", "1.2.0", mockVersion{published: "2021-03-11T00:10:00.000Z"}) // the next day: too new + return m +} + +func TestAutomaticPinUsesTheDayTheRootWasPublished(t *testing.T) { + root := mockVersion{deps: deps("a", "^1.0.0")} + + m := publishedAround(t) + out, err := runNpmCache(t, m, root, true) + if err != nil { + t.Fatal(err) + } + if got := cacheVersions(t, out, "a"); len(got) != 1 || got[0] != "1.1.0" { + t.Errorf("npm cache: a = %v, want 1.1.0 (the newest published by the end of the root's day)", got) + } + + m = publishedAround(t) + modOut, err := runModule(t, m, root) + if err != nil { + t.Fatal(err) + } + if got := versionOf(t, filepath.Join(modOut, ".deps", "a")); got != "1.1.0" { + t.Errorf("ts_module: a = %s, want 1.1.0", got) + } +} + +func TestPinningIsSkippedWhenTheRegistryDoesNotKnowTheRoot(t *testing.T) { + m := newMockRegistry(t) // root-pkg is not published here, say a private tarball + m.add("a", "1.0.0", mockVersion{published: "2021-03-01T09:00:00.000Z"}) + m.add("a", "1.2.0", mockVersion{published: "2024-03-11T00:10:00.000Z"}) + out, err := runNpmCache(t, m, mockVersion{deps: deps("a", "^1.0.0")}, true) + if err != nil { + t.Fatalf("an unknown root must not fail the build: %v", err) + } + if got := cacheVersions(t, out, "a"); len(got) != 1 || got[0] != "1.2.0" { + t.Errorf("a = %v, want the newest when there is no date to pin to", got) + } +} + +func TestPinningIsSkippedWhenTheRootHasNoPublishTime(t *testing.T) { + m := newMockRegistry(t) + m.add("root-pkg", "1.0.0", mockVersion{}) // no time recorded + m.add("a", "1.0.0", mockVersion{published: "2021-03-01T09:00:00.000Z"}) + m.add("a", "1.2.0", mockVersion{published: "2024-03-11T00:10:00.000Z"}) + out, err := runNpmCache(t, m, mockVersion{deps: deps("a", "^1.0.0")}, true) + if err != nil { + t.Fatal(err) + } + if got := cacheVersions(t, out, "a"); len(got) != 1 || got[0] != "1.2.0" { + t.Errorf("a = %v", got) + } +} + +func TestResolverPinsOnlyOnce(t *testing.T) { + m := publishedAround(t) + r := newResolver(m.URL()) + r.pinToRoot("root-pkg", "1.0.0") + first := r.asOf + if first.IsZero() || first.Format("2006-01-02T15:04:05") != "2021-03-10T23:59:59" { + t.Fatalf("asOf = %v, want the end of 2021-03-10", first) + } + m.add("root-pkg", "2.0.0", mockVersion{published: "2024-01-01T00:00:00.000Z"}) + r.pinToRoot("root-pkg", "2.0.0") + if !r.asOf.Equal(first) { + t.Error("a resolver keeps the date it was pinned to") + } +} diff --git a/tools/please_ts/unpack/unpack.go b/tools/please_ts/unpack/unpack.go index c6aa76b..ae55bba 100644 --- a/tools/please_ts/unpack/unpack.go +++ b/tools/please_ts/unpack/unpack.go @@ -7,13 +7,9 @@ import ( "encoding/json" "fmt" "io" - "net/http" - "net/url" "os" "path/filepath" - "strconv" "strings" - "time" "tools/please_ts/importmap" ) @@ -28,6 +24,7 @@ type Options struct { Symlink string // optional symlink name for extracted binary ResolveTransitive bool // recursively resolve and unpack transitive dependencies Registry string // npm registry URL (default https://registry.npmjs.org) + NpmCache bool // build a Deno npm cache slice from the tarball } // Validate checks whether the required options are provided. @@ -70,22 +67,6 @@ type PackageJSON struct { PeerDependencies map[string]string `json:"peerDependencies"` } -type npmVersionData struct { - Version string `json:"version"` - Dependencies map[string]string `json:"dependencies"` - Dist struct { - Tarball string `json:"tarball"` - } `json:"dist"` -} - -type npmManifest struct { - Name string `json:"name"` - DistTags struct { - Latest string `json:"latest"` - } `json:"dist-tags"` - Versions map[string]npmVersionData `json:"versions"` -} - // Run executes the unpacking operation based on provided Options. func Run(opts Options) error { if err := opts.Validate(); err != nil { @@ -94,6 +75,9 @@ func Run(opts Options) error { if opts.Binary != "" { return unpackToolchain(opts.ArchivePath(), opts.Out, opts.Binary, opts.Symlink) } + if opts.NpmCache { + return buildNpmCache(opts) + } return unpackModule(opts) } @@ -114,8 +98,8 @@ func unpackModule(opts Options) error { peerDepsMap := copyStringMap(pkg.PeerDependencies) if opts.ResolveTransitive && hasDependencies(pkg) { - if err := resolveTransitiveDependencies(opts.Out, pkg, opts.RegistryURL(), depsMap); err != nil { - fmt.Fprintf(os.Stderr, "Warning: transitive dependency resolution failed: %v\n", err) + if err := resolveTransitiveDependencies(opts.Out, pkg, newResolver(opts.RegistryURL()), depsMap); err != nil { + return fmt.Errorf("failed resolving the dependencies of %s: %w", pkg.Name, err) } } @@ -253,152 +237,6 @@ func findLocalEntry(pkgDir string) string { return "" } -func resolveTransitiveDependencies(outDir string, rootPkg PackageJSON, registry string, depsMap map[string]string) error { - client := &http.Client{Timeout: 30 * time.Second} - visited := make(map[string]bool) - visited[rootPkg.Name] = true - - queue := make(map[string]string) - for k, v := range rootPkg.Dependencies { - queue[k] = v - } - for k, v := range rootPkg.PeerDependencies { - queue[k] = v - } - - for len(queue) > 0 { - var currentPkg, currentConstraint string - for k, v := range queue { - currentPkg = k - currentConstraint = v - delete(queue, k) - break - } - - if visited[currentPkg] { - continue - } - visited[currentPkg] = true - - destDir := filepath.Join(outDir, ".deps", currentPkg) - if _, err := os.Stat(filepath.Join(destDir, "package.json")); err == nil { - entry := findLocalEntry(destDir) - if entry != "" { - depsMap[currentPkg] = cleanRelativePath(filepath.Join(".deps", currentPkg, entry)) - } - continue - } - - pkgInfo, err := fetchAndExtractPackage(client, registry, currentPkg, currentConstraint, destDir) - if err != nil { - return fmt.Errorf("failed fetching package %s: %w", currentPkg, err) - } - - entry := determineEntry(destDir, *pkgInfo) - if entry != "" { - depsMap[currentPkg] = cleanRelativePath(filepath.Join(".deps", currentPkg, entry)) - } - - for nextDep, nextVer := range pkgInfo.Dependencies { - if !visited[nextDep] { - queue[nextDep] = nextVer - } - } - } - - return nil -} - -func fetchAndExtractPackage(client *http.Client, registry, pkgName, constraint, destDir string) (*PackageJSON, error) { - manifest, err := fetchPackageManifest(client, registry, pkgName) - if err != nil { - return nil, err - } - - resolvedVer := resolveVersion(manifest.DistTags.Latest, manifest.Versions, constraint) - verData, ok := manifest.Versions[resolvedVer] - if !ok { - return nil, fmt.Errorf("version %s not found in manifest for %s", resolvedVer, pkgName) - } - - tarballURL := verData.Dist.Tarball - if tarballURL == "" { - base := pkgName - if strings.Contains(base, "/") { - base = base[strings.LastIndex(base, "/")+1:] - } - tarballURL = fmt.Sprintf("%s/%s/-/%s-%s.tgz", strings.TrimSuffix(registry, "/"), pkgName, base, resolvedVer) - } - - tarballFile, err := os.CreateTemp("", "please_ts_dep_*.tgz") - if err != nil { - return nil, err - } - defer os.Remove(tarballFile.Name()) - defer tarballFile.Close() - - if err := downloadTarball(client, tarballURL, tarballFile); err != nil { - return nil, err - } - _ = tarballFile.Close() - - if err := os.MkdirAll(destDir, 0755); err != nil { - return nil, err - } - - if err := extractTarGz(tarballFile.Name(), destDir); err != nil { - return nil, fmt.Errorf("failed extracting %s: %w", tarballFile.Name(), err) - } - - pkg := readPackageJSON(destDir) - if pkg.Version == "" { - pkg.Version = resolvedVer - } - ensureCommonJSType(destDir) - - return &pkg, nil -} - -func fetchPackageManifest(client *http.Client, registry, pkgName string) (*npmManifest, error) { - manifestURL := strings.TrimSuffix(registry, "/") + "/" + url.PathEscape(pkgName) - req, err := http.NewRequest("GET", manifestURL, nil) - if err != nil { - return nil, err - } - req.Header.Set("Accept", "application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8, */*") - - resp, err := client.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("registry returned status %d for %s", resp.StatusCode, manifestURL) - } - - var manifest npmManifest - if err := json.NewDecoder(resp.Body).Decode(&manifest); err != nil { - return nil, fmt.Errorf("failed decoding manifest for %s: %w", pkgName, err) - } - return &manifest, nil -} - -func downloadTarball(client *http.Client, tarballURL string, destFile *os.File) error { - dlResp, err := client.Get(tarballURL) - if err != nil { - return fmt.Errorf("failed downloading tarball from %s: %w", tarballURL, err) - } - defer dlResp.Body.Close() - - if dlResp.StatusCode != http.StatusOK { - return fmt.Errorf("failed downloading tarball from %s: status %d", tarballURL, dlResp.StatusCode) - } - - _, err = io.Copy(destFile, dlResp.Body) - return err -} - func ensureCommonJSType(dir string) { pkgJSONPath := filepath.Join(dir, "package.json") data, err := os.ReadFile(pkgJSONPath) @@ -417,91 +255,6 @@ func ensureCommonJSType(dir string) { } } -func resolveVersion(latest string, versions map[string]npmVersionData, constraint string) string { - clean := strings.TrimSpace(constraint) - if clean == "" || clean == "*" || clean == "latest" { - if latest != "" { - return latest - } - } - if _, ok := versions[clean]; ok { - return clean - } - - prefix := "" - if strings.HasPrefix(clean, "^") || strings.HasPrefix(clean, "~") { - prefix = clean[:1] - clean = clean[1:] - } else if strings.HasPrefix(clean, ">=") { - prefix = ">=" - clean = strings.TrimSpace(clean[2:]) - } - - targetParts := parseSemver(clean) - var bestMatch string - var bestParts [3]int - - for ver := range versions { - parts := parseSemver(ver) - if matchesConstraint(parts, targetParts, prefix) { - if bestMatch == "" || compareSemver(parts, bestParts) > 0 { - bestMatch = ver - bestParts = parts - } - } - } - - if bestMatch != "" { - return bestMatch - } - if latest != "" { - return latest - } - for ver := range versions { - return ver - } - return constraint -} - -func matchesConstraint(parts, targetParts [3]int, prefix string) bool { - switch prefix { - case "^": - return parts[0] == targetParts[0] && compareSemver(parts, targetParts) >= 0 - case "~": - return parts[0] == targetParts[0] && parts[1] == targetParts[1] && compareSemver(parts, targetParts) >= 0 - case ">=": - return compareSemver(parts, targetParts) >= 0 - default: - return compareSemver(parts, targetParts) == 0 - } -} - -func parseSemver(v string) [3]int { - v = strings.TrimPrefix(v, "v") - if idx := strings.IndexAny(v, "-+"); idx != -1 { - v = v[:idx] - } - parts := strings.Split(v, ".") - var res [3]int - for i := 0; i < len(parts) && i < 3; i++ { - n, _ := strconv.Atoi(parts[i]) - res[i] = n - } - return res -} - -func compareSemver(a, b [3]int) int { - for i := 0; i < 3; i++ { - if a[i] > b[i] { - return 1 - } - if a[i] < b[i] { - return -1 - } - } - return 0 -} - func extractArchive(archivePath, destDir string) error { if strings.HasSuffix(archivePath, ".zip") { return extractZip(archivePath, destDir) diff --git a/tools/please_ts/unpack/unpack_test.go b/tools/please_ts/unpack/unpack_test.go index 88f7831..641206b 100644 --- a/tools/please_ts/unpack/unpack_test.go +++ b/tools/please_ts/unpack/unpack_test.go @@ -5,6 +5,8 @@ import ( "archive/zip" "bytes" "compress/gzip" + "crypto/sha512" + "encoding/base64" "encoding/json" "fmt" "net/http" @@ -46,85 +48,6 @@ func createTestTarball(t *testing.T, files map[string]string) []byte { return buf.Bytes() } -func TestSemverParseMatrix(t *testing.T) { - tests := []struct { - input string - want [3]int - }{ - {"1.2.3", [3]int{1, 2, 3}}, - {"v2.10.4", [3]int{2, 10, 4}}, - {"0.4.0-alpha.1", [3]int{0, 4, 0}}, - {"1.0.0+build.1", [3]int{1, 0, 0}}, - {"3", [3]int{3, 0, 0}}, - {"1.5", [3]int{1, 5, 0}}, - } - - for _, tt := range tests { - t.Run(tt.input, func(t *testing.T) { - got := parseSemver(tt.input) - if got != tt.want { - t.Errorf("parseSemver(%q) = %v, want %v", tt.input, got, tt.want) - } - }) - } -} - -func TestSemverCompareMatrix(t *testing.T) { - tests := []struct { - a [3]int - b [3]int - want int - }{ - {[3]int{1, 2, 3}, [3]int{1, 2, 3}, 0}, - {[3]int{2, 0, 0}, [3]int{1, 9, 9}, 1}, - {[3]int{1, 2, 0}, [3]int{1, 3, 0}, -1}, - {[3]int{1, 2, 4}, [3]int{1, 2, 3}, 1}, - {[3]int{1, 2, 3}, [3]int{1, 2, 4}, -1}, - } - - for _, tt := range tests { - t.Run(fmt.Sprintf("%v_vs_%v", tt.a, tt.b), func(t *testing.T) { - got := compareSemver(tt.a, tt.b) - if got != tt.want { - t.Errorf("compareSemver(%v, %v) = %d, want %d", tt.a, tt.b, got, tt.want) - } - }) - } -} - -func TestResolveVersionMatrix(t *testing.T) { - versions := map[string]npmVersionData{ - "1.0.0": {Version: "1.0.0"}, - "1.1.0": {Version: "1.1.0"}, - "1.2.3": {Version: "1.2.3"}, - "2.0.0": {Version: "2.0.0"}, - "2.1.0": {Version: "2.1.0"}, - } - - tests := []struct { - constraint string - latest string - want string - }{ - {"^1.0.0", "2.1.0", "1.2.3"}, - {"~1.1.0", "2.1.0", "1.1.0"}, - {">=2.0.0", "2.1.0", "2.1.0"}, - {"1.0.0", "2.1.0", "1.0.0"}, - {"latest", "2.1.0", "2.1.0"}, - {"*", "2.1.0", "2.1.0"}, - {"", "2.1.0", "2.1.0"}, - } - - for _, tt := range tests { - t.Run(tt.constraint, func(t *testing.T) { - got := resolveVersion(tt.latest, versions, tt.constraint) - if got != tt.want { - t.Errorf("resolveVersion(latest=%q, constraint=%q) = %q, want %q", tt.latest, tt.constraint, got, tt.want) - } - }) - } -} - func TestCleanRelativePathMatrix(t *testing.T) { tests := []struct { input string @@ -323,6 +246,8 @@ func TestUnpackWithMockRegistryMatrix(t *testing.T) { "index.js": `module.exports = { helper: true };`, }) + helperSum := sha512.Sum512(helperTarball) + helperIntegrity := "sha512-" + base64.StdEncoding.EncodeToString(helperSum[:]) var server *httptest.Server server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if strings.HasSuffix(r.URL.Path, "/helper-pkg") { @@ -332,10 +257,10 @@ func TestUnpackWithMockRegistryMatrix(t *testing.T) { "versions": { "1.0.0": { "version": "1.0.0", - "dist": { "tarball": "%s/helper-pkg/-/helper-pkg-1.0.0.tgz" } + "dist": { "tarball": "%s/helper-pkg/-/helper-pkg-1.0.0.tgz", "integrity": "%s" } } } - }`, server.URL) + }`, server.URL, helperIntegrity) w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(manifest)) return