diff --git a/.plzconfig b/.plzconfig index 04adcab..0817696 100644 --- a/.plzconfig +++ b/.plzconfig @@ -14,6 +14,7 @@ PleaseTsTool = //tools/please_ts:please_ts DenoTool = //tools/ts_toolchain:toolchain|deno BrowserTool = //tools/browser_toolchain:toolchain|chromium VitestTool = //tools/vitest_toolchain:toolchain +EsbuildTool = //tools/esbuild_toolchain:toolchain|esbuild [cover] fileextension = .ts @@ -45,6 +46,12 @@ DefaultValue = //tools/vitest_toolchain:toolchain Help = Path to the hermetic Vitest cache or toolchain target to use Inherit = true +[PluginConfig "esbuild_tool"] +ConfigKey = EsbuildTool +DefaultValue = //tools/esbuild_toolchain:toolchain|esbuild +Help = Path to the pinned esbuild binary or toolchain target that bundles npm packages +Inherit = true + [PluginConfig "coverage"] ConfigKey = Coverage DefaultValue = true diff --git a/CHANGELOG.md b/CHANGELOG.md index 40b1d92..7725ae4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,12 +28,25 @@ and this project adheres to fixes the result and nothing needs configuring. - `please_ts unpack --npm-cache`, and the `npmcache`, `registry` and `semver` packages behind it. +- npm packages (`ts_npm_module`) work in `ts_bundle`, `ts_binary`, the Vitest + runner and `ts_browser_test`, not only in type checks and Deno tests. Bundles + are built by `deno bundle` with the pinned esbuild of the new + `esbuild_toolchain` (`EsbuildTool`; Deno would otherwise download its own, and + the build fails if it did). `ts_binary` embeds the packages and runs without a + cache or network. Vitest gets them as a `node_modules` directory that Deno + materializes offline from the merged cache in the test's working directory + (under `plz-out/tmp` only). - Fixtures for `debug` (CommonJS, transitive dependency), `highlight.js` (CommonJS, subpaths) and `@codemirror/legacy-modes` (exports map only), automatic and fully pinned. ### Fixed +- Targets that use the Vitest runner no longer write into the shared + `vitest_toolchain` output. Vitest and the type check of a `ts_test` ran the + unversioned `npm:vitest`, which resolves to the registry's latest release + rather than the cached one and was downloaded into that cache at run time; + they now use the cached version, and the `package.json` for coverage does too. - `ts_module`'s automatic dependency resolution (`resolve_transitive`) is now strict and verified. It used to fall back to the `latest` version when no version satisfied a range, verify no download against the registry's integrity diff --git a/build_defs/ts/constants.build_defs b/build_defs/ts/constants.build_defs index 670c8a7..af8325a 100644 --- a/build_defs/ts/constants.build_defs +++ b/build_defs/ts/constants.build_defs @@ -24,6 +24,31 @@ DENO_PLATFORM_MAP = { }, } +# The esbuild that `deno bundle` runs, as the npm package that carries its binary. Deno looks for +# it under DENO_DIR/dl/esbuild- and downloads its own otherwise: the version here must be +# the one the default Deno version (above) uses, and tools/please_ts/bundle/denobundle.go maps +# the Deno version to the directory name. +DEFAULT_ESBUILD_VERSION = "0.25.5" + +ESBUILD_PLATFORM_MAP = { + "linux_amd64": { + "url": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.5.tgz", + "hash": "95a928d8187c6f7ad632a3f3bbf01f66dfd5b5adb724b8bfeec5fff73ff2d91a", + }, + "linux_arm64": { + "url": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.5.tgz", + "hash": "ae5a580130af1a7c7b8760466c9ae85a25782e63330b8c2855cc15ed007dd30d", + }, + "darwin_arm64": { + "url": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.5.tgz", + "hash": "61a312bcb8249d058639c405cf6378dd3107de5535a9974973c48a6dd0d2d062", + }, + "darwin_amd64": { + "url": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.5.tgz", + "hash": "d2890be89bd7e322cd83e1665516d3c71b0ac3952655cb27076eddb621a4af24", + }, +} + DEFAULT_CHROMIUM_VERSION = "154.0.8037.57" CHROMIUM_PLATFORM_MAP = { diff --git a/build_defs/ts/ts.build_defs b/build_defs/ts/ts.build_defs index 04ffa82..201d7f6 100644 --- a/build_defs/ts/ts.build_defs +++ b/build_defs/ts/ts.build_defs @@ -24,6 +24,15 @@ def _browser_tool_arg(tools, browser_binary = ""): return f"--browser-binary '{tool}'" return "" +def _esbuild_tool_arg(tools): + tool = CONFIG.TS.ESBUILD_TOOL + if tool and (tool.startswith("//") or tool.startswith("///") or tool.startswith(":")): + tools["ESBUILD"] = [tool] + return '--esbuild-binary "$TOOLS_ESBUILD"' + elif tool: + return f"--esbuild-binary '{tool}'" + return "" + def _vitest_tool_arg(tools, vitest_dir = ""): tool = vitest_dir or CONFIG.TS.VITEST_TOOL if not tool: @@ -145,6 +154,47 @@ def browser_toolchain( building_description = f"Assembling hermetic headless Chromium toolchain {version}...", ) +def esbuild_toolchain( + name: str = "toolchain", + version: str = DEFAULT_ESBUILD_VERSION, + esbuild_url: str = "", + esbuild_hash: str = "", + target_platform: str = "", + labels: list = None, + visibility: list = None): + """Provides the pinned esbuild binary that `deno bundle` runs to bundle npm packages. + + Deno downloads its own esbuild when it does not find one in its cache. Bundling puts this + one there, taken from a hash-pinned download, so that nothing is fetched while building. + The version must be the one the Deno version in use expects (see DEFAULT_ESBUILD_VERSION). + """ + plat = target_platform or f"{CONFIG.OS}_{CONFIG.ARCH}" + esbuild_info = ESBUILD_PLATFORM_MAP.get(plat, ESBUILD_PLATFORM_MAP["linux_amd64"]) + + dl_target = f"_{name}_esbuild_dl" + remote_file( + name = dl_target, + url = esbuild_url or esbuild_info["url"], + hashes = [esbuild_hash or esbuild_info["hash"]], + out = f"{name}.tgz", + ) + + tools = {"TOOL": [_ts_tool()]} + return build_rule( + name = name, + srcs = [f":{dl_target}"], + outs = [name], + cmd = '$TOOLS_TOOL unpack --archive "$SRCS" --out "$OUT/bin" --binary "esbuild"', + entry_points = { + "esbuild": f"{name}/bin/esbuild", + }, + binary = True, + tools = tools, + visibility = visibility or ["PUBLIC"], + labels = labels or ["ts", "toolchain", "esbuild"], + building_description = f"Unpacking the pinned esbuild {version}...", + ) + def vitest_toolchain( name: str = "toolchain", version: str = DEFAULT_VITEST_VERSION, @@ -433,6 +483,8 @@ def ts_bundle( flags_joined = " ".join(flags) if flags else "" flags_str = f"--flags '{flags_joined}'" if flags else "" + esbuild_arg = _esbuild_tool_arg(tools) + resolved_module = _resolve_module_name(name, module_name) bundle_args = [ '--out "$OUT"', @@ -444,6 +496,7 @@ def ts_bundle( '--minify' if minify else "", '--sourcemap' if sourcemap else "", f'--bundler-tool "{bundler_tool}"' if bundler_tool else "", + esbuild_arg, deno_arg, flags_str, ] @@ -554,6 +607,7 @@ def ts_browser_test( tools = {"TOOL": [_ts_tool()]} deno_arg = _deno_tool_arg(tools) browser_arg = _browser_tool_arg(tools, browser_binary) + esbuild_arg = _esbuild_tool_arg(tools) flags_joined = " ".join(flags) if flags else "" flags_str = f"--flags '{flags_joined}'" if flags else "" @@ -576,6 +630,7 @@ def ts_browser_test( f'--runner "{runner}"', f'--browser "{browser}"', browser_arg, + esbuild_arg, f'--srcs "{srcs_list}"', '--deps "${DEPS:-}"', f'--module-name "{resolved_module}"' if resolved_module else "", diff --git a/docs/ts/rules.md b/docs/ts/rules.md index 62e6734..1bb5a40 100644 --- a/docs/ts/rules.md +++ b/docs/ts/rules.md @@ -162,6 +162,11 @@ ts_bundle( | `sourcemap` | `bool` | `False` | Whether to generate source maps. | | `bundler_tool` | `str` | `""` | Optional external bundler executable (e.g. esbuild). | +With npm packages (`ts_npm_module` dependencies) the bundle is built by +`deno bundle`, using the pinned esbuild of `esbuild_toolchain`, and +`bundler_tool` is not used. See +[Bundling and compiling](usage.md#bundling-and-compiling). + --- ## `ts_test` @@ -217,6 +222,30 @@ vitest_toolchain( --- +## `esbuild_toolchain` + +Provides the pinned esbuild binary that `deno bundle` runs to bundle npm +packages (Deno would download its own). The `EsbuildTool` config key points at +it (`//tools/esbuild_toolchain:toolchain|esbuild` by default). + +```starlark +esbuild_toolchain( + name = "toolchain", +) +``` + +### Arguments + +| Argument | Type | Default | Description | +| :---------------- | :---- | :------------ | :----------------------------------------------------------------- | +| `name` | `str` | `"toolchain"` | Target name. | +| `version` | `str` | `"0.25.5"` | esbuild version; must be the one the Deno version uses. | +| `esbuild_url` | `str` | `""` | Override the download URL (the `@esbuild/` npm tarball). | +| `esbuild_hash` | `str` | `""` | sha256 of that download. | +| `target_platform` | `str` | `""` | Platform (defaults to the host). | + +--- + ## `browser_toolchain` Assembles a 100% hermetic headless browser (`chrome-headless-shell`) binary diff --git a/docs/ts/usage.md b/docs/ts/usage.md index 4c6a586..be18e1f 100644 --- a/docs/ts/usage.md +++ b/docs/ts/usage.md @@ -197,8 +197,9 @@ The Deno runner (the default) uses `deno test --coverage`; the Vitest runner from lcov. Vitest writes its lcov report to a scratch directory (Vitest empties its reports directory, so it cannot be the test working directory). - **Vitest note**: when coverage is active the runner writes a `package.json` - declaring `vitest` and `@vitest/coverage-v8` in the test working directory (if - none exists) so Deno can resolve the provider. + declaring `vitest` and `@vitest/coverage-v8`, at the versions in the Vitest + cache, in the test working directory (if none exists) so Deno can resolve the + provider. ### Raw lcov export (functions and branches) @@ -287,11 +288,47 @@ import createDebug from "debug"; // CommonJS, and it needs the package "ms" - **Resolution is Deno's**: `exports` maps, CommonJS, subpaths and a package's own types work as they do for `npm:` specifiers. Imports use the plain package name. -- **Not wired yet**: `ts_bundle` and `ts_binary`, the Vitest and browser - runners, and a helper that prints pinned declarations. The cache layout +- **Where it works**: `ts_library` (type checks), `ts_test` with the Deno and + the Vitest runner, `ts_browser_test`, `ts_bundle` and `ts_binary`. Not done + yet: a helper that prints pinned declarations. The cache layout (`registry.json`, including its `_deno.packumentFormat` key) is internal to Deno, so it is tied to the Deno version the plugin pins. +### Bundling and compiling + +`ts_bundle` (and the bundle `ts_browser_test` builds for the browser) runs +`deno bundle` when the target depends on npm packages, so Deno resolves them +exactly as it does for `ts_test`. `deno bundle` runs esbuild, which Deno would +download itself; the plugin puts the pinned binary of `esbuild_toolchain` +(`EsbuildTool`, `//tools/esbuild_toolchain:toolchain|esbuild` by default, hash +pinned per platform) in the cache where Deno looks for it, and fails the build +if Deno downloaded one anyway. The bundle names its inputs relative to the +working directory, so the same sources give the same bundle. Targets without npm +packages bundle through esbuild with import map aliases, as before. + +`ts_binary` merges the slices and runs `deno compile --cached-only`; the +executable embeds the packages and runs without a cache or network. +(`deno compile` fetches its own `denort` runtime from `dl.deno.land` on every +build, with or without npm packages: not pinned yet.) + +Two limits: `deno bundle` is marked experimental by Deno, and the directory it +looks in for esbuild is internal to Deno, so the plugin knows it per Deno +version (`esbuildCacheVersions` in `tools/please_ts/bundle/denobundle.go`, and +`--esbuild-cache-version` for a version it does not know). The bundles and the +binary in `test/ts/npm_cache` fail on a Deno bump until that is updated. + +### Vitest + +Vitest cannot import `npm:` specifiers (nor can Vite resolve them), so for a +target with npm packages the Vitest runner declares them, at their exact +versions, in a `package.json` and lets +`deno run --node-modules-dir=auto --cached-only` materialize them in a +`node_modules` directory, offline, from the merged cache. Both exist only in the +test's working directory under `plz-out/tmp`, which Please deletes after the +run; nothing is written to the source tree. The Vitest cache of +`vitest_toolchain` is copied for such a target, so the slices never end up in +that shared output, and Vitest itself is pinned to the version in that cache. + `ts_module` resolves its dependencies by the same rules (strictly, verified, as of the root's publish day), with the limit of one version per package: two dependents that need incompatible versions fail the build and name each other. diff --git a/test/ts/npm_cache/BUILD b/test/ts/npm_cache/BUILD index 101bc0d..7dd4bf6 100644 --- a/test/ts/npm_cache/BUILD +++ b/test/ts/npm_cache/BUILD @@ -131,3 +131,107 @@ ts_test( ":modes_auto", ], ) + +# Bundling and compiling with npm packages: Deno resolves them from the cache slices (no network, +# no node_modules), and `deno bundle` runs the pinned esbuild. +ts_bundle( + name = "highlight_bundle", + srcs = ["highlight_main.ts"], + format = "iife", + main = "highlight_main.ts", + deps = [ + ":highlight", + "//test/ts/npm_cache/modules:highlight_js", + ], +) + +ts_binary( + name = "highlight_app", + srcs = ["highlight_main.ts"], + main = "highlight_main.ts", + deps = [ + ":highlight", + "//test/ts/npm_cache/modules:highlight_js", + ], +) + +# The exports-only package with its declared closure, and with the closure resolved automatically. +ts_bundle( + name = "modes_bundle", + srcs = ["modes.ts"], + format = "iife", + main = "modes.ts", + deps = [ + "//test/ts/npm_cache/modules:codemirror_language", + "//test/ts/npm_cache/modules:codemirror_legacy_modes", + ], +) + +ts_bundle( + name = "modes_auto_bundle", + srcs = ["modes.ts"], + format = "iife", + main = "modes.ts", + deps = [ + ":codemirror_language_auto", + ":legacy_modes_auto", + ], +) + +gentest( + name = "bundles_test", + data = { + "highlight": [":highlight_bundle"], + "modes": [":modes_bundle"], + "modes_auto": [":modes_auto_bundle"], + "app": [":highlight_app"], + }, + no_test_output = True, + test_cmd = " && ".join([ + # self-contained: the npm packages are inlined, no specifier is left to resolve at run time + '! grep -q "npm:" $DATA_HIGHLIGHT', + '! grep -q "highlight.js/lib" $DATA_HIGHLIGHT', + # ...and their code really is in there: highlight.js's Go grammar, CodeMirror's StreamLanguage + 'grep -q "registerLanguage" $DATA_HIGHLIGHT', + 'grep -q "golang" $DATA_HIGHLIGHT', + '! grep -q "npm:" $DATA_MODES', + 'grep -q "StreamLanguage" $DATA_MODES', + 'grep -q "StreamLanguage" $DATA_MODES_AUTO', + # the compiled program carries highlight.js and runs without any cache or network + 'out=$(DENO_DIR=/nonexistent $DATA_APP)', + 'echo "$out" | grep -q "hljs-keyword"', + ]), +) + +# The browser runner bundles the test with its npm packages, then runs it in headless Chromium. +ts_browser_test( + name = "highlight_browser_test", + srcs = ["highlight_browser_test.ts"], + deps = [ + ":highlight", + "//test/ts/npm_cache/modules:highlight_js", + ], +) + +# Vitest imports the npm packages natively (Deno resolves them from the merged cache) instead of +# trying to resolve them itself. +ts_test( + name = "highlight_vitest_test", + srcs = ["highlight_vitest_test.ts"], + runner = "vitest", + deps = [ + ":highlight", + "//test/ts/npm_cache/modules:highlight_js", + ], +) + +ts_test( + name = "modes_vitest_test", + srcs = ["modes_vitest_test.ts"], + runner = "vitest", + deps = [ + ":modes", + "//test/ts/npm_cache/modules:codemirror_language", + "//test/ts/npm_cache/modules:codemirror_legacy_modes", + ], +) diff --git a/test/ts/npm_cache/highlight_browser_test.ts b/test/ts/npm_cache/highlight_browser_test.ts new file mode 100644 index 0000000..95ec844 --- /dev/null +++ b/test/ts/npm_cache/highlight_browser_test.ts @@ -0,0 +1,12 @@ +import { highlight } from "@test/highlight"; + +declare const Deno: { + test: (name: string, fn: () => void | Promise) => void; +}; +declare const expect: (actual: any) => { + toBeTruthy: () => void; +}; + +Deno.test("highlight.js (an npm package) runs inside the browser", () => { + expect(highlight("package main").includes("hljs-keyword")).toBeTruthy(); +}); diff --git a/test/ts/npm_cache/highlight_main.ts b/test/ts/npm_cache/highlight_main.ts new file mode 100644 index 0000000..bb2b1fc --- /dev/null +++ b/test/ts/npm_cache/highlight_main.ts @@ -0,0 +1,3 @@ +import { highlight } from "@test/highlight"; + +console.log(highlight("package main")); diff --git a/test/ts/npm_cache/highlight_vitest_test.ts b/test/ts/npm_cache/highlight_vitest_test.ts new file mode 100644 index 0000000..693f711 --- /dev/null +++ b/test/ts/npm_cache/highlight_vitest_test.ts @@ -0,0 +1,8 @@ +import { describe, expect, it } from "vitest"; +import { highlight } from "@test/highlight"; + +describe("an npm package under Vitest", () => { + it("is imported by a first-party library through subpaths of a CommonJS package", () => { + expect(highlight("package main")).toContain("hljs-keyword"); + }); +}); diff --git a/test/ts/npm_cache/modes_vitest_test.ts b/test/ts/npm_cache/modes_vitest_test.ts new file mode 100644 index 0000000..91ee06a --- /dev/null +++ b/test/ts/npm_cache/modes_vitest_test.ts @@ -0,0 +1,8 @@ +import { describe, expect, it } from "vitest"; +import { goLanguage } from "@test/modes"; + +describe("an exports-only npm package and its dependency closure under Vitest", () => { + it("builds a CodeMirror language from a legacy mode", () => { + expect(goLanguage().name).toBe("go"); + }); +}); diff --git a/tools/esbuild_toolchain/BUILD b/tools/esbuild_toolchain/BUILD new file mode 100644 index 0000000..615d199 --- /dev/null +++ b/tools/esbuild_toolchain/BUILD @@ -0,0 +1,6 @@ +subinclude("//build_defs/ts:ts") + +esbuild_toolchain( + name = "toolchain", + visibility = ["PUBLIC"], +) diff --git a/tools/please_ts/binary/BUILD b/tools/please_ts/binary/BUILD index 49a68ef..7774dc7 100644 --- a/tools/please_ts/binary/BUILD +++ b/tools/please_ts/binary/BUILD @@ -4,7 +4,10 @@ go_library( name = "binary", srcs = ["binary.go"], visibility = ["//tools/please_ts/..."], - deps = ["//tools/please_ts/importmap"], + deps = [ + "//tools/please_ts/importmap", + "//tools/please_ts/npmcache", + ], ) go_test( @@ -13,5 +16,8 @@ go_test( "binary.go", "binary_test.go", ], - deps = ["//tools/please_ts/importmap"], + deps = [ + "//tools/please_ts/importmap", + "//tools/please_ts/npmcache", + ], ) diff --git a/tools/please_ts/binary/binary.go b/tools/please_ts/binary/binary.go index 1972486..456fde5 100644 --- a/tools/please_ts/binary/binary.go +++ b/tools/please_ts/binary/binary.go @@ -7,6 +7,7 @@ import ( "path/filepath" "tools/please_ts/importmap" + "tools/please_ts/npmcache" ) // Options holds configuration for compiling a standalone executable via deno compile. @@ -45,6 +46,12 @@ func Run(opts Options) error { return fmt.Errorf("failed to create DENO_DIR: %w", err) } + // npm packages provided by ts_npm_module targets are merged into the per-run Deno cache, + // and the executable embeds them from there. + if _, err := npmcache.Prepare(".", denoCacheDir); err != nil { + return fmt.Errorf("failed preparing the npm cache: %w", err) + } + // Synthesize import map importMapPath := ".import_map.json" im, err := importmap.Synthesize(opts.ModuleName, opts.Srcs, opts.Deps, ".") @@ -69,6 +76,11 @@ func Run(opts Options) error { "--import-map", importMapPath, "-o", opts.Out, } + // npm packages come from the merged cache only: a ts_npm_module missing from the + // target's deps must fail here, not be downloaded. + if im.HasNpmSpecifiers() { + args = append(args, "--cached-only") + } args = append(args, opts.Flags...) args = append(args, opts.Main) diff --git a/tools/please_ts/binary/binary_test.go b/tools/please_ts/binary/binary_test.go index 0a179b5..635fd51 100644 --- a/tools/please_ts/binary/binary_test.go +++ b/tools/please_ts/binary/binary_test.go @@ -1,7 +1,12 @@ package binary import ( + "os" + "path/filepath" + "strings" "testing" + + "tools/please_ts/npmcache" ) func TestOptionsValidation(t *testing.T) { @@ -15,3 +20,73 @@ func TestOptionsValidation(t *testing.T) { t.Errorf("expected error when Out is empty") } } + +// fakeDeno writes a `deno` that records its arguments and the files below DENO_DIR/npm. +func fakeDeno(t *testing.T) (bin, log string) { + t.Helper() + dir := t.TempDir() + log = filepath.Join(dir, "calls.log") + bin = filepath.Join(dir, "deno") + script := `#!/bin/sh +{ + echo "args: $*" + (cd "$DENO_DIR" && find npm -type d 2>/dev/null | sort | sed 's/^/dir: /') +} >> "` + log + `" +out="" +while [ $# -gt 0 ]; do if [ "$1" = "-o" ]; then out="$2"; fi; shift; done +[ -n "$out" ] && echo binary > "$out" +` + if err := os.WriteFile(bin, []byte(script), 0755); err != nil { + t.Fatal(err) + } + return bin, log +} + +func inDir(t *testing.T) string { + t.Helper() + dir := t.TempDir() + old, _ := os.Getwd() + if err := os.Chdir(dir); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chdir(old) }) + return dir +} + +func TestRunUsesTheNpmCacheSlicesOfItsDependenciesOffline(t *testing.T) { + dir := inDir(t) + slice := filepath.Join(dir, "third_party", "debug") + if err := os.MkdirAll(filepath.Join(slice, "npm", "registry.npmjs.org", "debug", "4.3.7"), 0755); err != nil { + t.Fatal(err) + } + if err := npmcache.Write(slice, npmcache.Slice{Name: "debug", Version: "4.3.7", Specifier: "npm:debug@4.3.7"}); err != nil { + t.Fatal(err) + } + deno, log := fakeDeno(t) + + if err := Run(Options{Deno: deno, Out: "bin/app", Main: "main.ts", Srcs: []string{"main.ts"}}); err != nil { + t.Fatal(err) + } + data, _ := os.ReadFile(log) + calls := string(data) + for _, want := range []string{ + "args: compile --no-remote --unstable-detect-cjs --import-map .import_map.json -o bin/app --cached-only main.ts", + "dir: npm/registry.npmjs.org/debug/4.3.7", // the slice was merged into the per-run DENO_DIR + } { + if !strings.Contains(calls, want) { + t.Errorf("missing %q in:\n%s", want, calls) + } + } +} + +func TestRunWithoutNpmPackagesDoesNotRestrictDenoToTheCache(t *testing.T) { + inDir(t) + deno, log := fakeDeno(t) + if err := Run(Options{Deno: deno, Out: "app", Main: "main.ts", Srcs: []string{"main.ts"}}); err != nil { + t.Fatal(err) + } + data, _ := os.ReadFile(log) + if strings.Contains(string(data), "--cached-only") { + t.Errorf("--cached-only without npm packages:\n%s", data) + } +} diff --git a/tools/please_ts/bundle/BUILD b/tools/please_ts/bundle/BUILD index 42c88b9..9aa8a50 100644 --- a/tools/please_ts/bundle/BUILD +++ b/tools/please_ts/bundle/BUILD @@ -2,9 +2,15 @@ subinclude("///go//build_defs:go") go_library( name = "bundle", - srcs = ["bundle.go"], + srcs = [ + "bundle.go", + "denobundle.go", + ], visibility = ["//tools/please_ts/..."], - deps = ["//tools/please_ts/importmap"], + deps = [ + "//tools/please_ts/importmap", + "//tools/please_ts/npmcache", + ], ) go_test( @@ -12,6 +18,11 @@ go_test( srcs = [ "bundle.go", "bundle_test.go", + "denobundle.go", + "denobundle_test.go", + ], + deps = [ + "//tools/please_ts/importmap", + "//tools/please_ts/npmcache", ], - deps = ["//tools/please_ts/importmap"], ) diff --git a/tools/please_ts/bundle/bundle.go b/tools/please_ts/bundle/bundle.go index 1bc6d39..2c13654 100644 --- a/tools/please_ts/bundle/bundle.go +++ b/tools/please_ts/bundle/bundle.go @@ -24,6 +24,14 @@ type Options struct { Minify bool Sourcemap bool Flags []string + + // EsbuildBinary is the pinned esbuild executable that `deno bundle` runs. It is needed + // when the target depends on npm packages (ts_npm_module), which esbuild cannot resolve + // itself: Deno does, and bundles through esbuild. + EsbuildBinary string + // EsbuildCacheVersion overrides the directory (esbuild-) under DENO_DIR/dl in + // which that Deno version looks for its esbuild, for a Deno version this tool does not know. + EsbuildCacheVersion string } // Run bundles the source files into a single distribution asset using esbuild by default. @@ -39,7 +47,17 @@ func Run(opts Options) error { return err } - return runEsbuildBundler(opts) + im, err := importmap.Synthesize(opts.ModuleName, opts.Srcs, opts.Deps, ".") + if err != nil { + return fmt.Errorf("failed synthesizing import map: %w", err) + } + + // npm packages are resolved by Deno from the cache slices of the ts_npm_module targets + // the target depends on; everything else goes through esbuild with aliases. + if im.HasNpmSpecifiers() { + return runDenoBundle(opts, im) + } + return runEsbuildBundler(opts, im) } func isValidEsbuildAlias(name string) bool { @@ -114,12 +132,7 @@ func buildEsbuildArgs(opts Options, im *importmap.ImportMap) []string { return args } -func runEsbuildBundler(opts Options) error { - im, err := importmap.Synthesize(opts.ModuleName, opts.Srcs, opts.Deps, ".") - if err != nil { - return fmt.Errorf("failed synthesizing import map: %w", err) - } - +func runEsbuildBundler(opts Options, im *importmap.ImportMap) error { args := buildEsbuildArgs(opts, im) // Execute bundler: diff --git a/tools/please_ts/bundle/denobundle.go b/tools/please_ts/bundle/denobundle.go new file mode 100644 index 0000000..ae0a9d1 --- /dev/null +++ b/tools/please_ts/bundle/denobundle.go @@ -0,0 +1,145 @@ +package bundle + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + + "tools/please_ts/importmap" + "tools/please_ts/npmcache" +) + +// esbuildCacheVersions maps a Deno version to the esbuild directory under DENO_DIR/dl in +// which `deno bundle` looks for the esbuild binary before it downloads one. The layout is +// internal to Deno: add an entry when the default Deno version of the toolchain moves (the npm +// bundle fixtures under test/ts/npm_cache fail until it is there). +var esbuildCacheVersions = map[string]string{ + "2.9.7": "0.25.5-1", +} + +// denoVersion returns the version of the deno executable. +func denoVersion(denoBin string) (string, error) { + out, err := exec.Command(denoBin, "--version").Output() + if err != nil { + return "", fmt.Errorf("running %s --version: %w", denoBin, err) + } + // The first line reads "deno 2.9.7 (stable, release, x86_64-unknown-linux-gnu)". + first, _, _ := strings.Cut(strings.TrimSpace(string(out)), "\n") + fields := strings.Fields(first) + if len(fields) < 2 || fields[0] != "deno" { + return "", fmt.Errorf("cannot read the version from %q", first) + } + return fields[1], nil +} + +// esbuildPlatform names the platform the way Deno does in its esbuild file name. +func esbuildPlatform(goos, goarch string) (string, error) { + arch := map[string]string{"amd64": "x64", "arm64": "arm64"}[goarch] + if (goos != "linux" && goos != "darwin") || arch == "" { + return "", fmt.Errorf("no esbuild binary for %s/%s", goos, goarch) + } + return goos + "-" + arch, nil +} + +// placeEsbuild makes `deno bundle` find the pinned esbuild in denoDir instead of +// downloading its own: Deno looks for dl/esbuild-/esbuild--. +func placeEsbuild(denoBin, denoDir string, opts Options) error { + if opts.EsbuildBinary == "" { + return fmt.Errorf("bundling npm packages needs the pinned esbuild binary (the esbuild_tool config, //tools/esbuild_toolchain by default); none was given") + } + version := opts.EsbuildCacheVersion + if version == "" { + deno, err := denoVersion(denoBin) + if err != nil { + return err + } + var ok bool + if version, ok = esbuildCacheVersions[deno]; !ok { + return fmt.Errorf("this tool does not know where Deno %s looks for esbuild; pass --esbuild-cache-version (the of DENO_DIR/dl/esbuild-)", deno) + } + } + platform, err := esbuildPlatform(runtime.GOOS, runtime.GOARCH) + if err != nil { + return err + } + src, err := filepath.Abs(opts.EsbuildBinary) + if err != nil { + return err + } + if _, err := os.Stat(src); err != nil { + return fmt.Errorf("esbuild binary: %w", err) + } + dir := filepath.Join(denoDir, "dl", "esbuild-"+version) + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + return os.Symlink(src, filepath.Join(dir, "esbuild-"+platform)) +} + +// runDenoBundle bundles with `deno bundle`, which resolves the npm: specifiers of the import +// map from the merged cache slices (offline: no node_modules, no lockfile, nothing fetched) +// and runs the pinned esbuild. +func runDenoBundle(opts Options, im *importmap.ImportMap) error { + denoBin := opts.Deno + if denoBin == "" { + denoBin = "deno" + } + + // The bundle names the files it was built from, relative to the working directory, so the + // per-run DENO_DIR is a fixed directory there and not a random one under the temp + // directory: the same sources must give the same bundle. + denoDir, err := filepath.Abs(".please_ts_deno") + if err != nil { + return err + } + if err := os.RemoveAll(denoDir); err != nil { + return err + } + if err := os.MkdirAll(denoDir, 0755); err != nil { + return fmt.Errorf("failed to create DENO_DIR: %w", err) + } + defer os.RemoveAll(denoDir) + if _, err := npmcache.Prepare(".", denoDir); err != nil { + return fmt.Errorf("failed preparing the npm cache: %w", err) + } + if err := placeEsbuild(denoBin, denoDir, opts); err != nil { + return err + } + + importMapPath := ".import_map.json" + if err := im.WriteToFile(importMapPath); err != nil { + return fmt.Errorf("failed writing import map: %w", err) + } + defer os.Remove(importMapPath) + + args := []string{"bundle", "--no-remote", "--import-map", importMapPath, "--platform", "browser", "-o", opts.Out} + if opts.Format != "" { + args = append(args, "--format", opts.Format) + } + if opts.Minify { + args = append(args, "--minify") + } + if opts.Sourcemap { + args = append(args, "--sourcemap") + } + args = append(args, opts.Flags...) + args = append(args, opts.Main) + + cmd := exec.Command(denoBin, args...) + cmd.Env = append(os.Environ(), "DENO_DIR="+denoDir) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("deno bundle failed: %w", err) + } + + // Deno fetches its own esbuild from the registry when it does not find the pinned one + // where it looks, and --no-remote does not stop that. Do not let that pass silently. + if _, err := os.Stat(filepath.Join(denoDir, "npm", "registry.npmjs.org", "@esbuild")); err == nil { + return fmt.Errorf("deno bundle downloaded esbuild instead of using the pinned binary: its lookup path for this Deno version differs from the one this tool uses (--esbuild-cache-version)") + } + return nil +} diff --git a/tools/please_ts/bundle/denobundle_test.go b/tools/please_ts/bundle/denobundle_test.go new file mode 100644 index 0000000..35837d6 --- /dev/null +++ b/tools/please_ts/bundle/denobundle_test.go @@ -0,0 +1,240 @@ +package bundle + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "tools/please_ts/npmcache" +) + +// fakeDeno writes an executable that behaves like `deno`: `--version` prints version, and any +// other invocation appends its arguments, DENO_DIR and the esbuild files visible under it to +// log, then runs extra (a shell snippet) and writes the bundle given by -o. +func fakeDeno(t *testing.T, version, extra string) (bin, log string) { + t.Helper() + dir := t.TempDir() + log = filepath.Join(dir, "calls.log") + bin = filepath.Join(dir, "deno") + script := `#!/bin/sh +if [ "$1" = "--version" ]; then echo "deno ` + version + ` (stable, release, x86_64-unknown-linux-gnu)"; exit 0; fi +{ + echo "args: $*" + echo "DENO_DIR=$DENO_DIR" + (cd "$DENO_DIR" && find dl -type l 2>/dev/null | sort | sed 's/^/esbuild: /') +} >> "` + log + `" +` + extra + ` +out="" +while [ $# -gt 0 ]; do if [ "$1" = "-o" ]; then out="$2"; fi; shift; done +[ -n "$out" ] && echo "bundled" > "$out" +exit 0 +` + if err := os.WriteFile(bin, []byte(script), 0755); err != nil { + t.Fatal(err) + } + return bin, log +} + +func fakeEsbuild(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "esbuild") + if err := os.WriteFile(path, []byte("#!/bin/sh\n"), 0755); err != nil { + t.Fatal(err) + } + return path +} + +// workDir makes a temporary directory holding one npm slice the current directory, as Please +// stages the slices of a target's dependencies. +func workDir(t *testing.T) string { + t.Helper() + dir := t.TempDir() + slice := filepath.Join(dir, "third_party", "debug") + if err := os.MkdirAll(filepath.Join(slice, "npm", "registry.npmjs.org", "debug", "4.3.7"), 0755); err != nil { + t.Fatal(err) + } + if err := npmcache.Write(slice, npmcache.Slice{Name: "debug", Version: "4.3.7", Specifier: "npm:debug@4.3.7"}); err != nil { + t.Fatal(err) + } + old, _ := os.Getwd() + if err := os.Chdir(dir); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chdir(old) }) + return dir +} + +func read(t *testing.T, path string) string { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return string(data) +} + +func TestEsbuildPlatform(t *testing.T) { + for _, c := range []struct{ os, arch, want string }{ + {"linux", "amd64", "linux-x64"}, + {"linux", "arm64", "linux-arm64"}, + {"darwin", "arm64", "darwin-arm64"}, + {"darwin", "amd64", "darwin-x64"}, + } { + if got, err := esbuildPlatform(c.os, c.arch); err != nil || got != c.want { + t.Errorf("%s/%s = %q, %v; want %q", c.os, c.arch, got, err, c.want) + } + } + for _, c := range [][2]string{{"windows", "amd64"}, {"linux", "386"}} { + if _, err := esbuildPlatform(c[0], c[1]); err == nil { + t.Errorf("%v: expected an error", c) + } + } +} + +func TestDenoVersion(t *testing.T) { + bin, _ := fakeDeno(t, "2.9.7", "") + if v, err := denoVersion(bin); err != nil || v != "2.9.7" { + t.Errorf("version = %q, %v", v, err) + } + notDeno := filepath.Join(t.TempDir(), "x") + _ = os.WriteFile(notDeno, []byte("#!/bin/sh\necho something else\n"), 0755) + if _, err := denoVersion(notDeno); err == nil || !strings.Contains(err.Error(), "something else") { + t.Errorf("err = %v", err) + } + if _, err := denoVersion(filepath.Join(t.TempDir(), "missing")); err == nil { + t.Error("expected an error for a missing deno") + } +} + +func TestPlaceEsbuild(t *testing.T) { + bin, _ := fakeDeno(t, "2.9.7", "") + esbuild := fakeEsbuild(t) + platform, err := esbuildPlatform(runtime.GOOS, runtime.GOARCH) + if err != nil { + t.Skip(err) + } + + denoDir := t.TempDir() + if err := placeEsbuild(bin, denoDir, Options{EsbuildBinary: esbuild}); err != nil { + t.Fatal(err) + } + link := filepath.Join(denoDir, "dl", "esbuild-0.25.5-1", "esbuild-"+platform) + if target, err := os.Readlink(link); err != nil || target != esbuild { + t.Errorf("link %s -> %q, %v; want %s", link, target, err, esbuild) + } + + // an explicit version wins over the table (and is the way out for an unknown Deno) + other, _ := fakeDeno(t, "9.9.9", "") + denoDir = t.TempDir() + if err := placeEsbuild(other, denoDir, Options{EsbuildBinary: esbuild, EsbuildCacheVersion: "1.2.3-4"}); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(filepath.Join(denoDir, "dl", "esbuild-1.2.3-4", "esbuild-"+platform)); err != nil { + t.Error(err) + } +} + +func TestPlaceEsbuildErrors(t *testing.T) { + bin, _ := fakeDeno(t, "2.9.7", "") + unknown, _ := fakeDeno(t, "9.9.9", "") + esbuild := fakeEsbuild(t) + + for name, c := range map[string]struct { + deno string + opts Options + want string + }{ + "no binary": {bin, Options{}, "needs the pinned esbuild"}, + "unknown deno": {unknown, Options{EsbuildBinary: esbuild}, "does not know where Deno 9.9.9 looks for esbuild"}, + "missing binary": {bin, Options{EsbuildBinary: filepath.Join(t.TempDir(), "gone")}, "esbuild binary"}, + "deno not found": {filepath.Join(t.TempDir(), "nodeno"), Options{EsbuildBinary: esbuild}, "--version"}, + } { + err := placeEsbuild(c.deno, t.TempDir(), c.opts) + if err == nil || !strings.Contains(err.Error(), c.want) { + t.Errorf("%s: err = %v, want one containing %q", name, err, c.want) + } + } +} + +func TestRunBundlesNpmPackagesWithDenoBundle(t *testing.T) { + workDir(t) + deno, log := fakeDeno(t, "2.9.7", "") + out := "dist/app.js" + opts := Options{ + Deno: deno, EsbuildBinary: fakeEsbuild(t), Out: out, Main: "app.ts", Srcs: []string{"app.ts"}, + Format: "iife", Minify: true, Sourcemap: true, Flags: []string{"--quiet"}, + } + if err := Run(opts); err != nil { + t.Fatal(err) + } + if got := read(t, out); got != "bundled\n" { + t.Errorf("bundle = %q", got) + } + calls := read(t, log) + for _, want := range []string{ + "args: bundle --no-remote --import-map .import_map.json --platform browser -o dist/app.js --format iife --minify --sourcemap --quiet app.ts", + "esbuild: dl/esbuild-0.25.5-1/esbuild-", + } { + if !strings.Contains(calls, want) { + t.Errorf("deno was not run with %q:\n%s", want, calls) + } + } + if _, err := os.Stat(".import_map.json"); err == nil { + t.Error("the import map was left behind") + } + // the per-run DENO_DIR is a fixed name in the working directory (it ends up in the bundle's + // comments, which must not depend on a random path), and is removed afterwards + denoDir := strings.SplitN(strings.SplitN(calls, "DENO_DIR=", 2)[1], "\n", 2)[0] + if filepath.Base(denoDir) != ".please_ts_deno" { + t.Errorf("DENO_DIR = %s", denoDir) + } + if _, err := os.Stat(denoDir); err == nil { + t.Errorf("DENO_DIR %s was not cleaned up", denoDir) + } +} + +func TestRunBundlesWithoutNpmPackagesThroughEsbuildAsBefore(t *testing.T) { + dir := t.TempDir() + old, _ := os.Getwd() + _ = os.Chdir(dir) + t.Cleanup(func() { _ = os.Chdir(old) }) + + tool := filepath.Join(t.TempDir(), "esbuild-tool") + _ = os.WriteFile(tool, []byte("#!/bin/sh\necho \"$@\" > args.txt\n"), 0755) + if err := Run(Options{BundlerTool: tool, Out: "out.js", Main: "app.ts", Srcs: []string{"app.ts"}, Format: "esm"}); err != nil { + t.Fatal(err) + } + if got := read(t, "args.txt"); !strings.Contains(got, "app.ts --outfile=out.js --bundle --format=esm") { + t.Errorf("esbuild args = %q", got) + } +} + +func TestRunNeedsThePinnedEsbuildForNpmPackages(t *testing.T) { + workDir(t) + deno, _ := fakeDeno(t, "2.9.7", "") + err := Run(Options{Deno: deno, Out: "out.js", Main: "app.ts", Srcs: []string{"app.ts"}}) + if err == nil || !strings.Contains(err.Error(), "needs the pinned esbuild") { + t.Errorf("err = %v", err) + } +} + +func TestRunFailsWhenDenoFetchedItsOwnEsbuild(t *testing.T) { + workDir(t) + // what Deno leaves behind after downloading @esbuild/ itself + deno, _ := fakeDeno(t, "2.9.7", `mkdir -p "$DENO_DIR/npm/registry.npmjs.org/@esbuild/linux-x64"`) + err := Run(Options{Deno: deno, EsbuildBinary: fakeEsbuild(t), Out: "out.js", Main: "app.ts", Srcs: []string{"app.ts"}}) + if err == nil || !strings.Contains(err.Error(), "downloaded esbuild instead of using the pinned binary") { + t.Errorf("err = %v", err) + } +} + +func TestRunReportsAFailingDenoBundle(t *testing.T) { + workDir(t) + deno, _ := fakeDeno(t, "2.9.7", "exit 3") + err := Run(Options{Deno: deno, EsbuildBinary: fakeEsbuild(t), Out: "out.js", Main: "app.ts", Srcs: []string{"app.ts"}}) + if err == nil || !strings.Contains(err.Error(), "deno bundle failed") { + t.Errorf("err = %v", err) + } +} diff --git a/tools/please_ts/commands.go b/tools/please_ts/commands.go index 97845bd..2a8933b 100644 --- a/tools/please_ts/commands.go +++ b/tools/please_ts/commands.go @@ -66,6 +66,8 @@ func handleBundle(args []string) error { depsFlag := cmd.String("deps", "", "Comma-separated dependency paths") deno := cmd.String("deno", "deno", "Path to deno binary") bundlerTool := cmd.String("bundler-tool", "", "External bundler executable (e.g. esbuild)") + esbuildBinary := cmd.String("esbuild-binary", "", "Pinned esbuild executable, used to bundle npm packages") + esbuildCacheVersion := cmd.String("esbuild-cache-version", "", "Directory esbuild- under DENO_DIR/dl where this Deno looks for esbuild (default: known per Deno version)") moduleName := cmd.String("module-name", "", "Module name alias") format := cmd.String("format", "esm", "Bundle format (esm or iife)") minify := cmd.Bool("minify", false, "Minify bundled output") @@ -88,15 +90,19 @@ func handleBundle(args []string) error { opts := bundle.Options{ Deno: *deno, BundlerTool: *bundlerTool, - Out: *out, - Main: *main, - Srcs: srcs, - Deps: deps, - ModuleName: *moduleName, - Format: *format, - Minify: *minify, - Sourcemap: *sourcemap, - Flags: extraFlags, + + EsbuildBinary: *esbuildBinary, + EsbuildCacheVersion: *esbuildCacheVersion, + + Out: *out, + Main: *main, + Srcs: srcs, + Deps: deps, + ModuleName: *moduleName, + Format: *format, + Minify: *minify, + Sourcemap: *sourcemap, + Flags: extraFlags, } return bundle.Run(opts) } @@ -158,6 +164,7 @@ func handleTestRunner(args []string) error { browser := cmd.String("browser", "", "Browser engine for browser tests (chromium, firefox, webkit)") browserBinary := cmd.String("browser-binary", "", "Path to hermetic browser executable") vitestDir := cmd.String("vitest-dir", "", "Path to hermetic Vitest cache directory") + esbuildBinary := cmd.String("esbuild-binary", "", "Pinned esbuild executable, used to bundle npm packages for browser tests") if err := cmd.Parse(args); err != nil { return err @@ -168,6 +175,7 @@ func handleTestRunner(args []string) error { deps := compile.ExpandCommaSeparated([]string{*depsFlag}) opts := testrunner.RunOptions{ + EsbuildBinary: *esbuildBinary, Deno: *deno, Runner: *runner, Srcs: srcs, diff --git a/tools/please_ts/compile/compile.go b/tools/please_ts/compile/compile.go index 0f69456..1fd5d18 100644 --- a/tools/please_ts/compile/compile.go +++ b/tools/please_ts/compile/compile.go @@ -42,21 +42,27 @@ func Run(opts Options) error { defer os.RemoveAll(tmpDir) denoCacheDir := filepath.Join(tmpDir, ".deno_cache") - if opts.VitestDir != "" { + // npm packages provided by ts_npm_module targets are merged into the per-run Deno cache and + // resolved from there, offline. + usesNpmSlices := len(npmcache.Discover(".")) > 0 + switch { + case opts.VitestDir != "" && !usesNpmSlices: + // The shared Vitest cache is used as it is, and never written to by this target. if absV, err := filepath.Abs(opts.VitestDir); err == nil { denoCacheDir = absV } else { denoCacheDir = opts.VitestDir } - } else { + default: if err := os.MkdirAll(denoCacheDir, 0755); err != nil { return fmt.Errorf("failed to create DENO_DIR: %w", err) } - } - - // npm packages provided by ts_npm_module targets are merged into the per-run Deno cache - // and resolved from there, offline. The shared Vitest cache is left as it is. - if opts.VitestDir == "" { + if opts.VitestDir != "" { + // A per-run copy of the Vitest cache, so the slices do not end up in the shared one. + if err := npmcache.Merge([]string{opts.VitestDir}, denoCacheDir); err != nil { + return fmt.Errorf("failed copying the Vitest cache: %w", err) + } + } if _, err := npmcache.Prepare(".", denoCacheDir); err != nil { return fmt.Errorf("failed preparing the npm cache: %w", err) } @@ -69,7 +75,11 @@ func Run(opts Options) error { return fmt.Errorf("failed synthesizing import map: %w", err) } if opts.VitestDir != "" && im.Imports["vitest"] == "" { - im.Imports["vitest"] = "npm:vitest" + spec, err := vitestSpecifier(opts.VitestDir) + if err != nil { + return err + } + im.Imports["vitest"] = spec } if err := im.WriteToFile(importMapPath); err != nil { return fmt.Errorf("failed writing import map: %w", err) @@ -83,9 +93,9 @@ func Run(opts Options) error { "--import-map", importMapPath, } // npm packages come from the merged cache only: a ts_npm_module missing from the - // target's deps must fail here, not be downloaded. (The shared Vitest cache is a + // target's deps must fail here, not be downloaded. (The shared Vitest cache alone is a // different mechanism and keeps its own behaviour.) - if opts.VitestDir == "" && im.HasNpmSpecifiers() { + if usesNpmSlices || (opts.VitestDir == "" && im.HasNpmSpecifiers()) { args = append(args, "--cached-only") } args = append(args, opts.Flags...) @@ -110,6 +120,20 @@ func Run(opts Options) error { return nil } +// vitestSpecifier is the npm: specifier of the vitest in the Vitest cache at dir: pinned to the +// cached version, because an unversioned one resolves to the registry's latest release, which +// Deno would then download into that (read-only) cache. +func vitestSpecifier(dir string) (string, error) { + version, err := npmcache.CachedVersion(dir, "vitest") + if err != nil { + return "", err + } + if version == "" { + return "npm:vitest", nil + } + return "npm:vitest@" + version, nil +} + func emitOutput(opts Options, im *importmap.ImportMap) error { outDir := opts.Out if err := os.MkdirAll(outDir, 0755); err != nil { diff --git a/tools/please_ts/compile/compile_test.go b/tools/please_ts/compile/compile_test.go index e59211b..7984725 100644 --- a/tools/please_ts/compile/compile_test.go +++ b/tools/please_ts/compile/compile_test.go @@ -3,7 +3,10 @@ package compile import ( "os" "path/filepath" + "strings" "testing" + + "tools/please_ts/npmcache" ) func TestEmitOutput(t *testing.T) { @@ -51,3 +54,96 @@ func TestExpandCommaSeparated(t *testing.T) { } } } + +// fakeDeno writes a `deno` that records its arguments, its DENO_DIR and the directories +// of the npm cache it sees there. +func fakeDeno(t *testing.T) (bin, log string) { + t.Helper() + dir := t.TempDir() + log = filepath.Join(dir, "calls.log") + bin = filepath.Join(dir, "deno") + script := `#!/bin/sh +{ + echo "args: $*" + echo "DENO_DIR=$DENO_DIR" + (cd "$DENO_DIR" && find npm -mindepth 3 -maxdepth 3 -type d 2>/dev/null | sort | sed 's/^/pkg: /') +} >> "` + log + `" +` + if err := os.WriteFile(bin, []byte(script), 0755); err != nil { + t.Fatal(err) + } + return bin, log +} + +func writePackage(t *testing.T, root, name, version string) { + t.Helper() + dir := filepath.Join(root, "npm", "registry.npmjs.org", name, version) + if err := os.MkdirAll(dir, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "package.json"), []byte("{}"), 0644); err != nil { + t.Fatal(err) + } +} + +// inWorkDir changes to a fresh directory holding the sources and, optionally, an npm slice. +func inWorkDir(t *testing.T, withSlice bool) { + t.Helper() + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "a.ts"), []byte("export const a = 1;\n"), 0644); err != nil { + t.Fatal(err) + } + if withSlice { + slice := filepath.Join(dir, "third_party", "debug") + writePackage(t, slice, "debug", "4.3.7") + if err := npmcache.Write(slice, npmcache.Slice{Name: "debug", Version: "4.3.7", Specifier: "npm:debug@4.3.7"}); err != nil { + t.Fatal(err) + } + } + old, _ := os.Getwd() + if err := os.Chdir(dir); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chdir(old) }) +} + +func TestRunVitestTargetWithNpmSlicesUsesAPerRunCacheAndNeverTheNetwork(t *testing.T) { + inWorkDir(t, true) + shared := t.TempDir() + writePackage(t, shared, "vitest", "5.0.3") + deno, log := fakeDeno(t) + + if err := Run(Options{Deno: deno, Srcs: []string{"a.ts"}, VitestDir: shared}); err != nil { + t.Fatal(err) + } + data, _ := os.ReadFile(log) + calls := string(data) + if !strings.Contains(calls, "--cached-only") { + t.Errorf("deno check can reach the network:\n%s", calls) + } + if strings.Contains(calls, "DENO_DIR="+shared+"\n") { + t.Errorf("deno wrote into the shared Vitest cache:\n%s", calls) + } + for _, want := range []string{"pkg: npm/registry.npmjs.org/vitest/5.0.3", "pkg: npm/registry.npmjs.org/debug/4.3.7"} { + if !strings.Contains(calls, want) { + t.Errorf("the per-run cache lacks %q:\n%s", want, calls) + } + } + // the shared cache did not get the slice + if _, err := os.Stat(filepath.Join(shared, "npm", "registry.npmjs.org", "debug")); err == nil { + t.Error("the npm slice was merged into the shared Vitest cache") + } +} + +func TestRunVitestTargetWithoutNpmSlicesStillUsesTheSharedCache(t *testing.T) { + inWorkDir(t, false) + shared := t.TempDir() + deno, log := fakeDeno(t) + if err := Run(Options{Deno: deno, Srcs: []string{"a.ts"}, VitestDir: shared}); err != nil { + t.Fatal(err) + } + data, _ := os.ReadFile(log) + if !strings.Contains(string(data), "DENO_DIR="+shared+"\n") || strings.Contains(string(data), "--cached-only") { + t.Errorf("unexpected invocation:\n%s", data) + } +} diff --git a/tools/please_ts/npmcache/npmcache.go b/tools/please_ts/npmcache/npmcache.go index 909215a..df0ad6b 100644 --- a/tools/please_ts/npmcache/npmcache.go +++ b/tools/please_ts/npmcache/npmcache.go @@ -112,6 +112,35 @@ func Merge(slices []string, denoDir string) error { return nil } +// CachedVersion returns the version of a package that the Deno cache at denoDir holds, or "" +// when it holds none of it. An unversioned npm: specifier resolves to the registry's latest +// release, which is not necessarily the cached one (Deno then downloads it, into a cache that +// is meant to be read-only), so callers pin the cached version. More than one cached version is +// an error: there is no telling which one is meant. +func CachedVersion(denoDir, name string) (string, error) { + entries, err := os.ReadDir(filepath.Join(denoDir, registryDir, filepath.FromSlash(name))) + if err != nil { + if os.IsNotExist(err) { + return "", nil + } + return "", err + } + var versions []string + for _, e := range entries { + if e.IsDir() { + versions = append(versions, e.Name()) + } + } + switch len(versions) { + case 0: + return "", nil + case 1: + return versions[0], nil + } + sort.Strings(versions) + return "", fmt.Errorf("%s holds several versions of %s (%s); cannot tell which one to use", denoDir, name, strings.Join(versions, ", ")) +} + // Prepare merges the slices found below root into denoDir and reports whether there were // any. Callers add --cached-only to the Deno command when it returns true, so a package // missing from the cache fails at once instead of reaching for the network. diff --git a/tools/please_ts/npmcache/npmcache_test.go b/tools/please_ts/npmcache/npmcache_test.go index 6a07336..1f80e8a 100644 --- a/tools/please_ts/npmcache/npmcache_test.go +++ b/tools/please_ts/npmcache/npmcache_test.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "reflect" + "strings" "testing" ) @@ -33,6 +34,34 @@ func makeSlice(t *testing.T, root, name, version string, deps map[string]string) return dir } +func TestCachedVersion(t *testing.T) { + root := t.TempDir() + mk := func(name, version string) { + if err := os.MkdirAll(filepath.Join(root, "npm", "registry.npmjs.org", filepath.FromSlash(name), version), 0755); err != nil { + t.Fatal(err) + } + } + mk("vitest", "5.0.1") + mk("@vitest/coverage-v8", "5.0.1") + mk("debug", "4.3.7") + mk("debug", "2.6.9") + // a packument next to the version directories is not a version + _ = os.WriteFile(filepath.Join(root, "npm", "registry.npmjs.org", "vitest", "registry.json"), []byte("{}"), 0644) + + if v, err := CachedVersion(root, "vitest"); err != nil || v != "5.0.1" { + t.Errorf("vitest = %q, %v", v, err) + } + if v, err := CachedVersion(root, "@vitest/coverage-v8"); err != nil || v != "5.0.1" { + t.Errorf("scoped = %q, %v", v, err) + } + if v, err := CachedVersion(root, "absent"); err != nil || v != "" { + t.Errorf("absent = %q, %v", v, err) + } + if _, err := CachedVersion(root, "debug"); err == nil || !strings.Contains(err.Error(), "2.6.9, 4.3.7") { + t.Errorf("several versions: err = %v", err) + } +} + func TestWriteReadRoundTrip(t *testing.T) { dir := t.TempDir() want := Slice{Name: "@scope/pkg", Version: "1.2.3", Specifier: "npm:@scope/pkg@1.2.3", Dependencies: map[string]string{"ms": "^2.1.3"}} diff --git a/tools/please_ts/testrunner/browser.go b/tools/please_ts/testrunner/browser.go index bdbf5fa..ac89841 100644 --- a/tools/please_ts/testrunner/browser.go +++ b/tools/please_ts/testrunner/browser.go @@ -45,6 +45,8 @@ func (opts RunOptions) runBrowserTest(resultsFile string) error { Deps: opts.Deps, ModuleName: opts.ModuleName, Format: "iife", + + EsbuildBinary: opts.EsbuildBinary, } if err := bundle.Run(bundleOpts); err != nil { _ = writeFallbackJUnit(resultsFile, opts.Srcs, err) diff --git a/tools/please_ts/testrunner/runner.go b/tools/please_ts/testrunner/runner.go index bdbc306..3d68381 100644 --- a/tools/please_ts/testrunner/runner.go +++ b/tools/please_ts/testrunner/runner.go @@ -20,6 +20,7 @@ type RunOptions struct { Browser string BrowserBinary string VitestDir string + EsbuildBinary string // pinned esbuild, to bundle npm packages for browser tests ExtraArgs []string } diff --git a/tools/please_ts/testrunner/vitest.go b/tools/please_ts/testrunner/vitest.go index 65bd942..dabec04 100644 --- a/tools/please_ts/testrunner/vitest.go +++ b/tools/please_ts/testrunner/vitest.go @@ -11,6 +11,7 @@ import ( "strings" "tools/please_ts/importmap" + "tools/please_ts/npmcache" ) func (opts RunOptions) runVitest(resultsFile string) error { @@ -28,6 +29,7 @@ func (opts RunOptions) runVitest(resultsFile string) error { } aliasMap := buildVitestAliases(im) + npmDeps := buildVitestNpmDependencies(im) resolvedSrcs := resolveSources(opts.Srcs) configPath := filepath.Join(tmpDir, "vitest.config.mjs") @@ -59,9 +61,9 @@ func (opts RunOptions) runVitest(resultsFile string) error { // test working directory (the default location of the coverage file). coverageDir = filepath.Join(tmpDir, "coverage") args = append(args, vitestCoverageArgs(coverageDir)...) - if err := writeCoveragePackageJSON(); err != nil { - return fmt.Errorf("failed preparing vitest coverage provider: %w", err) - } + } + if err := writeVitestPackageJSON(opts.VitestDir, coverageDir != "", npmDeps); err != nil { + return fmt.Errorf("failed preparing the vitest dependencies: %w", err) } args = append(args, opts.ExtraArgs...) @@ -75,22 +77,42 @@ func (opts RunOptions) runVitest(resultsFile string) error { if opts.VitestDir != "" { denoArgs = append(denoArgs, "--no-remote") } - if coverageDir != "" { + // npm packages come from the merged cache only: a ts_npm_module missing from the target's + // deps must fail, not be downloaded. + if len(npmDeps) > 0 { + denoArgs = append(denoArgs, "--cached-only") + } + if coverageDir != "" || len(npmDeps) > 0 { // The v8 coverage provider is a peer dependency that vitest resolves from the - // project's node_modules; deno only materializes it when asked to. + // project's node_modules, and so are the npm packages of the target (see + // buildVitestNpmDependencies); deno only materializes them when asked to. denoArgs = append(denoArgs, "--node-modules-dir=auto") } - denoArgs = append(denoArgs, "-A", "npm:vitest") + vitestSpec := "npm:vitest" + if opts.VitestDir != "" { + // Pinned to the cached version: an unversioned specifier resolves to the registry's + // latest release, which Deno would download into the shared (read-only) cache. + version, err := npmcache.CachedVersion(opts.VitestDir, "vitest") + if err != nil { + _ = writeFallbackJUnit(resultsFile, opts.Srcs, err) + return err + } + if version != "" { + vitestSpec += "@" + version + } + } + denoArgs = append(denoArgs, "-A", vitestSpec) denoArgs = append(denoArgs, args...) cmd := exec.Command(denoBin, denoArgs...) env := os.Environ() if opts.VitestDir != "" { - if absV, err := filepath.Abs(opts.VitestDir); err == nil { - env = append(env, "DENO_DIR="+absV) - } else { - env = append(env, "DENO_DIR="+opts.VitestDir) + denoDir, err := opts.vitestDenoDir(tmpDir, len(npmDeps) > 0) + if err != nil { + _ = writeFallbackJUnit(resultsFile, opts.Srcs, err) + return err } + env = append(env, "DENO_DIR="+denoDir) } if opts.BrowserBinary != "" { env = append(env, "PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH="+opts.BrowserBinary) @@ -111,18 +133,68 @@ func (opts RunOptions) runVitest(resultsFile string) error { return testErr } +// vitestDenoDir returns the DENO_DIR Vitest runs with. It is the shared Vitest cache itself, +// unless the target uses npm packages: then it is a per-run copy of that cache holding the cache +// slices of the target's ts_npm_module dependencies as well, so the shared cache is never written. +func (opts RunOptions) vitestDenoDir(tmpDir string, withNpm bool) (string, error) { + shared, err := filepath.Abs(opts.VitestDir) + if err != nil { + shared = opts.VitestDir + } + if !withNpm { + return shared, nil + } + denoDir := filepath.Join(tmpDir, ".deno_cache") + if err := os.MkdirAll(denoDir, 0755); err != nil { + return "", fmt.Errorf("failed creating DENO_DIR: %w", err) + } + if err := npmcache.Merge([]string{shared}, denoDir); err != nil { + return "", fmt.Errorf("failed copying the Vitest cache: %w", err) + } + if _, err := npmcache.Prepare(".", denoDir); err != nil { + return "", fmt.Errorf("failed preparing the npm cache: %w", err) + } + return denoDir, nil +} + // vitestCoverageArgs returns the vitest flags that collect lcov coverage into dir. func vitestCoverageArgs(dir string) []string { return []string{"--coverage.enabled", "--coverage.provider=v8", "--coverage.reporter=lcov", "--coverage.reportsDirectory=" + dir} } -// writeCoveragePackageJSON declares the vitest coverage provider in the working -// directory unless the target already ships a package.json. -func writeCoveragePackageJSON() error { +// writeVitestPackageJSON declares in the working directory what deno must materialize in +// node_modules: the vitest coverage provider when coverage is on, and the npm packages of the +// target, at the versions that are cached. A package.json the target already ships is left as +// it is. +func writeVitestPackageJSON(vitestDir string, coverage bool, npmDeps map[string]string) error { + deps := map[string]string{} + if coverage { + // the versions in the Vitest cache: "*" would mean the registry's latest release + for _, name := range []string{"vitest", "@vitest/coverage-v8"} { + version, err := npmcache.CachedVersion(vitestDir, name) + if err != nil { + return err + } + if version == "" { + version = "*" + } + deps[name] = version + } + } + for name, version := range npmDeps { + deps[name] = version + } + if len(deps) == 0 { + return nil + } if _, err := os.Stat("package.json"); err == nil { return nil } - return os.WriteFile("package.json", []byte(`{"dependencies":{"vitest":"*","@vitest/coverage-v8":"*"}}`+"\n"), 0644) + data, err := json.Marshal(map[string]map[string]string{"dependencies": deps}) + if err != nil { + return err + } + return os.WriteFile("package.json", append(data, '\n'), 0644) } // convertVitestCoverage turns the lcov report vitest wrote into dir into the @@ -151,6 +223,10 @@ func buildVitestAliases(im *importmap.ImportMap) map[string]string { if strings.HasSuffix(k, "/") { continue } + // npm packages are not files: Deno imports them (see buildVitestNpmImports). + if strings.HasPrefix(target, "npm:") { + continue + } if k == "vitest" || strings.HasPrefix(k, "vitest/") || k == "chai" || strings.HasPrefix(k, "chai/") { continue } @@ -166,6 +242,32 @@ func buildVitestAliases(im *importmap.ImportMap) map[string]string { return aliasMap } +// buildVitestNpmDependencies lists the npm packages (ts_npm_module) of the import map as +// package.json dependencies, name to exact version. Vite cannot resolve an npm: specifier, and +// Vitest cannot import one (Deno refuses it in the loader Vitest runs under). With these in a +// package.json, `deno run --node-modules-dir=auto` materializes a node_modules directory in the +// test's working directory from the merged cache, offline, and Vite resolves and externalizes +// the packages as it does any others: exports, CommonJS and subpaths included. +func buildVitestNpmDependencies(im *importmap.ImportMap) map[string]string { + deps := map[string]string{} + if im == nil { + return deps + } + for k, target := range im.Imports { + spec, ok := strings.CutPrefix(target, "npm:") + if !ok || strings.HasSuffix(k, "/") { + continue + } + // npm:name@version or npm:@scope/name@version + at := strings.LastIndex(spec, "@") + if at <= 0 { + continue + } + deps[spec[:at]] = spec[at+1:] + } + return deps +} + // aliasEntry is one Vite resolve.alias entry in the generated config. type aliasEntry struct { Find string `json:"find"` diff --git a/tools/please_ts/testrunner/vitest_test.go b/tools/please_ts/testrunner/vitest_test.go index 51f5038..55b3544 100644 --- a/tools/please_ts/testrunner/vitest_test.go +++ b/tools/please_ts/testrunner/vitest_test.go @@ -1,12 +1,14 @@ package testrunner import ( + "encoding/json" "os" "path/filepath" "strings" "testing" "tools/please_ts/importmap" + "tools/please_ts/npmcache" ) func TestBuildVitestAliasesMatrix(t *testing.T) { @@ -187,3 +189,163 @@ func TestConvertVitestCoverageExportsRawLcov(t *testing.T) { } } } + +func TestBuildVitestNpmDependencies(t *testing.T) { + im := importmap.New() + im.Imports = map[string]string{ + "debug": "npm:debug@4.3.7", + "debug/": "npm:/debug@4.3.7/", + "@codemirror/legacy-modes": "npm:@codemirror/legacy-modes@6.5.1", + "@codemirror/legacy-modes/": "npm:/@codemirror/legacy-modes@6.5.1/", + "@test/lib": "/abs/lib.ts", + } + got := buildVitestNpmDependencies(im) + want := map[string]string{"debug": "4.3.7", "@codemirror/legacy-modes": "6.5.1"} + if len(got) != len(want) || got["debug"] != want["debug"] || got["@codemirror/legacy-modes"] != want["@codemirror/legacy-modes"] { + t.Errorf("got %v, want %v", got, want) + } + if len(buildVitestNpmDependencies(nil)) != 0 || len(buildVitestNpmDependencies(importmap.New())) != 0 { + t.Error("expected no dependencies") + } +} + +func TestBuildVitestAliasesLeavesNpmPackagesToTheirNodeModules(t *testing.T) { + im := importmap.New() + im.Imports = map[string]string{"debug": "npm:debug@4.3.7", "@test/lib": "lib/lib.ts"} + aliases := buildVitestAliases(im) + if _, ok := aliases["debug"]; ok { + t.Errorf("an npm: target became a file alias: %v", aliases) + } + if _, ok := aliases["@test/lib"]; !ok { + t.Errorf("first-party alias missing: %v", aliases) + } +} + +func inTempDir(t *testing.T) { + t.Helper() + old, _ := os.Getwd() + if err := os.Chdir(t.TempDir()); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chdir(old) }) +} + +func cachedPackage(t *testing.T, denoDir, name, version string) { + t.Helper() + if err := os.MkdirAll(filepath.Join(denoDir, "npm", "registry.npmjs.org", filepath.FromSlash(name), version), 0755); err != nil { + t.Fatal(err) + } +} + +func TestWriteVitestPackageJSON(t *testing.T) { + cache := t.TempDir() + cachedPackage(t, cache, "vitest", "5.0.1") + cachedPackage(t, cache, "@vitest/coverage-v8", "5.0.1") + + read := func() map[string]map[string]string { + data, err := os.ReadFile("package.json") + if err != nil { + t.Fatal(err) + } + var p map[string]map[string]string + if err := json.Unmarshal(data, &p); err != nil { + t.Fatal(err) + } + return p + } + + t.Run("nothing to declare writes nothing", func(t *testing.T) { + inTempDir(t) + if err := writeVitestPackageJSON(cache, false, nil); err != nil { + t.Fatal(err) + } + if _, err := os.Stat("package.json"); err == nil { + t.Error("package.json was written") + } + }) + + t.Run("coverage declares the cached versions, not latest", func(t *testing.T) { + inTempDir(t) + if err := writeVitestPackageJSON(cache, true, nil); err != nil { + t.Fatal(err) + } + deps := read()["dependencies"] + if deps["vitest"] != "5.0.1" || deps["@vitest/coverage-v8"] != "5.0.1" || len(deps) != 2 { + t.Errorf("deps = %v", deps) + } + }) + + t.Run("npm packages are declared with their exact version", func(t *testing.T) { + inTempDir(t) + if err := writeVitestPackageJSON(cache, true, map[string]string{"debug": "4.3.7"}); err != nil { + t.Fatal(err) + } + deps := read()["dependencies"] + if deps["debug"] != "4.3.7" || deps["vitest"] != "5.0.1" || len(deps) != 3 { + t.Errorf("deps = %v", deps) + } + }) + + t.Run("a package.json the target ships is kept", func(t *testing.T) { + inTempDir(t) + _ = os.WriteFile("package.json", []byte(`{"name":"mine"}`), 0644) + if err := writeVitestPackageJSON(cache, true, map[string]string{"debug": "4.3.7"}); err != nil { + t.Fatal(err) + } + if data, _ := os.ReadFile("package.json"); string(data) != `{"name":"mine"}` { + t.Errorf("package.json = %s", data) + } + }) + + t.Run("several cached versions are an error", func(t *testing.T) { + inTempDir(t) + two := t.TempDir() + cachedPackage(t, two, "vitest", "5.0.1") + cachedPackage(t, two, "vitest", "5.0.3") + if err := writeVitestPackageJSON(two, true, nil); err == nil || !strings.Contains(err.Error(), "several versions") { + t.Errorf("err = %v", err) + } + }) +} + +func TestVitestDenoDir(t *testing.T) { + shared := t.TempDir() + cachedPackage(t, shared, "vitest", "5.0.1") + opts := RunOptions{VitestDir: shared} + + t.Run("without npm packages it is the shared cache", func(t *testing.T) { + got, err := opts.vitestDenoDir(t.TempDir(), false) + if err != nil || got != shared { + t.Errorf("got %q, %v; want %q", got, err, shared) + } + }) + + t.Run("with npm packages it is a per-run copy that also holds the slices", func(t *testing.T) { + dir := t.TempDir() + old, _ := os.Getwd() + _ = os.Chdir(dir) + t.Cleanup(func() { _ = os.Chdir(old) }) + slice := filepath.Join(dir, "third_party", "debug") + cachedPackage(t, slice, "debug", "4.3.7") + if err := npmcache.Write(slice, npmcache.Slice{Name: "debug", Version: "4.3.7", Specifier: "npm:debug@4.3.7"}); err != nil { + t.Fatal(err) + } + + tmp := t.TempDir() + got, err := opts.vitestDenoDir(tmp, true) + if err != nil { + t.Fatal(err) + } + if got == shared { + t.Fatal("the shared cache would be written to") + } + for _, pkg := range []string{"vitest/5.0.1", "debug/4.3.7"} { + if _, err := os.Stat(filepath.Join(got, "npm", "registry.npmjs.org", filepath.FromSlash(pkg))); err != nil { + t.Errorf("the per-run cache lacks %s: %v", pkg, err) + } + } + if _, err := os.Stat(filepath.Join(shared, "npm", "registry.npmjs.org", "debug")); err == nil { + t.Error("the slice was merged into the shared cache") + } + }) +}