feat(ts): npm packages in ts_bundle, ts_binary, Vitest and browser tests

.plzconfig

@@ -14,6 +14,7 @@ PleaseTsTool = //tools/please_ts:please_ts
 DenoTool = //tools/ts_toolchain:toolchain|deno
 BrowserTool = //tools/browser_toolchain:toolchain|chromium
 VitestTool = //tools/vitest_toolchain:toolchain
+EsbuildTool = //tools/esbuild_toolchain:toolchain|esbuild
 
 [cover]
 fileextension = .ts
@@ -45,6 +46,12 @@ DefaultValue = //tools/vitest_toolchain:toolchain
 Help = Path to the hermetic Vitest cache or toolchain target to use
 Inherit = true
 
+[PluginConfig "esbuild_tool"]
+ConfigKey = EsbuildTool
+DefaultValue = //tools/esbuild_toolchain:toolchain|esbuild
+Help = Path to the pinned esbuild binary or toolchain target that bundles npm packages
+Inherit = true
+
 [PluginConfig "coverage"]
 ConfigKey = Coverage
 DefaultValue = true

CHANGELOG.md

@@ -28,12 +28,25 @@ and this project adheres to
   fixes the result and nothing needs configuring.
 - `please_ts unpack --npm-cache`, and the `npmcache`, `registry` and `semver`
   packages behind it.
+- npm packages (`ts_npm_module`) work in `ts_bundle`, `ts_binary`, the Vitest
+  runner and `ts_browser_test`, not only in type checks and Deno tests. Bundles
+  are built by `deno bundle` with the pinned esbuild of the new
+  `esbuild_toolchain` (`EsbuildTool`; Deno would otherwise download its own, and
+  the build fails if it did). `ts_binary` embeds the packages and runs without a
+  cache or network. Vitest gets them as a `node_modules` directory that Deno
+  materializes offline from the merged cache in the test's working directory
+  (under `plz-out/tmp` only).
 - Fixtures for `debug` (CommonJS, transitive dependency), `highlight.js`
   (CommonJS, subpaths) and `@codemirror/legacy-modes` (exports map only),
   automatic and fully pinned.
 
 ### Fixed
 
+- Targets that use the Vitest runner no longer write into the shared
+  `vitest_toolchain` output. Vitest and the type check of a `ts_test` ran the
+  unversioned `npm:vitest`, which resolves to the registry's latest release
+  rather than the cached one and was downloaded into that cache at run time;
+  they now use the cached version, and the `package.json` for coverage does too.
 - `ts_module`'s automatic dependency resolution (`resolve_transitive`) is now
   strict and verified. It used to fall back to the `latest` version when no
   version satisfied a range, verify no download against the registry's integrity

build_defs/ts/constants.build_defs

@@ -24,6 +24,31 @@ DENO_PLATFORM_MAP = {
     },
 }
 
+# The esbuild that `deno bundle` runs, as the npm package that carries its binary. Deno looks for
+# it under DENO_DIR/dl/esbuild-<version> and downloads its own otherwise: the version here must be
+# the one the default Deno version (above) uses, and tools/please_ts/bundle/denobundle.go maps
+# the Deno version to the directory name.
+DEFAULT_ESBUILD_VERSION = "0.25.5"
+
+ESBUILD_PLATFORM_MAP = {
+    "linux_amd64": {
+        "url": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.5.tgz",
+        "hash": "95a928d8187c6f7ad632a3f3bbf01f66dfd5b5adb724b8bfeec5fff73ff2d91a",
+    },
+    "linux_arm64": {
+        "url": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.5.tgz",
+        "hash": "ae5a580130af1a7c7b8760466c9ae85a25782e63330b8c2855cc15ed007dd30d",
+    },
+    "darwin_arm64": {
+        "url": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.5.tgz",
+        "hash": "61a312bcb8249d058639c405cf6378dd3107de5535a9974973c48a6dd0d2d062",
+    },
+    "darwin_amd64": {
+        "url": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.5.tgz",
+        "hash": "d2890be89bd7e322cd83e1665516d3c71b0ac3952655cb27076eddb621a4af24",
+    },
+}
+
 DEFAULT_CHROMIUM_VERSION = "154.0.8037.57"
 
 CHROMIUM_PLATFORM_MAP = {

build_defs/ts/ts.build_defs

@@ -24,6 +24,15 @@ def _browser_tool_arg(tools, browser_binary = ""):
         return f"--browser-binary '{tool}'"
     return ""
 
+def _esbuild_tool_arg(tools):
+    tool = CONFIG.TS.ESBUILD_TOOL
+    if tool and (tool.startswith("//") or tool.startswith("///") or tool.startswith(":")):
+        tools["ESBUILD"] = [tool]
+        return '--esbuild-binary "$TOOLS_ESBUILD"'
+    elif tool:
+        return f"--esbuild-binary '{tool}'"
+    return ""
+
 def _vitest_tool_arg(tools, vitest_dir = ""):
     tool = vitest_dir or CONFIG.TS.VITEST_TOOL
     if not tool:
@@ -145,6 +154,47 @@ def browser_toolchain(
         building_description = f"Assembling hermetic headless Chromium toolchain {version}...",
     )
 
+def esbuild_toolchain(
+        name: str = "toolchain",
+        version: str = DEFAULT_ESBUILD_VERSION,
+        esbuild_url: str = "",
+        esbuild_hash: str = "",
+        target_platform: str = "",
+        labels: list = None,
+        visibility: list = None):
+    """Provides the pinned esbuild binary that `deno bundle` runs to bundle npm packages.
+
+    Deno downloads its own esbuild when it does not find one in its cache. Bundling puts this
+    one there, taken from a hash-pinned download, so that nothing is fetched while building.
+    The version must be the one the Deno version in use expects (see DEFAULT_ESBUILD_VERSION).
+    """
+    plat = target_platform or f"{CONFIG.OS}_{CONFIG.ARCH}"
+    esbuild_info = ESBUILD_PLATFORM_MAP.get(plat, ESBUILD_PLATFORM_MAP["linux_amd64"])
+
+    dl_target = f"_{name}_esbuild_dl"
+    remote_file(
+        name = dl_target,
+        url = esbuild_url or esbuild_info["url"],
+        hashes = [esbuild_hash or esbuild_info["hash"]],
+        out = f"{name}.tgz",
+    )
+
+    tools = {"TOOL": [_ts_tool()]}
+    return build_rule(
+        name = name,
+        srcs = [f":{dl_target}"],
+        outs = [name],
+        cmd = '$TOOLS_TOOL unpack --archive "$SRCS" --out "$OUT/bin" --binary "esbuild"',
+        entry_points = {
+            "esbuild": f"{name}/bin/esbuild",
+        },
+        binary = True,
+        tools = tools,
+        visibility = visibility or ["PUBLIC"],
+        labels = labels or ["ts", "toolchain", "esbuild"],
+        building_description = f"Unpacking the pinned esbuild {version}...",
+    )
+
 def vitest_toolchain(
         name: str = "toolchain",
         version: str = DEFAULT_VITEST_VERSION,
@@ -433,6 +483,8 @@ def ts_bundle(
     flags_joined = " ".join(flags) if flags else ""
     flags_str = f"--flags '{flags_joined}'" if flags else ""
 
+    esbuild_arg = _esbuild_tool_arg(tools)
+
     resolved_module = _resolve_module_name(name, module_name)
     bundle_args = [
         '--out "$OUT"',
@@ -444,6 +496,7 @@ def ts_bundle(
         '--minify' if minify else "",
         '--sourcemap' if sourcemap else "",
         f'--bundler-tool "{bundler_tool}"' if bundler_tool else "",
+        esbuild_arg,
         deno_arg,
         flags_str,
     ]
@@ -554,6 +607,7 @@ def ts_browser_test(
     tools = {"TOOL": [_ts_tool()]}
     deno_arg = _deno_tool_arg(tools)
     browser_arg = _browser_tool_arg(tools, browser_binary)
+    esbuild_arg = _esbuild_tool_arg(tools)
 
     flags_joined = " ".join(flags) if flags else ""
     flags_str = f"--flags '{flags_joined}'" if flags else ""
@@ -576,6 +630,7 @@ def ts_browser_test(
         f'--runner "{runner}"',
         f'--browser "{browser}"',
         browser_arg,
+        esbuild_arg,
         f'--srcs "{srcs_list}"',
         '--deps "${DEPS:-}"',
         f'--module-name "{resolved_module}"' if resolved_module else "",

docs/ts/rules.md

@@ -162,6 +162,11 @@ ts_bundle(
 | `sourcemap`    | `bool` | `False`    | Whether to generate source maps.                     |
 | `bundler_tool` | `str`  | `""`       | Optional external bundler executable (e.g. esbuild). |
 
+With npm packages (`ts_npm_module` dependencies) the bundle is built by
+`deno bundle`, using the pinned esbuild of `esbuild_toolchain`, and
+`bundler_tool` is not used. See
+[Bundling and compiling](usage.md#bundling-and-compiling).
+
 ---
 
 ## `ts_test`
@@ -217,6 +222,30 @@ vitest_toolchain(
 
 ---
 
+## `esbuild_toolchain`
+
+Provides the pinned esbuild binary that `deno bundle` runs to bundle npm
+packages (Deno would download its own). The `EsbuildTool` config key points at
+it (`//tools/esbuild_toolchain:toolchain|esbuild` by default).
+
+```starlark
+esbuild_toolchain(
+    name = "toolchain",
+)
+```
+
+### Arguments
+
+| Argument          | Type  | Default       | Description                                                        |
+| :---------------- | :---- | :------------ | :----------------------------------------------------------------- |
+| `name`            | `str` | `"toolchain"` | Target name.                                                       |
+| `version`         | `str` | `"0.25.5"`    | esbuild version; must be the one the Deno version uses.            |
+| `esbuild_url`     | `str` | `""`          | Override the download URL (the `@esbuild/<platform>` npm tarball). |
+| `esbuild_hash`    | `str` | `""`          | sha256 of that download.                                           |
+| `target_platform` | `str` | `""`          | Platform (defaults to the host).                                   |
+
+---
+
 ## `browser_toolchain`
 
 Assembles a 100% hermetic headless browser (`chrome-headless-shell`) binary

docs/ts/usage.md

@@ -197,8 +197,9 @@ The Deno runner (the default) uses `deno test --coverage`; the Vitest runner
   from lcov. Vitest writes its lcov report to a scratch directory (Vitest
   empties its reports directory, so it cannot be the test working directory).
 - **Vitest note**: when coverage is active the runner writes a `package.json`
-  declaring `vitest` and `@vitest/coverage-v8` in the test working directory (if
-  none exists) so Deno can resolve the provider.
+  declaring `vitest` and `@vitest/coverage-v8`, at the versions in the Vitest
+  cache, in the test working directory (if none exists) so Deno can resolve the
+  provider.
 
 ### Raw lcov export (functions and branches)
 
@@ -287,11 +288,47 @@ import createDebug from "debug"; // CommonJS, and it needs the package "ms"
 - **Resolution is Deno's**: `exports` maps, CommonJS, subpaths and a package's
   own types work as they do for `npm:` specifiers. Imports use the plain package
   name.
-- **Not wired yet**: `ts_bundle` and `ts_binary`, the Vitest and browser
-  runners, and a helper that prints pinned declarations. The cache layout
+- **Where it works**: `ts_library` (type checks), `ts_test` with the Deno and
+  the Vitest runner, `ts_browser_test`, `ts_bundle` and `ts_binary`. Not done
+  yet: a helper that prints pinned declarations. The cache layout
   (`registry.json`, including its `_deno.packumentFormat` key) is internal to
   Deno, so it is tied to the Deno version the plugin pins.
 
+### Bundling and compiling
+
+`ts_bundle` (and the bundle `ts_browser_test` builds for the browser) runs
+`deno bundle` when the target depends on npm packages, so Deno resolves them
+exactly as it does for `ts_test`. `deno bundle` runs esbuild, which Deno would
+download itself; the plugin puts the pinned binary of `esbuild_toolchain`
+(`EsbuildTool`, `//tools/esbuild_toolchain:toolchain|esbuild` by default, hash
+pinned per platform) in the cache where Deno looks for it, and fails the build
+if Deno downloaded one anyway. The bundle names its inputs relative to the
+working directory, so the same sources give the same bundle. Targets without npm
+packages bundle through esbuild with import map aliases, as before.
+
+`ts_binary` merges the slices and runs `deno compile --cached-only`; the
+executable embeds the packages and runs without a cache or network.
+(`deno compile` fetches its own `denort` runtime from `dl.deno.land` on every
+build, with or without npm packages: not pinned yet.)
+
+Two limits: `deno bundle` is marked experimental by Deno, and the directory it
+looks in for esbuild is internal to Deno, so the plugin knows it per Deno
+version (`esbuildCacheVersions` in `tools/please_ts/bundle/denobundle.go`, and
+`--esbuild-cache-version` for a version it does not know). The bundles and the
+binary in `test/ts/npm_cache` fail on a Deno bump until that is updated.
+
+### Vitest
+
+Vitest cannot import `npm:` specifiers (nor can Vite resolve them), so for a
+target with npm packages the Vitest runner declares them, at their exact
+versions, in a `package.json` and lets
+`deno run --node-modules-dir=auto --cached-only` materialize them in a
+`node_modules` directory, offline, from the merged cache. Both exist only in the
+test's working directory under `plz-out/tmp`, which Please deletes after the
+run; nothing is written to the source tree. The Vitest cache of
+`vitest_toolchain` is copied for such a target, so the slices never end up in
+that shared output, and Vitest itself is pinned to the version in that cache.
+
 `ts_module` resolves its dependencies by the same rules (strictly, verified, as
 of the root's publish day), with the limit of one version per package: two
 dependents that need incompatible versions fail the build and name each other.

test/ts/npm_cache/BUILD

@@ -131,3 +131,107 @@ ts_test(
         ":modes_auto",
     ],
 )
+
+# Bundling and compiling with npm packages: Deno resolves them from the cache slices (no network,
+# no node_modules), and `deno bundle` runs the pinned esbuild.
+ts_bundle(
+    name = "highlight_bundle",
+    srcs = ["highlight_main.ts"],
+    format = "iife",
+    main = "highlight_main.ts",
+    deps = [
+        ":highlight",
+        "//test/ts/npm_cache/modules:highlight_js",
+    ],
+)
+
+ts_binary(
+    name = "highlight_app",
+    srcs = ["highlight_main.ts"],
+    main = "highlight_main.ts",
+    deps = [
+        ":highlight",
+        "//test/ts/npm_cache/modules:highlight_js",
+    ],
+)
+
+# The exports-only package with its declared closure, and with the closure resolved automatically.
+ts_bundle(
+    name = "modes_bundle",
+    srcs = ["modes.ts"],
+    format = "iife",
+    main = "modes.ts",
+    deps = [
+        "//test/ts/npm_cache/modules:codemirror_language",
+        "//test/ts/npm_cache/modules:codemirror_legacy_modes",
+    ],
+)
+
+ts_bundle(
+    name = "modes_auto_bundle",
+    srcs = ["modes.ts"],
+    format = "iife",
+    main = "modes.ts",
+    deps = [
+        ":codemirror_language_auto",
+        ":legacy_modes_auto",
+    ],
+)
+
+gentest(
+    name = "bundles_test",
+    data = {
+        "highlight": [":highlight_bundle"],
+        "modes": [":modes_bundle"],
+        "modes_auto": [":modes_auto_bundle"],
+        "app": [":highlight_app"],
+    },
+    no_test_output = True,
+    test_cmd = " && ".join([
+        # self-contained: the npm packages are inlined, no specifier is left to resolve at run time
+        '! grep -q "npm:" $DATA_HIGHLIGHT',
+        '! grep -q "highlight.js/lib" $DATA_HIGHLIGHT',
+        # ...and their code really is in there: highlight.js's Go grammar, CodeMirror's StreamLanguage
+        'grep -q "registerLanguage" $DATA_HIGHLIGHT',
+        'grep -q "golang" $DATA_HIGHLIGHT',
+        '! grep -q "npm:" $DATA_MODES',
+        'grep -q "StreamLanguage" $DATA_MODES',
+        'grep -q "StreamLanguage" $DATA_MODES_AUTO',
+        # the compiled program carries highlight.js and runs without any cache or network
+        'out=$(DENO_DIR=/nonexistent $DATA_APP)',
+        'echo "$out" | grep -q "hljs-keyword"',
+    ]),
+)
+
+# The browser runner bundles the test with its npm packages, then runs it in headless Chromium.
+ts_browser_test(
+    name = "highlight_browser_test",
+    srcs = ["highlight_browser_test.ts"],
+    deps = [
+        ":highlight",
+        "//test/ts/npm_cache/modules:highlight_js",
+    ],
+)
+
+# Vitest imports the npm packages natively (Deno resolves them from the merged cache) instead of
+# trying to resolve them itself.
+ts_test(
+    name = "highlight_vitest_test",
+    srcs = ["highlight_vitest_test.ts"],
+    runner = "vitest",
+    deps = [
+        ":highlight",
+        "//test/ts/npm_cache/modules:highlight_js",
+    ],
+)
+
+ts_test(
+    name = "modes_vitest_test",
+    srcs = ["modes_vitest_test.ts"],
+    runner = "vitest",
+    deps = [
+        ":modes",
+        "//test/ts/npm_cache/modules:codemirror_language",
+        "//test/ts/npm_cache/modules:codemirror_legacy_modes",
+    ],
+)