diff --git a/.github/workflows/review-desktop-preview.yml b/.github/workflows/review-desktop-preview.yml index 1f7619a42..39ea1b56e 100644 --- a/.github/workflows/review-desktop-preview.yml +++ b/.github/workflows/review-desktop-preview.yml @@ -105,10 +105,12 @@ jobs: name: Publish npm package needs: version runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 45 permissions: contents: read id-token: write + outputs: + whiteboard_remote: ${{ steps.remote.outputs.whiteboard_remote }} env: VERSION: ${{ needs.version.outputs.version }} COMMIT: ${{ needs.version.outputs.commit }} @@ -140,12 +142,64 @@ jobs: fi echo "dist_tag=$DIST_TAG" >> "$GITHUB_OUTPUT" echo "published=$PUBLISHED" >> "$GITHUB_OUTPUT" + - name: Install Code OSS native build prerequisites + run: | + # The hosted runner image lists an HTTP Azure mirror first. That + # mirror can accept the release file and then hang on package indexes. + echo 'https://archive.ubuntu.com/ubuntu/' | sudo tee /etc/apt/apt-mirrors.txt >/dev/null + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + libx11-dev libxkbfile-dev libkrb5-dev libsecret-1-dev + - name: Setup Code OSS Node + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + with: + node-version-file: apps/review-desktop/code-oss/.nvmrc + cache: npm + cache-dependency-path: apps/review-desktop/code-oss/**/package-lock.json + - name: Compute Code OSS dependency key + id: code-oss-inputs + run: | + DIGEST=$(node apps/review-desktop/scripts/code-oss-bootstrap.mjs digest apps/review-desktop/code-oss) + echo "digest=$DIGEST" >> "$GITHUB_OUTPUT" + - name: Restore Code OSS dependencies + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: | + apps/review-desktop/code-oss/node_modules + apps/review-desktop/code-oss/build/node_modules + apps/review-desktop/code-oss/extensions/node_modules + apps/review-desktop/code-oss/extensions/*/node_modules + apps/review-desktop/code-oss/extensions/*/server/node_modules + apps/review-desktop/code-oss/.build/distro/npm/node_modules + apps/review-desktop/code-oss/.build/distro/npm/remote/node_modules + apps/review-desktop/code-oss/.build/distro/npm/remote/web/node_modules + ~/.cache/node-gyp + key: ${{ runner.os }}-code-oss-deps-v3-${{ steps.code-oss-inputs.outputs.digest }} + restore-keys: | + ${{ runner.os }}-code-oss-deps-v3- + - name: Build the VS Code server + run: | + source apps/review-desktop/scripts/code-oss-dependencies.sh + ensure_code_oss_dependencies "$PWD/apps/review-desktop" "$PWD/apps/review-desktop/code-oss" + BUILD_SOURCEVERSION="$COMMIT" node apps/review-desktop/scripts/build-remote-runtime.mjs - name: Build release tarball run: | jq -n --arg version "$VERSION" --arg commit "$COMMIT" '{$version, $commit}' > release-plan.json node scripts/pack-review-cli.mjs release-artifacts - name: Test installed tarball without Desktop run: node scripts/smoke-review-cli.mjs "release-artifacts/dev.fast-whiteboard-${VERSION}.tgz" "$VERSION" + - name: Pin the remote package and Node + id: remote + env: + PUBLISHED: ${{ steps.npm.outputs.published }} + DRY_RUN: ${{ inputs.dry_run }} + run: | + TARBALL="release-artifacts/dev.fast-whiteboard-${VERSION}.tgz" + if [ "$PUBLISHED" = true ] && [ "$DRY_RUN" != true ]; then + TARBALL="$RUNNER_TEMP/$(cd "$RUNNER_TEMP" && npm pack "@dev.fast/whiteboard@${VERSION}" --silent)" + fi + PIN=$(node apps/review-desktop/scripts/stamp-remote-package.mjs "$TARBALL") + echo "whiteboard_remote=$PIN" >> "$GITHUB_OUTPUT" - name: Validate npm publish command if: ${{ steps.npm.outputs.published != 'true' }} env: @@ -175,6 +229,7 @@ jobs: with: commit: ${{ needs.version.outputs.commit }} version: ${{ needs.version.outputs.version }} + whiteboard_remote: ${{ needs.npm.outputs.whiteboard_remote }} release_signing: ${{ !inputs.dry_run }} secrets: inherit @@ -188,6 +243,7 @@ jobs: with: commit: ${{ needs.version.outputs.commit }} version: ${{ needs.version.outputs.version }} + whiteboard_remote: ${{ needs.npm.outputs.whiteboard_remote }} sign: true macos: @@ -200,6 +256,7 @@ jobs: with: commit: ${{ needs.version.outputs.commit }} version: ${{ needs.version.outputs.version }} + whiteboard_remote: ${{ needs.npm.outputs.whiteboard_remote }} channel: preview secrets: inherit diff --git a/.github/workflows/review-desktop-release.yml b/.github/workflows/review-desktop-release.yml index 3d9d327e9..496f4145d 100644 --- a/.github/workflows/review-desktop-release.yml +++ b/.github/workflows/review-desktop-release.yml @@ -278,10 +278,12 @@ jobs: name: Publish npm package needs: version runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 45 permissions: contents: read id-token: write + outputs: + whiteboard_remote: ${{ steps.remote.outputs.whiteboard_remote }} env: VERSION: ${{ needs.version.outputs.version }} COMMIT: ${{ needs.version.outputs.commit }} @@ -313,12 +315,64 @@ jobs: fi echo "dist_tag=$DIST_TAG" >> "$GITHUB_OUTPUT" echo "published=$PUBLISHED" >> "$GITHUB_OUTPUT" + - name: Install Code OSS native build prerequisites + run: | + # The hosted runner image lists an HTTP Azure mirror first. That + # mirror can accept the release file and then hang on package indexes. + echo 'https://archive.ubuntu.com/ubuntu/' | sudo tee /etc/apt/apt-mirrors.txt >/dev/null + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + libx11-dev libxkbfile-dev libkrb5-dev libsecret-1-dev + - name: Setup Code OSS Node + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + with: + node-version-file: apps/review-desktop/code-oss/.nvmrc + cache: npm + cache-dependency-path: apps/review-desktop/code-oss/**/package-lock.json + - name: Compute Code OSS dependency key + id: code-oss-inputs + run: | + DIGEST=$(node apps/review-desktop/scripts/code-oss-bootstrap.mjs digest apps/review-desktop/code-oss) + echo "digest=$DIGEST" >> "$GITHUB_OUTPUT" + - name: Restore Code OSS dependencies + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: | + apps/review-desktop/code-oss/node_modules + apps/review-desktop/code-oss/build/node_modules + apps/review-desktop/code-oss/extensions/node_modules + apps/review-desktop/code-oss/extensions/*/node_modules + apps/review-desktop/code-oss/extensions/*/server/node_modules + apps/review-desktop/code-oss/.build/distro/npm/node_modules + apps/review-desktop/code-oss/.build/distro/npm/remote/node_modules + apps/review-desktop/code-oss/.build/distro/npm/remote/web/node_modules + ~/.cache/node-gyp + key: ${{ runner.os }}-code-oss-deps-v3-${{ steps.code-oss-inputs.outputs.digest }} + restore-keys: | + ${{ runner.os }}-code-oss-deps-v3- + - name: Build the VS Code server + run: | + source apps/review-desktop/scripts/code-oss-dependencies.sh + ensure_code_oss_dependencies "$PWD/apps/review-desktop" "$PWD/apps/review-desktop/code-oss" + BUILD_SOURCEVERSION="$COMMIT" node apps/review-desktop/scripts/build-remote-runtime.mjs - name: Build release tarball run: | jq -n --arg version "$VERSION" --arg commit "$COMMIT" '{$version, $commit}' > release-plan.json node scripts/pack-review-cli.mjs release-artifacts - name: Test installed tarball without Desktop run: node scripts/smoke-review-cli.mjs "release-artifacts/dev.fast-whiteboard-${VERSION}.tgz" "$VERSION" + - name: Pin the remote package and Node + id: remote + env: + PUBLISHED: ${{ steps.npm.outputs.published }} + DRY_RUN: ${{ inputs.dry_run }} + run: | + TARBALL="release-artifacts/dev.fast-whiteboard-${VERSION}.tgz" + if [ "$PUBLISHED" = true ] && [ "$DRY_RUN" != true ]; then + TARBALL="$RUNNER_TEMP/$(cd "$RUNNER_TEMP" && npm pack "@dev.fast/whiteboard@${VERSION}" --silent)" + fi + PIN=$(node apps/review-desktop/scripts/stamp-remote-package.mjs "$TARBALL") + echo "whiteboard_remote=$PIN" >> "$GITHUB_OUTPUT" - name: Validate npm publish command if: ${{ steps.npm.outputs.published != 'true' }} env: @@ -346,6 +400,7 @@ jobs: with: commit: ${{ needs.version.outputs.commit }} version: ${{ needs.version.outputs.version }} + whiteboard_remote: ${{ needs.npm.outputs.whiteboard_remote }} release_signing: ${{ !inputs.dry_run }} secrets: inherit @@ -360,6 +415,7 @@ jobs: with: commit: ${{ needs.version.outputs.commit }} version: ${{ needs.version.outputs.version }} + whiteboard_remote: ${{ needs.npm.outputs.whiteboard_remote }} sign: true macos: @@ -372,6 +428,7 @@ jobs: with: commit: ${{ needs.version.outputs.commit }} version: ${{ needs.version.outputs.version }} + whiteboard_remote: ${{ needs.npm.outputs.whiteboard_remote }} channel: stable secrets: inherit diff --git a/.github/workflows/review-linux-build.yml b/.github/workflows/review-linux-build.yml index 4d7864ebe..2c82554dd 100644 --- a/.github/workflows/review-linux-build.yml +++ b/.github/workflows/review-linux-build.yml @@ -9,6 +9,10 @@ on: version: type: string default: "" + whiteboard_remote: + description: The npm job's remote pin (stamp-remote-package.mjs); required with a version. + type: string + default: "" package_revision: type: string default: "1" @@ -48,6 +52,11 @@ jobs: QUALITY=stable if [[ "$VERSION" == *-preview.* ]]; then QUALITY=preview; fi node apps/review-desktop/scripts/stamp-release-channel.mjs --version "$VERSION" --quality "$QUALITY" + - name: Pin the remote package and Node + if: inputs.version != '' + env: + WHITEBOARD_REMOTE: ${{ inputs.whiteboard_remote }} + run: node apps/review-desktop/scripts/stamp-remote-package.mjs --pinned "$WHITEBOARD_REMOTE" - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 with: diff --git a/.github/workflows/review-macos-build.yml b/.github/workflows/review-macos-build.yml index 651cd9eef..6cc8034ec 100644 --- a/.github/workflows/review-macos-build.yml +++ b/.github/workflows/review-macos-build.yml @@ -12,6 +12,10 @@ on: channel: required: true type: string + whiteboard_remote: + description: The npm job's remote pin (stamp-remote-package.mjs). + required: true + type: string permissions: contents: read @@ -155,6 +159,11 @@ jobs: --version "$RELEASE_VERSION" \ --quality "${{ inputs.channel }}" + - name: Pin the remote package and Node + env: + WHITEBOARD_REMOTE: ${{ inputs.whiteboard_remote }} + run: node apps/review-desktop/scripts/stamp-remote-package.mjs --pinned "$WHITEBOARD_REMOTE" + - name: Setup pnpm uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 diff --git a/.github/workflows/review-windows-build.yml b/.github/workflows/review-windows-build.yml index 8644e59b7..280574614 100644 --- a/.github/workflows/review-windows-build.yml +++ b/.github/workflows/review-windows-build.yml @@ -9,6 +9,10 @@ on: version: type: string required: true + whiteboard_remote: + description: The npm job's remote pin (stamp-remote-package.mjs); required with a version. + type: string + default: "" artifact_name: type: string default: windows-packages @@ -65,6 +69,10 @@ jobs: QUALITY=stable if [[ "$VERSION" == *-preview.* ]]; then QUALITY=preview; fi node apps/review-desktop/scripts/stamp-release-channel.mjs --version "$VERSION" --quality "$QUALITY" + - name: Pin the remote package and Node + env: + WHITEBOARD_REMOTE: ${{ inputs.whiteboard_remote }} + run: node apps/review-desktop/scripts/stamp-remote-package.mjs --pinned "$WHITEBOARD_REMOTE" - name: Restore native dependencies id: native-cache uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ceae9a2e0..3c7256fc9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,15 +66,6 @@ Run the full check suite before you open a pull request: pnpm run ci ``` -DOM-facing tests run in Chromium through Vitest Browser Mode. Install the -browser once, then run the headless suite or watch mode: - -```sh -pnpm --filter @dev.fast/whiteboard exec playwright install chromium -pnpm --filter @dev.fast/whiteboard test:browser -pnpm --filter @dev.fast/whiteboard test:browser:watch -``` - Pure Node, filesystem, and server tests run through `pnpm --filter @dev.fast/whiteboard test:node`. diff --git a/apps/review-desktop/UPSTREAM b/apps/review-desktop/UPSTREAM index 89ebcb15b..665f17e88 100644 --- a/apps/review-desktop/UPSTREAM +++ b/apps/review-desktop/UPSTREAM @@ -355,6 +355,26 @@ one of these entries: - `src/vs/platform/terminal/common/terminalEnvironment.ts`: quote complete path arguments with each shell's native rules instead of stripping metacharacters and emitting invalid POSIX quoting for apostrophes. +- `src/vs/platform/remote/common/remoteAgentConnection.ts`: a permanent + failure ends only its own connection. Upstream's permanent failure is one + static flag that ends every connection in the window; with several remotes + per window a failed remote must end only its own connection. A failed + connection also sets upstream's never-written per-instance flag so it does + not reconnect again. A Management connection's reconnection failure is not + fatal either: it ends that connection only. Isolation is unconditional, so a + remote Source window gets no upstream reload dialog; it reconnects through + the fork's resolver and shows its host state in the status bar. +- `src/vs/workbench/electron-browser/desktop.main.ts`: `createWorkspaceService` + is protected, not private, so the navigator entry's `DesktopMain` subclass + wraps a Source window's configuration service + (`review/services/configuration/reviewSourceWindowConfiguration.ts`): no + extension default, workspace file, folder or host machine setting can lower + its read-only rule, and the fork sets its title. +- `src/vs/platform/window/common/window.ts` and + `src/vs/platform/native/electron-main/nativeHostMainService.ts`: the + `openWindow` options `reviewReferencesToShow` and `reviewSourceTitle` run a + navigator command in the opened window (Find References, the title's review and side, + and a remote review's host alias for the window's host-state entry). - `extensions/package.json` and the extension `package.json`/`package-lock.json` files: `@vscode/extension-telemetry` is hoisted to the shared extensions install (github keeps its own ^1.0.0 copy; the shared copy is ^0.9.8). The @@ -379,8 +399,10 @@ one of these entries: were regenerated rather than kept at the stock Code OSS GUIDs. Two identity gaps remain deliberately: `serverApplicationName`, `serverDataFolderName`, and `tunnelApplicationName` still hold their upstream - values because the fork ships no REH build and the agent-host stack uses those - names to locate a server on *remote* hosts; and `darwinProfileUUID` / + values because the agent-host stack uses those names to locate a server on + *remote* hosts. The fork does ship a trimmed remote server (below), but its + launcher (`whiteboard remote attach`) always passes `--server-data-dir`, so the + `serverDataFolderName` default is never used; and `darwinProfileUUID` / `darwinProfilePayloadUUID` are untouched upstream values kept only because `build/lib/policies/policyGenerator.ts` hard-throws without them, on a path nothing in this tree invokes. @@ -478,7 +500,8 @@ one of these entries: `workbench-dev.html`: allow the local Review session connection and the native canvas Trusted Types policy in built and development launches. - `src/vs/base/common/product.ts`: declares the `reviewVersion` field described - above, so the About panel can read it typed. + above, so the About panel can read it typed, and `whiteboardRemote`, the + remote package and Node that release builds pin for remote installs. - `src/vs/code/electron-main/app.ts`: own the embedded Review server lifecycle and register the Review connection channel that hands the renderer the server endpoint the main process validated. Also swaps two update- and menu-related @@ -581,6 +604,35 @@ one of these entries: - `licenses/`: fork-added third-party notices (eventsource-parser, zod, elkjs, libavoid-js, KaTeX, @vscode/codicons). `build/gulpfile.vscode.ts` copies this directory into packaged builds. +- The remote server: a trimmed copy of upstream's server starts the stock + extension host on a remote machine, so remote reviews get language features + from the same extensions. `scripts/build-remote-runtime.mjs` bundles it. + Restored from the upstream commit above, unmodified: `src/server-main.ts`, + `src/bootstrap-server.ts`, and in `src/vs/server/node/`: + `extensionHostConnection.ts`, `extensionHostStatusService.ts`, + `extensionsScannerService.ts`, `remoteAgentEnvironmentImpl.ts`, + `remoteExtensionManagement.ts`, `remoteFileSystemProviderServer.ts`, + `remoteLanguagePacks.ts`, `serverConnectionToken.ts`, + `serverEnvironmentService.ts`, `serverLifetimeService.ts`. + Restored and trimmed, each trim marked with a `Whiteboard:` comment: + - `serverServices.ts`: keeps the logger, file, configuration, user data + profile, extension host status, lifetime and extensions scanner services, + and the `logger`, `userDataProfiles`, `remoteextensionsenvironment`, + `remoteExtensionsScanner`, `remoteFilesystem` and extension host debug + broadcast channels. `telemetry` is a no-op channel and `ITelemetryService` + is `NullTelemetryService`. Cuts requests, telemetry appenders and machine + ids, the gallery and extension installs, language packs, terminals, the + agent host, MCP and the sandbox helper. Installs the process error + listeners upstream's node `ErrorTelemetry` installs, so a cancelled request + after a clean disconnect does not end the server. + - `remoteExtensionsScanner.ts`: scans only; install-on-start, the gallery and + language pack installs are cut. + - `remoteExtensionHostAgentServer.ts`: the web client and + `/vscode-remote-resource` are cut, with a local `serveError`. + - `server.main.ts`: `spawnCli` reports that the extension management CLI is + not included. + The rest of `src/vs/server/` and `src/server-cli.ts` stay deleted. + `src/vs/server/node/reviewServerHandshake.test.ts` is fork-owned. ## Serialize the fork diff --git a/apps/review-desktop/code-oss/build/buildfile.ts b/apps/review-desktop/code-oss/build/buildfile.ts index 18794225e..4bcff92df 100644 --- a/apps/review-desktop/code-oss/build/buildfile.ts +++ b/apps/review-desktop/code-oss/build/buildfile.ts @@ -26,7 +26,8 @@ export const workbenchDesktop = [ createModuleDescription('vs/workbench/api/node/extensionHostProcess'), createModuleDescription('vs/review/review.desktop.main'), createModuleDescription('vs/review/navigator.desktop.main'), - createModuleDescription('vs/review/electron-utility/reviewDesktopHostMain') + createModuleDescription('vs/review/electron-utility/reviewDesktopHostMain'), + createModuleDescription('vs/review/node/reviewSshAskpassMain') ]; export const keyboardMaps = [ diff --git a/apps/review-desktop/code-oss/build/next/index.ts b/apps/review-desktop/code-oss/build/next/index.ts index e0d703b08..3c19fd479 100644 --- a/apps/review-desktop/code-oss/build/next/index.ts +++ b/apps/review-desktop/code-oss/build/next/index.ts @@ -96,6 +96,7 @@ const desktopEntryPoints = [ 'vs/review/review.desktop.main', 'vs/review/navigator.desktop.main', 'vs/review/electron-utility/reviewDesktopHostMain', + 'vs/review/node/reviewSshAskpassMain', 'vs/workbench/contrib/debug/node/telemetryApp', 'vs/platform/files/node/watcher/watcherMain', 'vs/platform/terminal/node/ptyHostMain', diff --git a/apps/review-desktop/code-oss/src/bootstrap-server.ts b/apps/review-desktop/code-oss/src/bootstrap-server.ts new file mode 100644 index 000000000..d440e4195 --- /dev/null +++ b/apps/review-desktop/code-oss/src/bootstrap-server.ts @@ -0,0 +1,7 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +// Keep bootstrap-esm.js from redefining 'fs'. +delete process.env['ELECTRON_RUN_AS_NODE']; diff --git a/apps/review-desktop/code-oss/src/server-main.ts b/apps/review-desktop/code-oss/src/server-main.ts new file mode 100644 index 000000000..08593e461 --- /dev/null +++ b/apps/review-desktop/code-oss/src/server-main.ts @@ -0,0 +1,408 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import './bootstrap-server.js'; // this MUST come before other imports as it changes global state +import * as path from 'node:path'; +import * as fs from 'node:fs'; +import * as http from 'node:http'; +import type { AddressInfo } from 'node:net'; +import * as os from 'node:os'; +import * as readline from 'node:readline'; +import { performance } from 'node:perf_hooks'; +import minimist from 'minimist'; +import { devInjectNodeModuleLookupPath, removeGlobalNodeJsModuleLookupPaths } from './bootstrap-node.js'; +import { bootstrapESM } from './bootstrap-esm.js'; +import { resolveNLSConfiguration } from './vs/base/node/nls.js'; +import { product } from './bootstrap-meta.js'; +import * as perf from './vs/base/common/performance.js'; +import { INLSConfiguration } from './vs/nls.js'; +import { IServerAPI } from './vs/server/node/remoteExtensionHostAgentServer.js'; + +perf.mark('code/server/start'); +(globalThis as { vscodeServerStartTime?: number }).vscodeServerStartTime = performance.now(); + +// Do a quick parse to determine if a server or the cli needs to be started +const parsedArgs = minimist(process.argv.slice(2), { + boolean: ['start-server', 'list-extensions', 'print-ip-address', 'help', 'version', 'accept-server-license-terms', 'update-extensions'], + string: ['install-extension', 'install-builtin-extension', 'uninstall-extension', 'locate-extension', 'socket-path', 'host', 'port', 'compatibility', 'agent-host-port', 'agent-host-path'], + alias: { help: 'h', version: 'v' } +}); +['host', 'port', 'accept-server-license-terms'].forEach(e => { + if (!parsedArgs[e]) { + const envValue = process.env[`VSCODE_SERVER_${e.toUpperCase().replace('-', '_')}`]; + if (envValue) { + parsedArgs[e] = envValue; + } + } +}); + +const extensionLookupArgs = ['list-extensions', 'locate-extension']; +const extensionInstallArgs = ['install-extension', 'install-builtin-extension', 'uninstall-extension', 'update-extensions']; + +const shouldSpawnCli = parsedArgs.help || parsedArgs.version || extensionLookupArgs.some(a => !!parsedArgs[a]) || (extensionInstallArgs.some(a => !!parsedArgs[a]) && !parsedArgs['start-server']); + +const nlsConfiguration = await resolveNLSConfiguration({ userLocale: 'en', osLocale: 'en', commit: product.commit, userDataPath: '', nlsMetadataPath: import.meta.dirname }); + +if (shouldSpawnCli) { + loadCode(nlsConfiguration).then((mod) => { + mod.spawnCli(); + }); +} else { + installServerProcessExitDiagnostics(); + + let _remoteExtensionHostAgentServer: IServerAPI | null = null; + let _remoteExtensionHostAgentServerPromise: Promise | null = null; + const getRemoteExtensionHostAgentServer = () => { + if (!_remoteExtensionHostAgentServerPromise) { + _remoteExtensionHostAgentServerPromise = loadCode(nlsConfiguration).then(async (mod) => { + const server = await mod.createServer(address); + _remoteExtensionHostAgentServer = server; + return server; + }); + } + return _remoteExtensionHostAgentServerPromise; + }; + + if (Array.isArray(product.serverLicense) && product.serverLicense.length) { + console.log(product.serverLicense.join('\n')); + if (product.serverLicensePrompt && parsedArgs['accept-server-license-terms'] !== true) { + if (hasStdinWithoutTty()) { + console.log('To accept the license terms, start the server with --accept-server-license-terms'); + process.exit(1); + } + try { + const accept = await prompt(product.serverLicensePrompt); + if (!accept) { + process.exit(1); + } + } catch (e) { + console.log(e); + process.exit(1); + } + } + } + + let firstRequest = true; + let firstWebSocket = true; + + let address: string | AddressInfo | null = null; + const server = http.createServer(async (req, res) => { + if (firstRequest) { + firstRequest = false; + perf.mark('code/server/firstRequest'); + } + const remoteExtensionHostAgentServer = await getRemoteExtensionHostAgentServer(); + return remoteExtensionHostAgentServer.handleRequest(req, res); + }); + server.on('upgrade', async (req, socket) => { + if (firstWebSocket) { + firstWebSocket = false; + perf.mark('code/server/firstWebSocket'); + } + const remoteExtensionHostAgentServer = await getRemoteExtensionHostAgentServer(); + // @ts-expect-error + return remoteExtensionHostAgentServer.handleUpgrade(req, socket); + }); + server.on('error', async (err) => { + const remoteExtensionHostAgentServer = await getRemoteExtensionHostAgentServer(); + return remoteExtensionHostAgentServer.handleServerError(err); + }); + + const host = sanitizeStringArg(parsedArgs['host']) || (parsedArgs['compatibility'] !== '1.63' ? 'localhost' : undefined); + const nodeListenOptions = ( + parsedArgs['socket-path'] + ? { path: sanitizeStringArg(parsedArgs['socket-path']) } + : { host, port: await parsePort(host, sanitizeStringArg(parsedArgs['port'])) } + ); + server.listen(nodeListenOptions, async () => { + let output = Array.isArray(product.serverGreeting) && product.serverGreeting.length ? `\n\n${product.serverGreeting.join('\n')}\n\n` : ``; + + if (typeof nodeListenOptions.port === 'number' && parsedArgs['print-ip-address']) { + const ifaces = os.networkInterfaces(); + Object.keys(ifaces).forEach(function (ifname) { + ifaces[ifname]?.forEach(function (iface) { + if (!iface.internal && iface.family === 'IPv4') { + output += `IP Address: ${iface.address}\n`; + } + }); + }); + } + + address = server.address(); + if (address === null) { + throw new Error('Unexpected server address'); + } + + output += `Server bound to ${typeof address === 'string' ? address : `${address.address}:${address.port} (${address.family})`}\n`; + // Do not change this line. VS Code looks for this in the output. + output += `Extension host agent listening on ${typeof address === 'string' ? address : address.port}\n`; + console.log(output); + + perf.mark('code/server/started'); + (globalThis as { vscodeServerListenTime?: number }).vscodeServerListenTime = performance.now(); + + await getRemoteExtensionHostAgentServer(); + }); + + process.on('exit', () => { + server.close(); + if (_remoteExtensionHostAgentServer) { + _remoteExtensionHostAgentServer.dispose(); + } + }); +} + +function sanitizeStringArg(val: unknown): string | undefined { + if (Array.isArray(val)) { // if an argument is passed multiple times, minimist creates an array + val = val.pop(); // take the last item + } + return typeof val === 'string' ? val : undefined; +} + +/** + * Records why/when the remote server process exits, to help debug unexpected + * server exits in the remote smoke tests (which surface to the client as + * `Unknown reconnection token` reconnection failures). The handlers tell apart a + * self-exit (`beforeExit`), an external kill (`signal`) and a crash + * (`uncaughtExceptionMonitor`). Gated behind the `VSCODE_SERVER_EXIT_DIAGNOSTICS` + * env var (set by the smoke tests) so it adds no product noise. Lines are + * appended synchronously to a `server-exit-diagnostics.log` file in the server's + * `--logsPath` directory (falling back to `os.tmpdir()` when `--logsPath` is not + * provided) so they survive process teardown (an async stdio write from an + * `exit` handler does not). + */ +function installServerProcessExitDiagnostics(): void { + if (!process.env['VSCODE_SERVER_EXIT_DIAGNOSTICS']) { + return; + } + + const startTime = Date.now(); + + // Append diagnostics synchronously to a file rather than relying on + // `console.error`: a process `exit` handler cannot flush an async pipe write + // (the server's stdio is piped through the test resolver, and on Windows + // additionally through a `cmd.exe`/batch wrapper) before the process dies, + // so the exit-time lines we care about most were being dropped. A synchronous + // `fs.appendFileSync` survives teardown. We target the server's `--logsPath` + // directory because it is captured as a smoke test artifact. + const logsPath = sanitizeStringArg(parsedArgs['logsPath']) || os.tmpdir(); + const diagnosticsFile = path.join(logsPath, 'server-exit-diagnostics.log'); + try { + fs.mkdirSync(logsPath, { recursive: true }); + } catch { + // best effort: the directory is normally created by the server already + } + + // The file write is authoritative: it is synchronous (so it survives process + // teardown) and goes to a captured smoke artifact. We additionally mirror to + // stderr for live visibility in the test resolver's output channel, but that + // mirror is dangerous precisely because these diagnostics fire when the + // server's stdio pipe is dying: a write to a broken pipe throws `EPIPE` + // synchronously and/or emits an async `error` event, either of which Node + // promotes to an uncaught exception — which re-enters the + // `uncaughtExceptionMonitor` handler below and loops (one CI run produced a + // 386MB log this way). We therefore make the mirror best-effort and latch it + // off after the first failure, and attach an `error` handler so async pipe + // errors are swallowed rather than crashing the process. + let mirrorToStderr = true; + try { + process.stderr.on('error', () => { mirrorToStderr = false; }); + } catch { + mirrorToStderr = false; + } + + const log = (message: string) => { + const line = `[server-exit-diagnostics][${new Date().toISOString()}][pid:${process.pid}][+${Date.now() - startTime}ms] ${message}`; + try { + fs.appendFileSync(diagnosticsFile, `${line}\n`); + } catch { + // ignore logging failures while the process is tearing down + } + if (mirrorToStderr) { + try { + process.stderr.write(`${line}\n`); + } catch { + // Broken pipe during teardown: stop mirroring so we can never + // throw (and thus loop) on subsequent diagnostics. + mirrorToStderr = false; + } + } + }; + + const describeState = (): string => { + try { + const processWithResources = process as NodeJS.Process & { getActiveResourcesInfo?(): string[] }; + const activeResources = processWithResources.getActiveResourcesInfo?.() ?? []; + const memory = process.memoryUsage(); + return `uptime=${process.uptime().toFixed(3)}s rss=${Math.round(memory.rss / 1024 / 1024)}MB activeResources=[${activeResources.join(', ')}]`; + } catch (err) { + return `(failed to collect process state: ${err})`; + } + }; + + log(`installed. ppid=${process.ppid} platform=${process.platform} node=${process.version} argv=${JSON.stringify(process.argv.slice(2))}`); + + process.on('beforeExit', code => log(`'beforeExit' (code: ${code}) — event loop drained, process will exit on its own. ${describeState()}`)); + process.on('exit', code => log(`'exit' (code: ${code}). ${describeState()}`)); + + // `uncaughtExceptionMonitor` is observational: it runs before the process + // crashes but does NOT prevent the default crash, so the real failure mode + // is preserved. It also fires for unhandled rejections that get promoted to + // uncaught exceptions by Node's default policy. Guard against re-entrancy: + // if logging an exception were to itself throw (and get promoted to another + // uncaught exception), we must not recurse into this handler forever. + let handlingUncaughtException = false; + process.on('uncaughtExceptionMonitor', (err, origin) => { + if (handlingUncaughtException) { + return; + } + handlingUncaughtException = true; + try { + log(`'uncaughtExceptionMonitor' (origin: ${origin}): ${err?.stack || err}`); + } finally { + handlingUncaughtException = false; + } + }); + + const signals: NodeJS.Signals[] = ['SIGTERM', 'SIGINT', 'SIGHUP', 'SIGBREAK', 'SIGQUIT']; + for (const signal of signals) { + try { + process.on(signal, () => { + log(`received signal '${signal}' — terminating. ${describeState()}`); + // Preserve default termination semantics after logging. + const signalNumber = (os.constants.signals as Record)[signal]; + process.exit(typeof signalNumber === 'number' ? 128 + signalNumber : 1); + }); + } catch { + // Not all signals can be listened to on all platforms (e.g. SIGBREAK). + } + } +} + +/** + * If `--port` is specified and describes a single port, connect to that port. + * + * If `--port`describes a port range + * then find a free port in that range. Throw error if no + * free port available in range. + * + * In absence of specified ports, connect to port 8000. + */ +async function parsePort(host: string | undefined, strPort: string | undefined): Promise { + if (strPort) { + let range: { start: number; end: number } | undefined; + if (strPort.match(/^\d+$/)) { + return parseInt(strPort, 10); + } else if (range = parseRange(strPort)) { + const port = await findFreePort(host, range.start, range.end); + if (port !== undefined) { + return port; + } + // Remote-SSH extension relies on this exact port error message, treat as an API + console.warn(`--port: Could not find free port in range: ${range.start} - ${range.end} (inclusive).`); + process.exit(1); + + } else { + console.warn(`--port "${strPort}" is not a valid number or range. Ranges must be in the form 'from-to' with 'from' an integer larger than 0 and not larger than 'end'.`); + process.exit(1); + } + } + return 8000; +} + +function parseRange(strRange: string): { start: number; end: number } | undefined { + const match = strRange.match(/^(\d+)-(\d+)$/); + if (match) { + const start = parseInt(match[1], 10), end = parseInt(match[2], 10); + if (start > 0 && start <= end && end <= 65535) { + return { start, end }; + } + } + return undefined; +} + +/** + * Starting at the `start` port, look for a free port incrementing + * by 1 until `end` inclusive. If no free port is found, undefined is returned. + */ +async function findFreePort(host: string | undefined, start: number, end: number): Promise { + const testPort = (port: number) => { + return new Promise((resolve) => { + const server = http.createServer(); + server.listen(port, host, () => { + server.close(); + resolve(true); + }).on('error', () => { + resolve(false); + }); + }); + }; + for (let port = start; port <= end; port++) { + if (await testPort(port)) { + return port; + } + } + return undefined; +} + +async function loadCode(nlsConfiguration: INLSConfiguration) { + + // required for `bootstrap-esm` to pick up NLS messages + process.env['VSCODE_NLS_CONFIG'] = JSON.stringify(nlsConfiguration); + + // See https://github.com/microsoft/vscode-remote-release/issues/6543 + // We would normally install a SIGPIPE listener in bootstrap-node.js + // But in certain situations, the console itself can be in a broken pipe state + // so logging SIGPIPE to the console will cause an infinite async loop + process.env['VSCODE_HANDLES_SIGPIPE'] = 'true'; + + if (process.env['VSCODE_DEV']) { + // When running out of sources, we need to load node modules from remote/node_modules, + // which are compiled against nodejs, not electron + process.env['VSCODE_DEV_INJECT_NODE_MODULE_LOOKUP_PATH'] = process.env['VSCODE_DEV_INJECT_NODE_MODULE_LOOKUP_PATH'] || path.join(import.meta.dirname, '..', 'remote', 'node_modules'); + devInjectNodeModuleLookupPath(process.env['VSCODE_DEV_INJECT_NODE_MODULE_LOOKUP_PATH']); + } else { + delete process.env['VSCODE_DEV_INJECT_NODE_MODULE_LOOKUP_PATH']; + } + + // Remove global paths from the node module lookup (node.js only) + removeGlobalNodeJsModuleLookupPaths(); + + // Bootstrap ESM + await bootstrapESM(); + + // Load Server + return import('./vs/server/node/server.main.js'); +} + +function hasStdinWithoutTty(): boolean { + try { + return !process.stdin.isTTY; // Via https://twitter.com/MylesBorins/status/782009479382626304 + } catch (error) { + // Windows workaround for https://github.com/nodejs/node/issues/11656 + } + return false; +} + +function prompt(question: string): Promise { + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout + }); + return new Promise((resolve, reject) => { + rl.question(question + ' ', async function (data) { + rl.close(); + const str = data.toString().trim().toLowerCase(); + if (str === '' || str === 'y' || str === 'yes') { + resolve(true); + } else if (str === 'n' || str === 'no') { + resolve(false); + } else { + process.stdout.write('\nInvalid Response. Answer either yes (y, yes) or no (n, no)\n'); + resolve(await prompt(question)); + } + }); + }); +} diff --git a/apps/review-desktop/code-oss/src/vs/base/common/product.ts b/apps/review-desktop/code-oss/src/vs/base/common/product.ts index 0f75ee30d..5f3e638cd 100644 --- a/apps/review-desktop/code-oss/src/vs/base/common/product.ts +++ b/apps/review-desktop/code-oss/src/vs/base/common/product.ts @@ -64,6 +64,16 @@ export type ExtensionVirtualWorkspaceSupport = { readonly override?: boolean; }; +export interface IWhiteboardRemoteNode { + readonly url: string; + readonly sha256: string; +} + +export interface IWhiteboardRemoteProduct { + readonly package: { readonly name: string; readonly version: string; readonly integrity: string }; + readonly node: { readonly version: string; readonly 'linux-x64': IWhiteboardRemoteNode; readonly 'linux-arm64': IWhiteboardRemoteNode }; +} + export interface IProductConfiguration { readonly version: string; readonly date?: string; @@ -78,6 +88,8 @@ export interface IProductConfiguration { */ readonly reviewVersion?: string; + readonly whiteboardRemote?: IWhiteboardRemoteProduct; + readonly nameShort: string; readonly nameLong: string; diff --git a/apps/review-desktop/code-oss/src/vs/platform/native/electron-main/nativeHostMainService.ts b/apps/review-desktop/code-oss/src/vs/platform/native/electron-main/nativeHostMainService.ts index 575f64792..b661cdaa0 100644 --- a/apps/review-desktop/code-oss/src/vs/platform/native/electron-main/nativeHostMainService.ts +++ b/apps/review-desktop/code-oss/src/vs/platform/native/electron-main/nativeHostMainService.ts @@ -313,6 +313,10 @@ export class NativeHostMainService extends Disposable implements INativeHostMain args: [URI.revive(references.resource).toString(), references.lineNumber, references.column], }); } + const sourceTitle = options.reviewSourceTitle; + if (sourceTitle && windows.length === 1) { + windows[0].sendWhenReady('vscode:runAction', CancellationToken.None, { id: 'review.action.setSourceTitle', from: 'review', args: [sourceTitle] }); + } } } diff --git a/apps/review-desktop/code-oss/src/vs/platform/remote/common/remoteAgentConnection.ts b/apps/review-desktop/code-oss/src/vs/platform/remote/common/remoteAgentConnection.ts index f87f8c67f..393437bfa 100644 --- a/apps/review-desktop/code-oss/src/vs/platform/remote/common/remoteAgentConnection.ts +++ b/apps/review-desktop/code-oss/src/vs/platform/remote/common/remoteAgentConnection.ts @@ -450,7 +450,6 @@ async function createInitialConnection; constructor(options: IConnectionOptions, remoteAuthority: string, clientId: string, reconnectionToken: string, protocol: PersistentProtocol) { - super(ConnectionType.Management, options, reconnectionToken, protocol, /*reconnectionFailureIsFatal*/true); + super(ConnectionType.Management, options, reconnectionToken, protocol, /*reconnectionFailureIsFatal*/false); this.client = this._register(new Client(protocol, { remoteAuthority: remoteAuthority, clientId: clientId diff --git a/apps/review-desktop/code-oss/src/vs/platform/window/common/window.ts b/apps/review-desktop/code-oss/src/vs/platform/window/common/window.ts index 7f9d2fc89..67890bae8 100644 --- a/apps/review-desktop/code-oss/src/vs/platform/window/common/window.ts +++ b/apps/review-desktop/code-oss/src/vs/platform/window/common/window.ts @@ -79,6 +79,9 @@ export interface IOpenWindowOptions extends IBaseOpenWindowsOptions { /** Run Find References at this source location in the opened window. */ readonly reviewReferencesToShow?: { readonly resource: URI; readonly lineNumber: number; readonly column: number }; + + /** The review and side the opened Source window shows; its title names both. `alias` names a remote review's host. */ + readonly reviewSourceTitle?: { readonly side: 'live' | 'base' | 'head'; readonly title: string; readonly alias?: string }; } export interface IAddRemoveFoldersRequest { diff --git a/apps/review-desktop/code-oss/src/vs/review/browser/parts/canvas/reviewApiEditorSerializer.test.ts b/apps/review-desktop/code-oss/src/vs/review/browser/parts/canvas/reviewApiEditorSerializer.test.ts index 0a474ab7c..f81b30931 100644 --- a/apps/review-desktop/code-oss/src/vs/review/browser/parts/canvas/reviewApiEditorSerializer.test.ts +++ b/apps/review-desktop/code-oss/src/vs/review/browser/parts/canvas/reviewApiEditorSerializer.test.ts @@ -26,14 +26,17 @@ test("opening historical source requests its version and opens a separate native { async getConnection() { return { serverUrl: "http://localhost", token: "test" }; } } as never, { async openWindow(...args: unknown[]) { opened.push(args); } } as never, { warn() {} } as never, + {} as never, + {} as never, + { reviews: [] } as never, ); t.after(() => tabs.dispose()); - await tabs.openApiSource({ reviewId: "review-a", kind: "version", version: 7 }, "Historical Review"); + await tabs.openApiSource({ reviewId: "review-a", kind: "version", version: 7 }, "Historical Review", false); assert.equal(requests[0], "http://localhost/reviews-api/review-a/navigator?version=7"); assert.equal(opened.length, 1); - assert.deepEqual(opened[0]![1], { forceNewWindow: true }); + assert.deepEqual(opened[0]![1], { forceNewWindow: true, reviewSourceTitle: { side: "head", title: "Historical Review" } }); ok = false; - await assert.rejects(tabs.openApiSource({ reviewId: "review-a", kind: "current" }, "Review")); + await assert.rejects(tabs.openApiSource({ reviewId: "review-a", kind: "current" }, "Review", false)); assert.equal(opened.length, 1); }); @@ -60,7 +63,7 @@ test("native group restoration preserves both reviews, order and pinned source v }; const groupService = { groups, mainPart: { activeGroup: undefined as EditorGroupModel | undefined } }; const createTabs = () => - new ReviewCanvasEditorTabsService(instantiation as never, editors as never, groupService as never, {} as never, {} as never, { warn() {} } as never); + new ReviewCanvasEditorTabsService(instantiation as never, editors as never, groupService as never, {} as never, {} as never, { warn() {} } as never, {} as never, {} as never, { reviews: [] } as never); tabs = createTabs(); const registry = Registry.as(EditorExtensions.EditorFactory); registry.start({ get: () => instantiation } as never); @@ -127,7 +130,7 @@ test("current Source tabs retain identity and main version tabs still restore", }, invokeFunction(fn: (accessor: { get(): ReviewCanvasEditorTabsService }) => unknown) { return fn({ get: () => tabs }); }, }; - tabs = new ReviewCanvasEditorTabsService(instantiation as never, { onDidCloseEditor: Event.None } as never, {} as never, {} as never, {} as never, { warn() {} } as never); + tabs = new ReviewCanvasEditorTabsService(instantiation as never, { onDidCloseEditor: Event.None } as never, {} as never, {} as never, {} as never, { warn() {} } as never, {} as never, {} as never, { reviews: [] } as never); try { const serializer = new ReviewApiEditorSerializer(); const restored = serializer.deserialize(instantiation as never, JSON.stringify({ kind: "api-source", reviewId: "a", title: "A", selection: { reviewId: "a", kind: "current" } })); diff --git a/apps/review-desktop/code-oss/src/vs/review/browser/parts/canvas/reviewCanvasPart.ts b/apps/review-desktop/code-oss/src/vs/review/browser/parts/canvas/reviewCanvasPart.ts index 49a02d506..35106b9cd 100644 --- a/apps/review-desktop/code-oss/src/vs/review/browser/parts/canvas/reviewCanvasPart.ts +++ b/apps/review-desktop/code-oss/src/vs/review/browser/parts/canvas/reviewCanvasPart.ts @@ -101,6 +101,8 @@ import "../../media/review.css"; import { ReviewSessionTelemetry } from "../../reviewSessionTelemetry.js"; import { applyReviewThemeChoice, currentReviewThemeChoice } from "../../reviewThemeChoice.js"; import { ReviewCanvasEditorInput } from "./reviewCanvasEditorInput.js"; +import { remoteEntry, withRemoteEntry } from "./reviewRemoteEntry.js"; +import { reviewRemoteHostsSettings } from "../../reviewRemoteHostsSettings.js"; interface ReviewCanvasAssetsModule extends ReviewCanvasModule { readonly clearReviewViewState: (config: ReviewRuntimeConfig) => void; @@ -212,6 +214,12 @@ export class ReviewCanvasEditorPane extends EditorPane { const previous = catalog; catalog = this.apiCatalog.reviews; this.sessionTelemetry.catalogChanged(previous, catalog); + const content = this.apiContent; + const updated = content && withRemoteEntry(content, catalog.find((review) => review.reviewId === content.reviewId)); + if (updated) { + this.apiContent = updated; + this.canvas.value?.update(updated); + } })); this.inlineEditors = this._register(reviewInstantiationService.createInstance(ReviewEmbeddedEditors)); this.refreshProgress = this._register(new LongRunningOperation(editorProgressService)); @@ -415,6 +423,7 @@ export class ReviewCanvasEditorPane extends EditorPane { }, kind: "api", reviewId, + ...remoteEntry(this.apiCatalog.reviews.find((review) => review.reviewId === reviewId)), structuralDiffEnabled: this.currentStructuralDiffEnabled(), softwareMapEnabled: this.currentSoftwareMapEnabled(), documentWidth: this.currentDocumentWidth(), @@ -443,7 +452,7 @@ export class ReviewCanvasEditorPane extends EditorPane { request: requestReviewApi, post: async (request) => { if (request.name === "openSourceTree") { - await this.tabsService.openApiSource(sourceSelection, input.getName()); + await this.tabsService.openApiSource(sourceSelection, input.getName(), !!sourceView.generation); return { ok: true }; } if (request.name === "reveal") { @@ -520,10 +529,11 @@ export class ReviewCanvasEditorPane extends EditorPane { this.reviewTelemetryService.capture("review_restored", { via: "home" }); return this.apiCatalog.attention(uuid, "restore"); }, + hostStates: () => this.desktopConnection.readRemoteHosts(), openSourceTree: (uuid) => { const api = this.apiCatalog.reviews.find((review) => review.reviewId === uuid); if (api) { - void this.tabsService.openApiSource({ reviewId: api.reviewId, kind: "current" }, api.title).catch(error => this.notificationService.error(error)); + void this.tabsService.openApiSource({ reviewId: api.reviewId, kind: "current" }, api.title, !!api.pins?.worktreeRevision).catch(error => this.notificationService.error(error)); return; } }, @@ -851,6 +861,11 @@ export class ReviewCanvasEditorPane extends EditorPane { ); return this.currentStructuralDiffEnabled(); }, + remoteHosts: reviewRemoteHostsSettings({ + get: (key) => this.configurationService.getValue(key), + update: (key, value) => this.configurationService.updateValue(key, value, ConfigurationTarget.APPLICATION), + connection: this.desktopConnection, + }), reloadWindow: async () => { await this.commandService.executeCommand("workbench.action.reloadWindow"); }, diffrConfig: { saveSummarizer: (input) => this.desktopConnection.saveDiffrSummarizer(input), diff --git a/apps/review-desktop/code-oss/src/vs/review/browser/parts/canvas/reviewRemoteEntry.test.ts b/apps/review-desktop/code-oss/src/vs/review/browser/parts/canvas/reviewRemoteEntry.test.ts new file mode 100644 index 000000000..1fc09d905 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/browser/parts/canvas/reviewRemoteEntry.test.ts @@ -0,0 +1,23 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import type { ReviewApiSummary } from "../../../common/reviewProtocol.js"; +import { remoteEntry, withRemoteEntry } from "./reviewRemoteEntry.js"; + +const available = { sourceWindows: false, languageFeatures: false }; +const entry = (host?: string) => ({ reviewId: "r1", title: "Remote", ...(host && { host, available }) }) as ReviewApiSummary; + +test("a review a remote opened before its list entry arrived takes the host once the entry does", () => { + const mounted = { kind: "api", reviewId: "r1", ...remoteEntry(undefined) }; + assert.equal(mounted.host, undefined); + assert.equal(withRemoteEntry(mounted, undefined), undefined); + + const updated = withRemoteEntry(mounted, entry("wb-a")); + assert.deepEqual(updated, { kind: "api", reviewId: "r1", host: "wb-a", available }); + assert.equal(withRemoteEntry(updated!, entry("wb-a")), undefined); +}); + +test("a laptop review's entry changes nothing", () => { + const content: { reviewId: string; host?: string } = { reviewId: "r1" }; + assert.equal(withRemoteEntry(content, entry()), undefined); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/browser/parts/canvas/reviewRemoteEntry.ts b/apps/review-desktop/code-oss/src/vs/review/browser/parts/canvas/reviewRemoteEntry.ts new file mode 100644 index 000000000..af1d21611 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/browser/parts/canvas/reviewRemoteEntry.ts @@ -0,0 +1,12 @@ +import type { ReviewApiSummary } from "../../../common/reviewProtocol.js"; + +type RemoteFields = Pick; + +export const remoteEntry = (review: ReviewApiSummary | undefined): RemoteFields => + review?.host ? { host: review.host, available: review.available } : {}; + +export function withRemoteEntry(content: T, review: ReviewApiSummary | undefined): T | undefined { + const entry = remoteEntry(review); + if (!entry.host || (entry.host === content.host && JSON.stringify(entry.available) === JSON.stringify(content.available))) return undefined; + return { ...content, ...entry }; +} diff --git a/apps/review-desktop/code-oss/src/vs/review/browser/reviewRemoteHostsSettings.test.ts b/apps/review-desktop/code-oss/src/vs/review/browser/reviewRemoteHostsSettings.test.ts new file mode 100644 index 000000000..2c6cbf868 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/browser/reviewRemoteHostsSettings.test.ts @@ -0,0 +1,57 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from "node:assert/strict"; +import test from "node:test"; + +import { REVIEW_REMOTE_HOSTS_ENABLED_SETTING, REVIEW_REMOTE_HOSTS_SETTING } from "../common/reviewConfigurationDefaults.js"; +import { reviewRemoteHostsSettings } from "./reviewRemoteHostsSettings.js"; + +function fixture(values: Record) { + const settings = new Map(Object.entries(values)); + const writes: unknown[] = []; + const retried: string[] = []; + const uninstalled: string[] = []; + const hosts = reviewRemoteHostsSettings({ + get: (key) => settings.get(key), + update: async (key, value) => { + writes.push(value); + settings.set(key, value); + }, + connection: { + listSshAliases: async () => ["devbox"], + readRemoteHosts: async () => [{ alias: "devbox", state: "online" }], + retryRemoteHost: async (alias) => { retried.push(alias); }, + installRemoteHost: async (alias) => { retried.push(`install ${alias}`); }, + detectRemoteAgents: async () => null, + connectRemoteAgents: async () => [], + uninstallRemoteHost: async (alias) => { uninstalled.push(alias); }, + }, + }); + return { hosts, writes, retried, uninstalled }; +} + +test("reads the two settings as main does", () => { + const on = fixture({ [REVIEW_REMOTE_HOSTS_ENABLED_SETTING]: true, [REVIEW_REMOTE_HOSTS_SETTING]: ["devbox", 7] }).hosts; + assert.equal(on.enabled, true); + assert.deepEqual(on.configured, ["devbox"]); + const off = fixture({ [REVIEW_REMOTE_HOSTS_ENABLED_SETTING]: "true", [REVIEW_REMOTE_HOSTS_SETTING]: "devbox" }).hosts; + assert.equal(off.enabled, false); + assert.deepEqual(off.configured, []); +}); + +test("refuses an alias the SSH command would refuse, with the reason, and writes nothing", async () => { + for (const [alias, reason] of [["-bad", "starts with -"], ["a b", "contains whitespace"], ["a;b", "contains ;"], ["", "is empty"]]) { + const { hosts, writes } = fixture({ [REVIEW_REMOTE_HOSTS_SETTING]: ["devbox"] }); + await assert.rejects(hosts.set(["devbox", alias]), { message: `The SSH alias ${JSON.stringify(alias)} ${reason}.` }); + assert.deepEqual(writes, []); + } +}); + +test("writes the aliases once each and answers with the stored list", async () => { + const { hosts, writes } = fixture({ [REVIEW_REMOTE_HOSTS_SETTING]: [] }); + assert.deepEqual(await hosts.set(["devbox", "other", "devbox"]), ["devbox", "other"]); + assert.deepEqual(writes, [["devbox", "other"]]); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/browser/reviewRemoteHostsSettings.ts b/apps/review-desktop/code-oss/src/vs/review/browser/reviewRemoteHostsSettings.ts new file mode 100644 index 000000000..940e1f91c --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/browser/reviewRemoteHostsSettings.ts @@ -0,0 +1,36 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { REVIEW_REMOTE_HOSTS_ENABLED_SETTING, REVIEW_REMOTE_HOSTS_SETTING } from "../common/reviewConfigurationDefaults.js"; +import type { ReviewRemoteHostsSettings } from "../common/reviewProtocol.js"; +import { remoteHostAliases, validateSshAlias } from "../common/reviewSshAlias.js"; +import type { IReviewDesktopConnectionService } from "../services/reviewDesktopConnectionService.js"; + +export function reviewRemoteHostsSettings(input: { + get(key: string): unknown; + update(key: string, value: string[]): Promise; + connection: Pick; +}): ReviewRemoteHostsSettings { + const configured = () => remoteHostAliases(input.get(REVIEW_REMOTE_HOSTS_SETTING)); + return { + enabled: input.get(REVIEW_REMOTE_HOSTS_ENABLED_SETTING) === true, + configured: configured(), + suggestions: () => input.connection.listSshAliases(), + states: () => input.connection.readRemoteHosts(), + retry: (alias) => input.connection.retryRemoteHost(alias), + install: (alias) => input.connection.installRemoteHost(alias), + agents: (alias) => input.connection.detectRemoteAgents(alias), + connectAgents: (alias, agents) => input.connection.connectRemoteAgents(alias, agents), + uninstall: (alias) => input.connection.uninstallRemoteHost(alias), + set: async (aliases) => { + for (const alias of aliases) { + const valid = validateSshAlias(alias); + if (!valid.ok) throw new Error(`The SSH alias ${JSON.stringify(alias)} ${valid.reason}.`); + } + await input.update(REVIEW_REMOTE_HOSTS_SETTING, [...new Set(aliases)]); + return configured(); + }, + }; +} diff --git a/apps/review-desktop/code-oss/src/vs/review/common/reviewConfiguration.ts b/apps/review-desktop/code-oss/src/vs/review/common/reviewConfiguration.ts index 677444cc0..c605136c0 100644 --- a/apps/review-desktop/code-oss/src/vs/review/common/reviewConfiguration.ts +++ b/apps/review-desktop/code-oss/src/vs/review/common/reviewConfiguration.ts @@ -20,7 +20,7 @@ import { localize } from '../../nls.js'; import { Registry } from '../../platform/registry/common/platform.js'; import { ConfigurationScope, Extensions, type IConfigurationRegistry } from '../../platform/configuration/common/configurationRegistry.js'; -import { REVIEW_CTRL_TAB_CHOICES, REVIEW_CTRL_TAB_SETTING, REVIEW_DOCUMENT_WIDTH_CHOICES, REVIEW_DOCUMENT_WIDTH_SETTING, REVIEW_KEYMAPS, REVIEW_KEYMAP_SETTING, REVIEW_READY_NOTIFICATION_CHOICES, REVIEW_READY_NOTIFICATION_SETTING, REVIEW_SOFTWARE_MAP_SETTING, REVIEW_STRUCTURAL_DIFF_SETTING, REVIEW_TELEMETRY_SETTING, curatedExtensionConfigurationDefaults, reviewConfigurationDefaults } from './reviewConfigurationDefaults.js'; +import { REVIEW_CTRL_TAB_CHOICES, REVIEW_CTRL_TAB_SETTING, REVIEW_DOCUMENT_WIDTH_CHOICES, REVIEW_DOCUMENT_WIDTH_SETTING, REVIEW_KEYMAPS, REVIEW_KEYMAP_SETTING, REVIEW_READY_NOTIFICATION_CHOICES, REVIEW_READY_NOTIFICATION_SETTING, REVIEW_REMOTE_HOSTS_ENABLED_SETTING, REVIEW_REMOTE_HOSTS_SETTING, REVIEW_REMOTE_INSTALL_MODES, REVIEW_REMOTE_INSTALL_SETTING, REVIEW_SOFTWARE_MAP_SETTING, REVIEW_STRUCTURAL_DIFF_SETTING, REVIEW_TELEMETRY_SETTING, curatedExtensionConfigurationDefaults, reviewConfigurationDefaults } from './reviewConfigurationDefaults.js'; const configurationRegistry = Registry.as(Extensions.Configuration); @@ -83,6 +83,30 @@ configurationRegistry.registerConfiguration({ default: false, description: localize('review.experimental.softwareMap.enabled', "Show the experimental Software Map view in sessions."), }, + [REVIEW_REMOTE_HOSTS_SETTING]: { + type: 'array', + items: { type: 'string' }, + default: [], + scope: ConfigurationScope.APPLICATION, + description: localize('review.remote.hosts', "SSH host aliases, from your SSH configuration, whose Whiteboard reviews to show."), + }, + [REVIEW_REMOTE_HOSTS_ENABLED_SETTING]: { + type: 'boolean', + default: false, + description: localize('review.experimental.remoteHosts.enabled', "Connect over SSH to the hosts in `review.remote.hosts`."), + }, + [REVIEW_REMOTE_INSTALL_SETTING]: { + type: 'string', + enum: [...REVIEW_REMOTE_INSTALL_MODES], + enumDescriptions: [ + localize('review.remote.install.ask', "Ask once per host before installing."), + localize('review.remote.install.always', "Install without asking."), + localize('review.remote.install.never', "Never install; use the Whiteboard you installed on the host."), + ], + default: 'ask', + scope: ConfigurationScope.APPLICATION, + description: localize('review.remote.install', "Whether Whiteboard installs its own version on a remote host, under ~/.dev/whiteboard-remote, when the host does not have it."), + }, }, }); diff --git a/apps/review-desktop/code-oss/src/vs/review/common/reviewConfigurationDefaults.ts b/apps/review-desktop/code-oss/src/vs/review/common/reviewConfigurationDefaults.ts index 335abc440..78286bea6 100644 --- a/apps/review-desktop/code-oss/src/vs/review/common/reviewConfigurationDefaults.ts +++ b/apps/review-desktop/code-oss/src/vs/review/common/reviewConfigurationDefaults.ts @@ -26,6 +26,10 @@ export const REVIEW_KEYMAP_SETTING = 'review.keymap'; export const REVIEW_TELEMETRY_SETTING = 'review.telemetry.enabled'; export const REVIEW_STRUCTURAL_DIFF_SETTING = 'review.experimental.structuralDiff.enabled'; export const REVIEW_SOFTWARE_MAP_SETTING = 'review.experimental.softwareMap.enabled'; +export const REVIEW_REMOTE_HOSTS_SETTING = 'review.remote.hosts'; +export const REVIEW_REMOTE_HOSTS_ENABLED_SETTING = 'review.experimental.remoteHosts.enabled'; +export const REVIEW_REMOTE_INSTALL_SETTING = 'review.remote.install'; +export const REVIEW_REMOTE_INSTALL_MODES = ['ask', 'always', 'never'] as const; export const REVIEW_KEYMAPS = ['none', 'vim', 'emacs', 'sublime'] as const; export type ReviewKeymap = typeof REVIEW_KEYMAPS[number]; export const REVIEW_CTRL_TAB_SETTING = 'review.tabs.ctrlTab'; @@ -37,6 +41,9 @@ export const REVIEW_DOCUMENT_WIDTH_CHOICES = ['standard', 'wide', 'full'] as con export const reviewConfigurationDefaults = { [REVIEW_SOFTWARE_MAP_SETTING]: false, + [REVIEW_REMOTE_HOSTS_SETTING]: [], + [REVIEW_REMOTE_HOSTS_ENABLED_SETTING]: false, + [REVIEW_REMOTE_INSTALL_SETTING]: 'ask', [REVIEW_STRUCTURAL_DIFF_SETTING]: true, [REVIEW_TELEMETRY_SETTING]: true, 'telemetry.telemetryLevel': 'off', @@ -145,6 +152,11 @@ export const curatedExtensionConfigurationDefaults = { // import Sublime Text settings. It has no setting to turn that off. } as const; +/** Every Source window is read-only; see `reviewSourceWindowConfiguration`. */ +export const reviewSourceWindowDefaults = { + 'files.readonlyInclude': { '**/*': true }, +} as const; + export const reviewAgentsWindowDefaultOverrides = { 'window.title': reviewConfigurationDefaults['window.title'], 'workbench.navigationControl.enabled': reviewConfigurationDefaults['workbench.navigationControl.enabled'], diff --git a/apps/review-desktop/code-oss/src/vs/review/common/reviewRemoteInstallPrompt.ts b/apps/review-desktop/code-oss/src/vs/review/common/reviewRemoteInstallPrompt.ts new file mode 100644 index 000000000..f2ce7b181 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/common/reviewRemoteInstallPrompt.ts @@ -0,0 +1,10 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +export const REVIEW_REMOTE_INSTALL_PROMPT_EVENT = "onRemoteInstallPrompt"; +export const REVIEW_REMOTE_INSTALL_ANSWER_CALL = "answerRemoteInstallPrompt"; + +export const REVIEW_REMOTE_INSTALL_YES = "install"; +export const REVIEW_REMOTE_INSTALL_NO = "decline"; diff --git a/apps/review-desktop/code-oss/src/vs/review/common/reviewSshAlias.ts b/apps/review-desktop/code-oss/src/vs/review/common/reviewSshAlias.ts new file mode 100644 index 000000000..a906f0abb --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/common/reviewSshAlias.ts @@ -0,0 +1,18 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +export function validateSshAlias(alias: string): { ok: true } | { ok: false; reason: string } { + if (!alias) return { ok: false, reason: "is empty" }; + if (alias.startsWith("-")) return { ok: false, reason: "starts with -" }; + if (/\s/.test(alias)) return { ok: false, reason: "contains whitespace" }; + if (/[\x00-\x1f\x7f-\x9f]/.test(alias)) return { ok: false, reason: "contains a control character" }; + const meta = /[`$;|&<>()'"\\]/.exec(alias); + if (meta) return { ok: false, reason: `contains ${meta[0]}` }; + return { ok: true }; +} + +export function remoteHostAliases(value: unknown): string[] { + return Array.isArray(value) ? value.filter((alias): alias is string => typeof alias === "string") : []; +} diff --git a/apps/review-desktop/code-oss/src/vs/review/common/reviewSshPrompt.ts b/apps/review-desktop/code-oss/src/vs/review/common/reviewSshPrompt.ts new file mode 100644 index 000000000..6fb30527e --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/common/reviewSshPrompt.ts @@ -0,0 +1,22 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +export type SshPromptKind = "secret" | "confirm" | "text"; + +export type ReviewSshPromptEvent = + | { readonly id: number; readonly alias: string; readonly text: string; readonly kind: SshPromptKind } + | { readonly id: number; readonly closed: true }; + +export const REVIEW_SSH_PROMPT_EVENT = "onSshPrompt"; +export const REVIEW_SSH_ANSWER_CALL = "answerSshPrompt"; + +export const REVIEW_SSH_ASKPASS_SOCKET_ENV = "DEV_FAST_REVIEW_SSH_ASKPASS_SOCKET"; +export const REVIEW_SSH_ASKPASS_ALIAS_ENV = "DEV_FAST_REVIEW_SSH_ASKPASS_ALIAS"; + +export function sshPromptKind(text: string): SshPromptKind { + if (/yes\/no/i.test(text)) return "confirm"; + if (/passphrase|password/i.test(text)) return "secret"; + return "text"; +} diff --git a/apps/review-desktop/code-oss/src/vs/review/contrib/remote/reviewRemoteInstallPrompt.contribution.ts b/apps/review-desktop/code-oss/src/vs/review/contrib/remote/reviewRemoteInstallPrompt.contribution.ts new file mode 100644 index 000000000..819ed5bfb --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/contrib/remote/reviewRemoteInstallPrompt.contribution.ts @@ -0,0 +1,71 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Disposable } from "../../../base/common/lifecycle.js"; +import { localize } from "../../../nls.js"; +import { IMainProcessService } from "../../../platform/ipc/common/mainProcessService.js"; +import { IQuickInputService, type IQuickInput } from "../../../platform/quickinput/common/quickInput.js"; +import { registerWorkbenchContribution2, WorkbenchPhase } from "../../../workbench/common/contributions.js"; +import { REVIEW_DESKTOP_CHANNEL } from "../../common/reviewDesktopBootstrap.js"; +import { REVIEW_REMOTE_INSTALL_ANSWER_CALL, REVIEW_REMOTE_INSTALL_PROMPT_EVENT } from "../../common/reviewRemoteInstallPrompt.js"; +import type { ReviewSshPromptEvent } from "../../common/reviewSshPrompt.js"; +import { createRemoteInstallPromptInput } from "./reviewRemoteInstallPromptInput.js"; +import type { ShownSshPrompt } from "./reviewSshPromptInput.js"; + +class ReviewRemoteInstallPrompts extends Disposable { + static readonly ID = "review.remoteInstallPrompts"; + + private current: { id: number; input: IQuickInput } | undefined; + private closing = false; + + constructor( + @IMainProcessService private readonly mainProcessService: IMainProcessService, + @IQuickInputService private readonly quickInputService: IQuickInputService, + ) { + super(); + const channel = this.mainProcessService.getChannel(REVIEW_DESKTOP_CHANNEL); + this._register( + channel.listen(REVIEW_REMOTE_INSTALL_PROMPT_EVENT)((event) => { + if ("closed" in event) { + if (this.current?.id === event.id) this.current.input.dispose(); + } else if (this.current?.id !== event.id) this.show(event); + }), + ); + this._register({ + dispose: () => { + this.closing = true; + this.current?.input.dispose(); + }, + }); + } + + private show(prompt: ShownSshPrompt): void { + this.current?.input.dispose(); + let settled = false; + const settle = (answer: string | undefined) => { + if (settled) return; + settled = true; + if (this.current?.id === prompt.id) this.current = undefined; + if (!this.closing) void this.mainProcessService.getChannel(REVIEW_DESKTOP_CHANNEL).call(REVIEW_REMOTE_INSTALL_ANSWER_CALL, { id: prompt.id, answer }); + input.dispose(); + }; + + const input = createRemoteInstallPromptInput( + this.quickInputService, + prompt, + localize("review.remoteInstallPrompt.title", "Install Whiteboard on {0}?", prompt.alias), + { + install: localize("review.remoteInstallPrompt.install", "Install"), + decline: localize("review.remoteInstallPrompt.decline", "Don't install"), + }, + settle, + ); + input.onDidHide(() => settle(undefined)); + this.current = { id: prompt.id, input }; + input.show(); + } +} + +registerWorkbenchContribution2(ReviewRemoteInstallPrompts.ID, ReviewRemoteInstallPrompts, WorkbenchPhase.AfterRestored); diff --git a/apps/review-desktop/code-oss/src/vs/review/contrib/remote/reviewRemoteInstallPromptInput.ts b/apps/review-desktop/code-oss/src/vs/review/contrib/remote/reviewRemoteInstallPromptInput.ts new file mode 100644 index 000000000..d395ab194 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/contrib/remote/reviewRemoteInstallPromptInput.ts @@ -0,0 +1,27 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import type { IQuickInputService, IQuickPick, IQuickPickItem } from "../../../platform/quickinput/common/quickInput.js"; +import { REVIEW_REMOTE_INSTALL_NO, REVIEW_REMOTE_INSTALL_YES } from "../../common/reviewRemoteInstallPrompt.js"; +import type { ShownSshPrompt } from "./reviewSshPromptInput.js"; + +export function createRemoteInstallPromptInput( + quickInputService: Pick, + prompt: ShownSshPrompt, + title: string, + labels: { install: string; decline: string }, + accept: (answer: string | undefined) => void, +): IQuickPick { + const pick = quickInputService.createQuickPick(); + pick.items = [ + { label: labels.install, answer: REVIEW_REMOTE_INSTALL_YES }, + { label: labels.decline, answer: REVIEW_REMOTE_INSTALL_NO }, + ]; + pick.onDidAccept(() => accept(pick.selectedItems[0]?.answer)); + pick.title = title; + pick.description = prompt.text; + pick.ignoreFocusOut = true; + return pick; +} diff --git a/apps/review-desktop/code-oss/src/vs/review/contrib/remote/reviewSshPrompt.contribution.ts b/apps/review-desktop/code-oss/src/vs/review/contrib/remote/reviewSshPrompt.contribution.ts new file mode 100644 index 000000000..0ccdb8cf2 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/contrib/remote/reviewSshPrompt.contribution.ts @@ -0,0 +1,65 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Disposable } from "../../../base/common/lifecycle.js"; +import { localize } from "../../../nls.js"; +import { IMainProcessService } from "../../../platform/ipc/common/mainProcessService.js"; +import { IQuickInputService, type IQuickInput } from "../../../platform/quickinput/common/quickInput.js"; +import { registerWorkbenchContribution2, WorkbenchPhase } from "../../../workbench/common/contributions.js"; +import { REVIEW_DESKTOP_CHANNEL } from "../../common/reviewDesktopBootstrap.js"; +import { REVIEW_SSH_ANSWER_CALL, REVIEW_SSH_PROMPT_EVENT, type ReviewSshPromptEvent } from "../../common/reviewSshPrompt.js"; +import { createSshPromptInput, type ShownSshPrompt } from "./reviewSshPromptInput.js"; + +class ReviewSshPrompts extends Disposable { + static readonly ID = "review.sshPrompts"; + + private current: { id: number; input: IQuickInput } | undefined; + private closing = false; + + constructor( + @IMainProcessService private readonly mainProcessService: IMainProcessService, + @IQuickInputService private readonly quickInputService: IQuickInputService, + ) { + super(); + const channel = this.mainProcessService.getChannel(REVIEW_DESKTOP_CHANNEL); + this._register( + channel.listen(REVIEW_SSH_PROMPT_EVENT)((event) => { + if ("closed" in event) { + if (this.current?.id === event.id) this.current.input.dispose(); + } else if (this.current?.id !== event.id) this.show(event); + }), + ); + this._register({ + dispose: () => { + this.closing = true; + this.current?.input.dispose(); + }, + }); + } + + private show(prompt: ShownSshPrompt): void { + this.current?.input.dispose(); + let settled = false; + const settle = (answer: string | undefined) => { + if (settled) return; + settled = true; + if (this.current?.id === prompt.id) this.current = undefined; + if (!this.closing) void this.mainProcessService.getChannel(REVIEW_DESKTOP_CHANNEL).call(REVIEW_SSH_ANSWER_CALL, { id: prompt.id, answer }); + input.dispose(); + }; + + const input = createSshPromptInput( + this.quickInputService, + prompt, + localize("review.sshPrompt.title", "SSH: {0}", prompt.alias), + settle, + ); + input.onDidHide(() => settle(undefined)); + this.current = { id: prompt.id, input }; + input.show(); + } +} + +registerWorkbenchContribution2(ReviewSshPrompts.ID, ReviewSshPrompts, WorkbenchPhase.AfterRestored); diff --git a/apps/review-desktop/code-oss/src/vs/review/contrib/remote/reviewSshPromptInput.test.ts b/apps/review-desktop/code-oss/src/vs/review/contrib/remote/reviewSshPromptInput.test.ts new file mode 100644 index 000000000..dee65627f --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/contrib/remote/reviewSshPromptInput.test.ts @@ -0,0 +1,61 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from "node:assert/strict"; +import test from "node:test"; + +import { createSshPromptInput } from "./reviewSshPromptInput.js"; + +function fakeInput() { + const handlers: Record void> = {}; + const on = (name: string) => (handler: () => void) => { + handlers[name] = handler; + return { dispose() {} }; + }; + return { + handlers, + value: "", + selectedItems: [] as { label: string }[], + onDidAccept: on("accept"), + onWillHide: on("willHide"), + } as Record & { handlers: Record void>; value: string; selectedItems: { label: string }[] }; +} + +function create(kind: "secret" | "confirm" | "text", text: string) { + const input = fakeInput(); + const answers: (string | undefined)[] = []; + const service = { createInputBox: () => input, createQuickPick: () => input } as never; + createSshPromptInput(service, { id: 1, alias: "wb-test-a", text, kind }, "SSH: wb-test-a", (answer) => answers.push(answer)); + return { input, answers }; +} + +test("ssh's text is shown as plain description, never as the prompt that renders links", () => { + const text = "[x](command:workbench.action.terminal.new) $(zap) Verification code: "; + for (const kind of ["secret", "confirm", "text"] as const) { + const { input } = create(kind, text); + assert.equal(input.description, text); + assert.equal(input.prompt, undefined); + assert.equal(input.title, "SSH: wb-test-a"); + } +}); + +test("a secret is hidden, answered with what was typed, and cleared before the widget hides", () => { + const { input, answers } = create("secret", "dev@127.0.0.1's password: "); + assert.equal(input.password, true); + + input.value = "typed"; + input.handlers.accept(); + input.handlers.willHide(); + + assert.deepEqual(answers, ["typed"]); + assert.equal(input.value, ""); +}); + +test("a confirm prompt answers with the picked yes or no", () => { + const { input, answers } = create("confirm", "Are you sure you want to continue connecting (yes/no/[fingerprint])? "); + input.selectedItems = [{ label: "no" }]; + input.handlers.accept(); + assert.deepEqual(answers, ["no"]); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/contrib/remote/reviewSshPromptInput.ts b/apps/review-desktop/code-oss/src/vs/review/contrib/remote/reviewSshPromptInput.ts new file mode 100644 index 000000000..abfdc3e48 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/contrib/remote/reviewSshPromptInput.ts @@ -0,0 +1,34 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import type { IQuickInput, IQuickInputService, IQuickPickItem } from "../../../platform/quickinput/common/quickInput.js"; +import type { ReviewSshPromptEvent } from "../../common/reviewSshPrompt.js"; + +export type ShownSshPrompt = Exclude; + +export function createSshPromptInput( + quickInputService: Pick, + prompt: ShownSshPrompt, + title: string, + accept: (answer: string | undefined) => void, +): IQuickInput { + let input: IQuickInput; + if (prompt.kind === "confirm") { + const pick = quickInputService.createQuickPick(); + pick.items = [{ label: "yes" }, { label: "no" }]; + pick.onDidAccept(() => accept(pick.selectedItems[0]?.label)); + input = pick; + } else { + const box = quickInputService.createInputBox(); + box.password = prompt.kind === "secret"; + box.onDidAccept(() => accept(box.value)); + box.onWillHide(() => (box.value = "")); + input = box; + } + input.title = title; + input.description = prompt.text; + input.ignoreFocusOut = true; + return input; +} diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewEnabledExtensionGroups.test.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewEnabledExtensionGroups.test.ts new file mode 100644 index 000000000..6c46464a5 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewEnabledExtensionGroups.test.ts @@ -0,0 +1,40 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from "node:assert/strict"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { reviewEnabledExtensionGroups } from "./reviewEnabledExtensionGroups.js"; + +const entry = (id: string, version: string) => ({ identifier: { id }, version, relativeLocation: `${id.toLowerCase()}-${version}` }); + +test("a group counts once its primary extension is installed and not being removed", async (t) => { + const dir = await mkdtemp(join(tmpdir(), "wb-groups-")); + t.after(() => rm(dir, { recursive: true, force: true })); + await writeFile( + join(dir, "extensions.json"), + JSON.stringify([ + entry("golang.Go", "0.56.0"), + entry("rust-lang.rust-analyzer", "0.4.2990"), + entry("llvm-vs-code-extensions.lldb-dap", "0.7.20260804"), + entry("ms-python.python", "2026.4.0"), + ]), + ); + await writeFile(join(dir, ".obsolete"), JSON.stringify({ "rust-lang.rust-analyzer-0.4.2990": true })); + + assert.deepEqual(await reviewEnabledExtensionGroups(dir), ["go"]); +}); + +test("no list, or an unreadable one, enables nothing", async (t) => { + const dir = await mkdtemp(join(tmpdir(), "wb-groups-")); + t.after(() => rm(dir, { recursive: true, force: true })); + + assert.deepEqual(await reviewEnabledExtensionGroups(dir), []); + await writeFile(join(dir, "extensions.json"), "{"); + assert.deepEqual(await reviewEnabledExtensionGroups(dir), []); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewEnabledExtensionGroups.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewEnabledExtensionGroups.ts new file mode 100644 index 000000000..a8c04ce42 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewEnabledExtensionGroups.ts @@ -0,0 +1,29 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { reviewOptionalExtensionCatalog } from "../../node/reviewOptionalExtensionCatalog.js"; + +export async function reviewEnabledExtensionGroups(extensionsPath: string): Promise { + const read = (name: string) => readFile(join(extensionsPath, name), "utf8").then(JSON.parse).catch(() => undefined) as Promise; + const [list, obsolete] = await Promise.all([read("extensions.json"), read(".obsolete")]); + if (!Array.isArray(list)) return []; + const removed = obsolete && typeof obsolete === "object" ? (obsolete as Record) : {}; + const installed = new Set(); + for (const entry of list as { identifier?: { id?: unknown }; relativeLocation?: unknown }[]) { + const id = entry?.identifier?.id; + if (typeof id !== "string") continue; + if (typeof entry.relativeLocation === "string" && removed[entry.relativeLocation] === true) continue; + installed.add(id.toLowerCase()); + } + return [ + ...new Set( + reviewOptionalExtensionCatalog + .filter((extension) => extension.role === "primary" && installed.has(extension.id.toLowerCase())) + .map((extension) => extension.group), + ), + ]; +} diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteAgents.test.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteAgents.test.ts new file mode 100644 index 000000000..b98ea8973 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteAgents.test.ts @@ -0,0 +1,93 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { chmod, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { parseRemoteAgents, parseRemoteConnect, remoteConnectScript } from "./reviewRemoteAgents.js"; + +test("detection keeps only known agents, once each, and nothing the remote adds", () => { + const stdout = [ + "Welcome to Ubuntu", + JSON.stringify({ event: "connect.detect", agents: [] }), + JSON.stringify({ + event: "connect.detect", + agents: [ + { id: "pi", name: "\u001b[31mEvil\u001b[0m", connected: false, manual: "yes", configPath: "/home/dev/.pi" }, + { id: "claude", name: "Claude Code", connected: true }, + { id: "pi", connected: true }, + { id: "cursor", connected: false }, + { id: "codex", connected: "no" }, + { id: "__proto__", connected: false }, + "codex", + null, + ], + }), + "bye", + ].join("\n"); + + assert.deepEqual(parseRemoteAgents(stdout), [ + { id: "claude", connected: true }, + { id: "pi", connected: false }, + ]); + assert.deepEqual(parseRemoteAgents(JSON.stringify({ event: "connect.detect", agents: [{ id: "codex", connected: false, manual: true }] })), [ + { id: "codex", connected: false, manual: true }, + ]); + assert.equal(parseRemoteAgents("bash: whiteboard: command not found\n"), undefined); + assert.equal(parseRemoteAgents(JSON.stringify({ event: "connect.detect", agents: "all" })), undefined); + const many = Array.from({ length: 10_000 }, () => ({ id: "codex", connected: false })); + assert.deepEqual(parseRemoteAgents(JSON.stringify({ event: "connect.detect", agents: [...many, { id: "pi", connected: true }] })), [ + { id: "codex", connected: false }, + ]); +}); + +test("a connect result's output is plain text, and bounded", () => { + const [result] = parseRemoteConnect( + JSON.stringify({ event: "connect.run", agents: [{ id: "pi", connected: false, output: `\u001b[31mnpm ERR!\u001b[0m\r\n404\u0007 ${"x".repeat(5000)}` }] }), + )!; + + assert.equal(result.id, "pi"); + assert.equal(result.connected, false); + assert.ok(result.output.length <= 1000); + assert.doesNotMatch(result.output, /[\x00-\x1f\x7f-\x9f]/); + assert.equal(parseRemoteConnect(JSON.stringify({ event: "connect.run", agents: [{ id: "pi", connected: true }] }))![0].output, ""); +}); + +async function runScript(t: test.TestContext, script: (home: string) => string, shellNoise = "motd\n"): Promise { + const home = await mkdtemp(join(tmpdir(), "wb-agents-")); + t.after(() => rm(home, { recursive: true, force: true })); + const echo = '#!/bin/sh\nprintf "%s|" "$0" "$@"; printf "\\nPATH=%s DELEGATE=%s\\n" "$PATH" "$DEV_FAST_REVIEW_CLI_NO_DELEGATE"\n'; + await mkdir(join(home, ".local", "bin"), { recursive: true }); + for (const file of [join(home, "node"), join(home, ".local", "bin", "whiteboard")]) { + await writeFile(file, echo); + await chmod(file, 0o755); + } + const shell = join(home, "login-shell"); + await writeFile(shell, `#!/bin/sh\nprintf '${shellNoise}'\nPATH=/agents/bin:$PATH\nexport PATH\nshift\nexec /bin/sh -c "$1"\n`); + await chmod(shell, 0o755); + return new Promise((resolve, reject) => { + const child = execFile("/bin/sh", ["-s"], { env: { HOME: home, SHELL: shell, PATH: "/usr/bin:/bin" } }, (error, stdout) => (error ? reject(error) : resolve(stdout))); + child.stdin!.end(script(home)); + }); +} + +test("the installed CLI runs by its paths, with the login shell's PATH, as itself", async (t) => { + const out = await runScript(t, (home) => remoteConnectScript({ nodePath: join(home, "node"), cliPath: "/opt/wb/it's/cli.js" }, ["--yes", "--json", "pi"]), "Welcome\nWHITEBOARD-PATH=/not/this\n"); + + const [argv, env] = out.trim().split("\n"); + assert.match(argv, /\/node\|\/opt\/wb\/it's\/cli\.js\|connect\|--yes\|--json\|pi\|$/); + assert.match(env, /^PATH=\/agents\/bin:\/usr\/bin:\/bin:\/usr\/bin:\/bin DELEGATE=1$/); +}); + +test("without an installed CLI, the one on PATH or in ~/.local/bin runs", async (t) => { + const out = await runScript(t, () => remoteConnectScript(undefined, ["--detect", "--json"])); + + assert.match(out, /\/\.local\/bin\/whiteboard\|connect\|--detect\|--json\|/); + assert.match(out, /DELEGATE=1/); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteAgents.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteAgents.ts new file mode 100644 index 000000000..0060f7808 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteAgents.ts @@ -0,0 +1,71 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { stripVTControlCharacters } from "node:util"; +import { REVIEW_REMOTE_AGENT_IDS, type ReviewRemoteAgent, type ReviewRemoteAgentId, type ReviewRemoteAgentResult } from "../../common/reviewProtocol.js"; +import { pathCliScript } from "./reviewRemoteAttachScript.js"; +import { shellQuote } from "./reviewRemoteInstallScript.js"; + +export type ReviewRemoteCli = { readonly nodePath: string; readonly cliPath: string } | undefined; + +const LOGIN_PATH = `if [ -n "$SHELL" ]; then + p=$("$SHELL" -lic 'printf "\\nWHITEBOARD-PATH=%s\\n" "$PATH"' /dev/null | tr -d '\\r' | sed -n 's/^WHITEBOARD-PATH=//p' | tail -n 1) + case "$p" in /*) PATH="$p:$PATH"; export PATH ;; esac +fi +DEV_FAST_REVIEW_CLI_NO_DELEGATE=1 +export DEV_FAST_REVIEW_CLI_NO_DELEGATE +`; + +export function remoteConnectScript(cli: ReviewRemoteCli, args: readonly string[]): string { + const words = ["connect", ...args].map((word) => shellQuote(word)).join(" "); + return `${LOGIN_PATH}${cli ? `exec ${shellQuote(cli.nodePath)} ${shellQuote(cli.cliPath)} ${words}\n` : pathCliScript(words)}`; +} + +export const plainText = (text: string) => stripVTControlCharacters(text).replace(/[\x00-\x1f\x7f-\x9f]+/g, " ").trim(); + +const MAX_AGENTS = 16; +const OUTPUT_LIMIT = 1000; + +export const isRemoteAgentId = (value: unknown): value is ReviewRemoteAgentId => REVIEW_REMOTE_AGENT_IDS.some((id) => id === value); + +function lastEvent(stdout: string, event: string): Record | undefined { + for (const line of stdout.split("\n").reverse()) { + let value: unknown; + try { + value = JSON.parse(line); + } catch { + continue; + } + if (value && typeof value === "object" && (value as Record).event === event) return value as Record; + } + return undefined; +} + +function agentsOf(record: Record | undefined, read: (item: Record) => T | undefined): T[] | undefined { + if (!record || !Array.isArray(record.agents)) return undefined; + const found = new Map(); + for (const item of record.agents.slice(0, MAX_AGENTS)) { + if (!item || typeof item !== "object" || !isRemoteAgentId((item as Record).id)) continue; + const agent = read(item as Record); + if (agent && !found.has(agent.id)) found.set(agent.id, agent); + } + return REVIEW_REMOTE_AGENT_IDS.flatMap((id) => found.get(id) ?? []); +} + +export function parseRemoteAgents(stdout: string): ReviewRemoteAgent[] | undefined { + return agentsOf(lastEvent(stdout, "connect.detect"), (item) => + typeof item.connected === "boolean" + ? { id: item.id as ReviewRemoteAgentId, connected: item.connected, ...(item.manual === true && { manual: true as const }) } + : undefined, + ); +} + +export function parseRemoteConnect(stdout: string): ReviewRemoteAgentResult[] | undefined { + return agentsOf(lastEvent(stdout, "connect.run"), (item) => + typeof item.connected === "boolean" + ? { id: item.id as ReviewRemoteAgentId, connected: item.connected, output: plainText(typeof item.output === "string" ? item.output : "").slice(-OUTPUT_LIMIT) } + : undefined, + ); +} diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteArtifacts.test.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteArtifacts.test.ts new file mode 100644 index 000000000..4707cc7dc --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteArtifacts.test.ts @@ -0,0 +1,148 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { after, test } from "node:test"; + +import type { IWhiteboardRemoteProduct } from "../../../base/common/product.js"; +import { fetchToLaptopCache, remoteArtifacts, remotePackageIntegrity } from "./reviewRemoteArtifacts.js"; + +const roots: string[] = []; +const servers: Server[] = []; + +after(async () => { + for (const server of servers) server.close(); + await Promise.all(roots.map((root) => rm(root, { recursive: true, force: true }))); +}); + +async function temporary(prefix: string): Promise { + const root = await mkdtemp(join(tmpdir(), prefix)); + roots.push(root); + return root; +} + +const sha256 = (bytes: Buffer | string) => createHash("sha256").update(bytes).digest("hex"); + +async function serve(files: Record) { + const requests: string[] = []; + const server = createServer((request, response) => { + requests.push(request.url ?? ""); + const body = files[request.url ?? ""]; + response.writeHead(body === undefined ? 404 : 200); + response.end(body); + }); + servers.push(server); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + return { base: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, requests }; +} + +test("a download is verified before it is cached, and verified again on each use", async () => { + const cacheDirectory = await temporary("wb-artifacts-cache-"); + const body = Buffer.from("node tarball bytes"); + const { base, requests } = await serve({ "/node.tar.xz": body }); + const artifact = { name: "node.tar.xz", url: `${base}/node.tar.xz`, sha256: sha256(body) }; + + const first = await fetchToLaptopCache(artifact, { cacheDirectory }); + const second = await fetchToLaptopCache(artifact, { cacheDirectory }); + + assert.equal(first, second); + assert.ok(first.startsWith(cacheDirectory)); + assert.deepEqual(await readFile(first), body); + assert.deepEqual(requests, ["/node.tar.xz"]); + + await writeFile(first, "tampered"); + await assert.rejects(fetchToLaptopCache(artifact, { cacheDirectory }), /checksum/); + assert.deepEqual(await readdir(cacheDirectory), []); +}); + +test("a file whose checksum is wrong is refused and not kept", async () => { + const cacheDirectory = await temporary("wb-artifacts-cache-"); + const { base } = await serve({ "/package.tgz": "not the pinned bytes" }); + + await assert.rejects( + fetchToLaptopCache({ name: "package.tgz", url: `${base}/package.tgz`, integrity: `sha512-${createHash("sha512").update("pinned").digest("base64")}` }, { cacheDirectory }), + /checksum/, + ); + await assert.rejects(fetchToLaptopCache({ name: "x", url: `${base}/missing` , sha256: sha256("x") }, { cacheDirectory }), /404/); + await assert.rejects(fetchToLaptopCache({ name: "x", url: `${base}/package.tgz` }, { cacheDirectory }), /no checksum/); + assert.deepEqual(await readdir(cacheDirectory), []); +}); + +const PIN: IWhiteboardRemoteProduct = { + package: { name: "@dev.fast/whiteboard", version: "0.1.6", integrity: `sha512-${"A".repeat(86)}==` }, + node: { + version: "24.18.0", + "linux-x64": { url: "https://nodejs.org/dist/v24.18.0/node-v24.18.0-linux-x64.tar.xz", sha256: "a".repeat(64) }, + "linux-arm64": { url: "https://nodejs.org/dist/v24.18.0/node-v24.18.0-linux-arm64.tar.xz", sha256: "b".repeat(64) }, + }, +}; + +test("a release build installs what it pinned", async () => { + const cacheDirectory = await temporary("wb-artifacts-cache-"); + + assert.deepEqual(await remoteArtifacts("linux-arm64", { pin: PIN, cacheDirectory }), { + package: { + name: "dev.fast-whiteboard-0.1.6.tgz", + url: "https://registry.npmjs.org/@dev.fast/whiteboard/-/whiteboard-0.1.6.tgz", + integrity: PIN.package.integrity, + }, + node: { name: "node-v24.18.0-linux-arm64.tar.xz", url: PIN.node["linux-arm64"].url, sha256: "b".repeat(64) }, + }); + await assert.rejects(remoteArtifacts("linux-x64", { pin: undefined, cacheDirectory }), /no pinned remote package/); + assert.equal(await remotePackageIntegrity({ pin: PIN, cacheDirectory }), PIN.package.integrity); +}); + +async function checkout(counter: string) { + const root = await temporary("wb-artifacts-checkout-"); + const run = (...args: string[]) => execFileSync("git", args, { cwd: root, stdio: "ignore" }); + await mkdir(join(root, "packages/review"), { recursive: true }); + await mkdir(join(root, "apps/review-desktop/code-oss"), { recursive: true }); + await writeFile(join(root, "apps/review-desktop/code-oss/.nvmrc"), "24.18.0\n"); + await writeFile( + join(root, "packages/review/package.json"), + JSON.stringify({ name: "@dev.fast/whiteboard", version: "0.0.1", files: ["index.js"], scripts: { prepack: `echo x >> '${counter}'` } }), + ); + await writeFile(join(root, "packages/review/index.js"), "export {};\n"); + run("init", "-b", "main"); + run("add", "."); + run("-c", "user.email=t@example.com", "-c", "user.name=t", "commit", "-m", "fixture"); + return root; +} + +test("a development build packs its checkout once per state and fetches Node's checksums once", async () => { + const counter = join(await temporary("wb-artifacts-packs-"), "packs.txt"); + const root = await checkout(counter); + const cacheDirectory = await temporary("wb-artifacts-cache-"); + const shasums = [`${"c".repeat(64)} node-v24.18.0-linux-x64.tar.xz`, `${"d".repeat(64)} node-v24.18.0-linux-arm64.tar.xz`, ""].join("\n"); + const { base, requests } = await serve({ "/dist/v24.18.0/SHASUMS256.txt": shasums }); + const options = { pin: undefined, checkout: root, cacheDirectory, nodeDist: `${base}/dist` }; + const packs = async () => (await readFile(counter, "utf8")).split("\n").filter(Boolean).length; + + const first = await remoteArtifacts("linux-x64", options); + const again = await remoteArtifacts("linux-x64", options); + + assert.deepEqual(again, first); + assert.equal(await remotePackageIntegrity(options), first.package.integrity); + assert.equal(await packs(), 1); + assert.equal(first.package.name, "dev.fast-whiteboard-0.0.1.tgz"); + const packed = await fetchToLaptopCache(first.package, { cacheDirectory }); + assert.equal(first.package.integrity, `sha512-${createHash("sha512").update(await readFile(packed)).digest("base64")}`); + assert.deepEqual(first.node, { name: "node-v24.18.0-linux-x64.tar.xz", url: `${base}/dist/v24.18.0/node-v24.18.0-linux-x64.tar.xz`, sha256: "c".repeat(64) }); + assert.equal((await remoteArtifacts("linux-arm64", options)).node.sha256, "d".repeat(64)); + assert.deepEqual(requests, ["/dist/v24.18.0/SHASUMS256.txt"]); + + await writeFile(join(root, "packages/review/index.js"), "export const changed = 1;\n"); + const edited = await remoteArtifacts("linux-x64", options); + + assert.equal(await packs(), 2); + assert.notEqual(edited.package.integrity, first.package.integrity); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteArtifacts.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteArtifacts.ts new file mode 100644 index 000000000..e830dab24 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteArtifacts.ts @@ -0,0 +1,199 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { spawn } from "node:child_process"; +import { createHash, randomBytes } from "node:crypto"; +import { createReadStream, createWriteStream } from "node:fs"; +import { mkdir, mkdtemp, readdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { pipeline } from "node:stream/promises"; +import { pathToFileURL } from "node:url"; + +import type { IWhiteboardRemoteProduct } from "../../../base/common/product.js"; +import type { ReviewRemoteTarget } from "./reviewRemoteProbe.js"; + +export interface ReviewRemoteArtifact { + name: string; + url: string; + sha256?: string; + integrity?: string; +} + +export interface ReviewRemoteArtifactsOptions { + readonly pin: IWhiteboardRemoteProduct | undefined; + readonly checkout?: string; + readonly cacheDirectory: string; + readonly nodeDist?: string; +} + +export const REVIEW_REMOTE_ARTIFACT_TIMEOUTS = { + download: 10 * 60_000, + pack: 5 * 60_000, + git: 30_000, +}; + +export const reviewRemoteCacheDirectory = (userDataPath: string) => join(userDataPath, "whiteboard-remote-cache"); + +const REGISTRY = "https://registry.npmjs.org"; +const NODE_DIST = "https://nodejs.org/dist"; + +export async function remoteArtifacts( + target: ReviewRemoteTarget, + options: ReviewRemoteArtifactsOptions, +): Promise<{ package: ReviewRemoteArtifact; node: ReviewRemoteArtifact }> { + const { pin } = options; + if (pin) { + const { name, version, integrity } = pin.package; + const node = pin.node[target]; + return { + package: { name: tarballName(name, version), url: `${REGISTRY}/${name}/-/${name.split("/").at(-1)}-${version}.tgz`, integrity }, + node: { name: node.url.split("/").at(-1)!, url: node.url, sha256: node.sha256 }, + }; + } + if (!options.checkout) throw new Error("This build has no pinned remote package and no checkout to pack one from."); + await mkdir(options.cacheDirectory, { recursive: true }); + return { + package: await packCheckout(options.checkout, options.cacheDirectory), + node: await developmentNode(target, options.checkout, options.cacheDirectory, options.nodeDist ?? NODE_DIST), + }; +} + +export async function remotePackageIntegrity(options: ReviewRemoteArtifactsOptions): Promise { + if (options.pin) return options.pin.package.integrity; + if (!options.checkout) throw new Error("This build has no pinned remote package and no checkout to pack one from."); + await mkdir(options.cacheDirectory, { recursive: true }); + const { integrity } = await packCheckout(options.checkout, options.cacheDirectory); + return integrity!; +} + +const tarballName = (name: string, version: string) => `${name.replace(/^@/, "").replace("/", "-")}-${version}.tgz`; + +function cachePath(cacheDirectory: string, artifact: ReviewRemoteArtifact): string { + if (artifact.integrity) { + const match = /^sha512-([A-Za-z0-9+/]{86}==)$/.exec(artifact.integrity); + if (!match) throw new Error(`${artifact.name} has a malformed integrity.`); + return join(cacheDirectory, `sha512-${Buffer.from(match[1], "base64").toString("hex")}`); + } + if (artifact.sha256 && /^[0-9a-f]{64}$/.test(artifact.sha256)) return join(cacheDirectory, `sha256-${artifact.sha256}`); + throw new Error(`${artifact.name} has no checksum to verify it by.`); +} + +async function digest(file: string, algorithm: "sha256" | "sha512", encoding: "hex" | "base64"): Promise { + const hash = createHash(algorithm); + for await (const chunk of createReadStream(file)) hash.update(chunk as Buffer); + return hash.digest(encoding); +} + +async function matches(file: string, artifact: ReviewRemoteArtifact): Promise { + if (artifact.integrity && `sha512-${await digest(file, "sha512", "base64")}` !== artifact.integrity) return false; + if (artifact.sha256 && (await digest(file, "sha256", "hex")) !== artifact.sha256) return false; + return true; +} + +async function exists(file: string): Promise { + return stat(file).then( + () => true, + () => false, + ); +} + +async function verify(file: string, artifact: ReviewRemoteArtifact): Promise { + if (await matches(file, artifact)) return; + await rm(file, { force: true }); + throw new Error(`${artifact.name} does not match its pinned checksum; the file was deleted.`); +} + +export async function fetchToLaptopCache(artifact: ReviewRemoteArtifact, options: { cacheDirectory: string }): Promise { + const file = cachePath(options.cacheDirectory, artifact); + if (await exists(file)) { + await verify(file, artifact); + return file; + } + if (!/^https?:/.test(artifact.url)) throw new Error(`${artifact.name} is not in the laptop cache.`); + await mkdir(options.cacheDirectory, { recursive: true }); + const part = `${file}.${randomBytes(4).toString("hex")}.part`; + try { + const response = await fetch(artifact.url, { signal: AbortSignal.timeout(REVIEW_REMOTE_ARTIFACT_TIMEOUTS.download) }); + if (!response.ok || !response.body) throw new Error(`Downloading ${artifact.name} failed: ${artifact.url} answered ${response.status}.`); + await pipeline(response.body, createWriteStream(part)); + await verify(part, artifact); + await rename(part, file); + } finally { + await rm(part, { force: true }); + } + return file; +} + +function run(command: string, args: string[], cwd: string, timeout: number, data: (chunk: Buffer) => void = () => {}): Promise { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { cwd, stdio: ["ignore", "pipe", "pipe"], timeout, killSignal: "SIGKILL" }); + let stderr = ""; + child.stdout.on("data", data); + child.stderr.setEncoding("utf8").on("data", (chunk: string) => (stderr = (stderr + chunk).slice(-2000))); + child.once("error", reject); + child.once("close", (code, signal) => { + if (code === 0) resolve(); + else reject(new Error(`${command} ${args.join(" ")} ${signal ? `was stopped (${signal})` : `exited with ${code}`}: ${stderr.trim()}`)); + }); + }); +} + +async function checkoutState(checkout: string): Promise { + const hash = createHash("sha256"); + const feed = (chunk: Buffer) => hash.update(chunk); + const git = (...args: string[]) => run("git", args, checkout, REVIEW_REMOTE_ARTIFACT_TIMEOUTS.git, feed); + await git("rev-parse", "HEAD"); + await git("status", "--porcelain=v1", "-z", "--untracked-files=all"); + await git("diff", "HEAD", "--binary"); + return hash.digest("hex").slice(0, 32); +} + +async function packCheckout(checkout: string, cacheDirectory: string): Promise { + const record = join(cacheDirectory, `dev-pack-${await checkoutState(checkout)}.json`); + const previous = await readFile(record, "utf8").then( + (text) => JSON.parse(text) as ReviewRemoteArtifact, + () => undefined, + ); + if (previous && (await exists(cachePath(cacheDirectory, previous)))) return previous; + + const directory = join(checkout, "packages", "review"); + const manifest = JSON.parse(await readFile(join(directory, "package.json"), "utf8")) as { name: string; version: string }; + const scratch = await mkdtemp(join(cacheDirectory, "dev-pack-")); + try { + await run("pnpm", ["--dir", directory, "pack", "--pack-destination", scratch], checkout, REVIEW_REMOTE_ARTIFACT_TIMEOUTS.pack); + const [packed] = (await readdir(scratch)).filter((name) => name.endsWith(".tgz")); + if (!packed) throw new Error(`pnpm pack of ${directory} wrote no tarball.`); + const integrity = `sha512-${await digest(join(scratch, packed), "sha512", "base64")}`; + const file = cachePath(cacheDirectory, { name: packed, url: "", integrity }); + await rename(join(scratch, packed), file); + const artifact = { name: tarballName(manifest.name, manifest.version), url: pathToFileURL(file).href, integrity }; + await writeFile(record, JSON.stringify(artifact)); + return artifact; + } finally { + await rm(scratch, { recursive: true, force: true }); + } +} + +async function developmentNode(target: ReviewRemoteTarget, checkout: string, cacheDirectory: string, dist: string): Promise { + const nvmrc = join(checkout, "apps", "review-desktop", "code-oss", ".nvmrc"); + const version = (await readFile(nvmrc, "utf8")).trim(); + if (!/^\d+\.\d+\.\d+$/.test(version)) throw new Error(`${nvmrc} does not hold an exact Node version.`); + const sums = join(cacheDirectory, `node-v${version}-SHASUMS256.txt`); + let text = await readFile(sums, "utf8").catch(() => undefined); + if (text === undefined) { + const url = `${dist}/v${version}/SHASUMS256.txt`; + const response = await fetch(url, { signal: AbortSignal.timeout(60_000) }); + if (!response.ok) throw new Error(`${url} answered ${response.status}.`); + text = await response.text(); + await writeFile(sums, text); + } + const name = `node-v${version}-${target}.tar.xz`; + const sha256 = text + .split("\n") + .map((line) => line.trim().split(/\s+/)) + .find(([, file]) => file === name)?.[0]; + if (!sha256 || !/^[0-9a-f]{64}$/.test(sha256)) throw new Error(`SHASUMS256.txt for Node ${version} has no line for ${name}.`); + return { name, url: `${dist}/v${version}/${name}`, sha256 }; +} diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteAttachScript.test.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteAttachScript.test.ts new file mode 100644 index 000000000..8993d6ffa --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteAttachScript.test.ts @@ -0,0 +1,94 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { chmod, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { REVIEW_REMOTE_ATTACH_BEGIN, REVIEW_REMOTE_ATTACH_END } from "../../common/reviewProtocol.js"; +import { installedAttachScript, parseRemoteAttach, reviewRemoteAttachScript } from "./reviewRemoteAttachScript.js"; + +async function executable(path: string, body: string) { + await writeFile(path, `#!/bin/sh\n${body}\n`); + await chmod(path, 0o755); +} + +function attach(home: string, shell: string, groups: string[] = []) { + return spawnSync("/bin/sh", ["-s"], { + input: reviewRemoteAttachScript(groups), + encoding: "utf8", + env: { HOME: home, SHELL: shell, PATH: "/usr/bin:/bin" }, + }); +} + +test("the CLI a login shell finds is used, past rc files that print paths", async (t) => { + const home = await mkdtemp(join(tmpdir(), "wb-attach-")); + t.after(() => rm(home, { recursive: true, force: true })); + const bin = join(home, ".nvm/versions/node/v24/bin"); + await mkdir(bin, { recursive: true }); + await executable(join(bin, "whiteboard"), 'echo "attached: $*"'); + await executable(join(home, "login-shell"), `echo /etc/profile.d/banner.sh\necho ${bin}/whiteboard\necho /home/u/.nvm/nvm.sh`); + + const result = attach(home, join(home, "login-shell")); + + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stdout, "attached: remote attach --json\n"); +}); + +test("with no CLI anywhere the script exits 127", async (t) => { + const home = await mkdtemp(join(tmpdir(), "wb-attach-")); + t.after(() => rm(home, { recursive: true, force: true })); + await executable(join(home, "login-shell"), "echo /home/u/.nvm/nvm.sh"); + + assert.equal(attach(home, join(home, "login-shell")).status, 127); +}); + +test("the Desktop's enabled extension groups reach the CLI, and nothing else can", async (t) => { + const home = await mkdtemp(join(tmpdir(), "wb-attach-")); + t.after(() => rm(home, { recursive: true, force: true })); + await mkdir(join(home, ".local/bin"), { recursive: true }); + await executable(join(home, ".local/bin/whiteboard"), 'echo "attached: $*"'); + + assert.equal(attach(home, "/bin/false", ["go", "rust"]).stdout, "attached: remote attach --json --groups go,rust\n"); + assert.throws(() => reviewRemoteAttachScript(["go; rm -rf ~"]), /Invalid extension group/); +}); + +test("the installed CLI's attach carries --replace and the groups; the PATH CLI never gets --replace", () => { + assert.equal(installedAttachScript("/n/node", "/v/cli.js", ["go", "rust"]), "exec '/n/node' '/v/cli.js' remote attach --json --replace --groups go,rust\n"); + assert.equal(installedAttachScript("/n/node", "/v/cli.js"), "exec '/n/node' '/v/cli.js' remote attach --json --replace\n"); + assert.throws(() => installedAttachScript("/n/node", "/v/cli.js", ["go rust"]), /Invalid extension group/); + assert.doesNotMatch(reviewRemoteAttachScript(["go"]), /--replace/); +}); + +test("one attach line with the language server, its groups, a replaced server and one left running is read whole", () => { + const line = { + event: "remote.attach", + version: "0.1.7", + serverId: "0199a3f2-7c1e-7d4a-9b2f-3e5d6c7b8a90", + url: "http://127.0.0.1:41234", + token: "remote-token", + languageServer: { port: 45678, connectionToken: "vscode-token", commit: "a".repeat(40) }, + languageGroups: [{ group: "go", installed: false, detail: "go was not found on the login shell's PATH" }], + replaced: true, + previousVersion: "0.1.6", + incompatibleRunning: { version: "0.1.8", pid: 7, startedBy: "desktop" }, + }; + + assert.deepEqual(parseRemoteAttach(`${REVIEW_REMOTE_ATTACH_BEGIN}\n${JSON.stringify(line)}\n${REVIEW_REMOTE_ATTACH_END}\n`), { + attach: { + version: "0.1.7", + serverId: line.serverId, + token: "remote-token", + port: 41234, + languageServer: line.languageServer, + languageGroups: line.languageGroups, + replaced: "0.1.6", + incompatibleRunning: { version: "0.1.8", startedBy: "desktop" }, + }, + }); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteAttachScript.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteAttachScript.ts new file mode 100644 index 000000000..32381231f --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteAttachScript.ts @@ -0,0 +1,157 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { REVIEW_REMOTE_ATTACH_BEGIN, REVIEW_REMOTE_ATTACH_END } from "../../common/reviewProtocol.js"; +import { REVIEW_REMOTE_VERSION, shellQuote } from "./reviewRemoteInstallScript.js"; + +export const REVIEW_REMOTE_FIND_CLI = `bounded() { + if timeout -k 1 1 true >/dev/null 2>&1; then timeout -k 1 "$@"; else shift; "$@"; fi +} +wb=$(command -v whiteboard 2>/dev/null) +case "$wb" in /*) ;; *) wb= ;; esac +if [ -z "$wb" ] && [ -x "$HOME/.local/bin/whiteboard" ]; then wb="$HOME/.local/bin/whiteboard"; fi +if [ -z "$wb" ] && [ -n "$SHELL" ]; then + wb=$(bounded 3 "$SHELL" -lic 'command -v whiteboard' /dev/null | tr -d '\\r' | grep '^/.*/whiteboard$' | tail -n 1) +fi +[ -n "$wb" ] && [ -x "$wb" ] || wb= +`; + +export function pathCliScript(words: string): string { + return `${REVIEW_REMOTE_FIND_CLI}if [ -z "$wb" ]; then exit 127; fi +PATH="\${wb%/*}:$PATH" +export PATH +exec "$wb" ${words} +`; +} + +function attachWords(groups: readonly string[], replace: boolean): string { + for (const group of groups) { + if (!/^[a-z0-9-]+$/.test(group)) throw new Error(`Invalid extension group ${JSON.stringify(group)}.`); + } + return `remote attach --json${replace ? " --replace" : ""}${groups.length ? ` --groups ${groups.join(",")}` : ""}`; +} + +export function reviewRemoteAttachScript(groups: readonly string[] = [], replace = false): string { + return pathCliScript(attachWords(groups, replace)); +} + +export function installedAttachScript(nodePath: string, cliPath: string, groups: readonly string[] = []): string { + return `exec ${shellQuote(nodePath)} ${shellQuote(cliPath)} ${attachWords(groups, true)}\n`; +} + +export interface ReviewRemoteLanguageServer { + readonly port: number; + readonly connectionToken: string; + readonly commit: string; +} + +export interface ReviewRemoteLanguageGroup { + readonly group: string; + readonly installed: boolean; + readonly detail?: string; +} + +export interface ReviewRemoteAttach { + readonly version: string | null; + readonly serverId: string | null; + readonly token: string; + readonly port: number; + readonly languageServer: ReviewRemoteLanguageServer | null; + readonly languageServerDetail?: string; + readonly languageServerPending?: true; + readonly languageGroups: readonly ReviewRemoteLanguageGroup[]; + readonly replaced?: string; + readonly incompatibleRunning?: { readonly version: string; readonly startedBy: "user" | "cli" | "desktop" }; +} + +export function parseRemoteAttach(stdout: string): { attach: ReviewRemoteAttach } | { error: string } | undefined { + const begin = stdout.indexOf(REVIEW_REMOTE_ATTACH_BEGIN); + if (begin < 0) return undefined; + const from = begin + REVIEW_REMOTE_ATTACH_BEGIN.length; + const end = stdout.indexOf(REVIEW_REMOTE_ATTACH_END, from); + if (end < 0) return undefined; + for (const line of stdout.slice(from, end).split("\n")) { + let value: unknown; + try { + value = JSON.parse(line); + } catch { + continue; + } + if (!value || typeof value !== "object") continue; + const record = value as Record; + if (record.event === "error") { + const error = record.error as Record | undefined; + return { error: typeof error?.message === "string" ? error.message : "remote attach failed." }; + } + if (record.event !== "remote.attach") continue; + const port = loopbackPort(record.url); + if (port === undefined || typeof record.token !== "string" || !record.token) { + return { error: "remote attach answered without a loopback URL and a token." }; + } + const running = record.incompatibleRunning as Record | undefined; + return { + attach: { + version: typeof record.version === "string" ? record.version : null, + serverId: typeof record.serverId === "string" && UUID.test(record.serverId) ? record.serverId : null, + token: record.token, + port, + ...languageServerOf(record), + languageGroups: languageGroupsOf(record.languageGroups), + ...(record.replaced === true && { replaced: versionText(record.previousVersion) }), + ...(running && + typeof running === "object" && { + incompatibleRunning: { version: versionText(running.version), startedBy: running.startedBy === "cli" || running.startedBy === "desktop" ? running.startedBy : "user" }, + }), + }, + }; + } + return { error: "remote attach printed nothing readable between its sentinels." }; +} + +function languageServerOf(record: Record): Pick { + const detail = typeof record.languageServerDetail === "string" ? record.languageServerDetail.slice(0, 2000) : undefined; + const server = record.languageServer as Record | null | undefined; + if (server && typeof server === "object") { + const { port, connectionToken, commit } = server; + if ( + typeof port === "number" && Number.isInteger(port) && port > 0 && port < 65536 && + typeof connectionToken === "string" && /^[0-9A-Za-z_-]+$/.test(connectionToken) && + typeof commit === "string" && /^[0-9a-f]{40}$/.test(commit) + ) { + return { languageServer: { port, connectionToken, commit } }; + } + return { languageServer: null, languageServerDetail: "remote attach reported a VS Code server without a port, a token and a commit." }; + } + return { + languageServer: null, + languageServerDetail: detail ?? "The Whiteboard on this host has no VS Code server.", + ...(record.languageServerPending === true && { languageServerPending: true as const }), + }; +} + +function languageGroupsOf(value: unknown): ReviewRemoteLanguageGroup[] { + if (!Array.isArray(value)) return []; + return value.slice(0, 16).flatMap((entry: unknown) => { + const { group, installed, detail } = (entry && typeof entry === "object" ? entry : {}) as Record; + if (typeof group !== "string" || !/^[a-z0-9-]{1,40}$/.test(group) || typeof installed !== "boolean") return []; + return [{ group, installed, ...(typeof detail === "string" && detail && { detail: detail.slice(0, 500) }) }]; + }); +} + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +const versionText = (value: unknown) => (typeof value === "string" && REVIEW_REMOTE_VERSION.test(value) ? value : "unknown"); + +function loopbackPort(url: unknown): number | undefined { + if (typeof url !== "string") return undefined; + try { + const parsed = new URL(url); + if (parsed.protocol !== "http:" || !["127.0.0.1", "localhost", "[::1]"].includes(parsed.hostname)) return undefined; + const port = Number(parsed.port); + return Number.isInteger(port) && port > 0 && port < 65536 ? port : undefined; + } catch { + return undefined; + } +} diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteHost.test.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteHost.test.ts new file mode 100644 index 000000000..e4481fee9 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteHost.test.ts @@ -0,0 +1,1255 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from "node:assert/strict"; +import { existsSync } from "node:fs"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import type { ReviewGatewayHost } from "../../common/reviewProtocol.js"; +import { attachOutput, detectOutput, FAKE_SERVER_ID, fakeClock, fakeSsh, until, type FakeRemote } from "./test/fakeSsh.js"; +import { classifySshFailure, languageCommitMismatch, ReviewRemoteHost, type ReviewRemoteHostOptions, type ReviewRemoteInstallFlow, type ReviewRemoteInstallMode, type ReviewRemoteInstallRunInput } from "./reviewRemoteHost.js"; +import { openRemoteInstallConsent } from "./reviewRemoteInstallConsent.js"; +import type { ReviewRemoteInstallProgress, ReviewRemoteInstallResult } from "./reviewRemoteInstaller.js"; +import { reviewSshSession } from "./reviewSshCommand.js"; + +async function healthServer(t: test.TestContext, servers?: Server[]): Promise { + const server: Server = createServer((_request, response) => response.end('{"ok":true}')); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + t.after(() => new Promise((resolve) => server.close(() => resolve(undefined)))); + servers?.push(server); + return (server.address() as AddressInfo).port; +} + +async function versionServer(t: test.TestContext, commit: string): Promise { + const server: Server = createServer((request, response) => response.end(request.url === "/version" ? commit : "")); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + t.after(() => new Promise((resolve) => server.close(() => resolve(undefined)))); + return (server.address() as AddressInfo).port; +} + +const COMMIT = "a".repeat(40); + +const NO_SERVER = { languageFeatures: false, languageFeaturesDetail: "The Whiteboard on this host has no VS Code server." }; + +function hostFor( + t: test.TestContext, + remote: FakeRemote, + ports: number | number[] | (() => Promise), + alias = "wb-test-a", + controlDirectory = "/tmp/wb-ssh-test", + install?: ReviewRemoteInstallFlow, + version = "0.1.6", + extra: Partial = {}, +) { + const free = typeof ports === "function" ? [] : [ports].flat(); + let next = 0; + const clock = fakeClock(); + const ssh = fakeSsh({ [alias]: remote }, clock); + const reports: ReviewGatewayHost[] = []; + const host = new ReviewRemoteHost({ + session: reviewSshSession(alias, controlDirectory), + spawn: ssh.spawn, + environment: async () => ({ PATH: "/usr/bin" }), + desktopVersion: async () => version, + groups: async () => ["go"], + freePort: typeof ports === "function" ? ports : async () => free[next++ % free.length], + report: (state) => reports.push(state), + log: () => {}, + clock, + timeouts: { poll: 1 }, + install, + ...extra, + }); + t.after(() => host.dispose()); + return { host, ssh, clock, reports, last: () => reports.at(-1) }; +} + +const INTEGRITY = `sha512-${"A".repeat(86)}==`; +const at = (version: string, integrity = INTEGRITY) => ({ version, integrity }); + +const INSTALLED: ReviewRemoteInstallResult = { + nodePath: "/home/dev/.dev/whiteboard-remote/node/v24.18.0/bin/node", + cliPath: "/home/dev/.dev/whiteboard-remote/versions/0.1.6/node_modules/@dev.fast/whiteboard/dist/cli.js", +}; + +async function installFlow( + t: test.TestContext, + mode: ReviewRemoteInstallMode, + options: { + answers?: (boolean | undefined)[]; + steps?: ReviewRemoteInstallProgress[]; + fails?: (call: number) => Error | undefined; + consentFile?: string; + } = {}, +) { + const dir = await mkdtemp(join(tmpdir(), "wb-flow-")); + t.after(() => rm(dir, { recursive: true, force: true })); + const consentFile = options.consentFile ?? join(dir, "consent.json"); + const prompts: { alias: string; text: string }[] = []; + const runs: ReviewRemoteInstallRunInput[] = []; + const answers = [...(options.answers ?? [])]; + const flow: ReviewRemoteInstallFlow = { + mode: () => mode, + consent: openRemoteInstallConsent(consentFile), + confirm: async (request) => { + prompts.push(request); + return answers.shift(); + }, + run: async (input) => { + runs.push(input); + for (const step of options.steps ?? [{ step: "done", cliPath: INSTALLED.cliPath }]) input.onProgress(step); + const error = options.fails?.(runs.length); + if (error) throw error; + return INSTALLED; + }, + integrity: async () => INTEGRITY, + }; + return { flow, prompts, runs, consentFile }; +} + +const STEPS: ReviewRemoteInstallProgress[] = [ + { step: "node", via: "upload" }, + { step: "package", via: "remote-download" }, + { step: "verifying" }, + { step: "done", cliPath: INSTALLED.cliPath }, +]; + +test("a successful attach reports an endpoint at the forwarded port", async (t) => { + const port = await healthServer(t); + const { host, ssh, last } = hostFor(t, { remotePort: 41234 }, port); + + host.start(); + await until(() => last()?.endpoint !== undefined); + + assert.deepEqual(last(), { alias: "wb-test-a", endpoint: { url: `http://127.0.0.1:${port}`, token: "remote-token" }, ...NO_SERVER }); + const [forward] = ssh.of("wb-test-a", "forward"); + assert.ok(forward.args.includes(`127.0.0.1:${port}:127.0.0.1:41234`)); + const kinds = ssh.calls.map((c) => c.kind); + assert.ok(kinds.indexOf("check") < kinds.indexOf("exec")); +}); + +test("the VS Code server gets a second forward on the same master, and only its endpoint reaches a window", async (t) => { + const ports = [await healthServer(t), await versionServer(t, COMMIT)]; + const languageServer = { port: 45678, connectionToken: "vscode-token", commit: COMMIT }; + const { host, ssh, last } = hostFor(t, { attach: { code: 0, stdout: attachOutput(41234, "remote-token", { languageServer }) } }, ports); + + host.start(); + await until(() => last()?.endpoint !== undefined); + + assert.deepEqual(last(), { alias: "wb-test-a", endpoint: { url: `http://127.0.0.1:${ports[0]}`, token: "remote-token" }, languageFeatures: true }); + assert.equal(ssh.of("wb-test-a", "master").length, 1); + const forwards = ssh.of("wb-test-a", "forward").map((call) => call.args.find((arg) => arg.startsWith("127.0.0.1:"))); + assert.deepEqual(forwards, [`127.0.0.1:${ports[0]}:127.0.0.1:41234`, `127.0.0.1:${ports[1]}:127.0.0.1:45678`]); + assert.deepEqual(await host.languageEndpoint(FAKE_SERVER_ID), { host: "127.0.0.1", port: ports[1], connectionToken: "vscode-token" }); + assert.equal(await host.languageEndpoint("another machine"), undefined); + assert.doesNotMatch(JSON.stringify(last()), /vscode-token/); +}); + +test("the Desktop's enabled groups go to remote attach", async (t) => { + const port = await healthServer(t); + const { host, ssh, last } = hostFor(t, {}, port); + + host.start(); + await until(() => last()?.endpoint !== undefined); + + assert.match(ssh.of("wb-test-a", "exec")[0].input!, /exec "\$wb" remote attach --json --groups go\n$/); +}); + +test("the remote's language groups reach the gateway: well-formed, asked for by this Desktop, once each", async (t) => { + const port = await healthServer(t); + const languageGroups = [ + { group: "Not A Group", installed: true }, + { group: "go" }, + "go", + { group: "python", installed: true }, + { group: "go", installed: true, detail: "x".repeat(900) }, + { group: "go", installed: false, detail: "a second go" }, + ]; + const stdout = `WHITEBOARD-REMOTE-BEGIN\n${JSON.stringify({ event: "remote.attach", serverId: FAKE_SERVER_ID, url: `http://127.0.0.1:${port}`, token: "remote-token", languageServer: null, languageServerDetail: "none", languageGroups })}\nWHITEBOARD-REMOTE-END\n`; + const { host, last } = hostFor(t, { attach: { code: 0, stdout } }, port); + + host.start(); + await until(() => last()?.endpoint !== undefined); + + assert.deepEqual(last()?.languageGroups, [{ group: "go", installed: true, detail: "x".repeat(500) }]); +}); + +test("a VS Code server of another commit leaves the review online without language features", async (t) => { + const ports = [await healthServer(t), await versionServer(t, COMMIT)]; + const languageServer = { port: 45678, connectionToken: "vscode-token", commit: COMMIT }; + const { host, ssh, last } = hostFor(t, { attach: { code: 0, stdout: attachOutput(41234, "remote-token", { languageServer }) } }, ports, "wb-test-a", "/tmp/wb-ssh-test", undefined, undefined, { desktopCommit: "b".repeat(40) }); + + host.start(); + await until(() => last()?.endpoint !== undefined); + + assert.deepEqual(last(), { + alias: "wb-test-a", + endpoint: { url: `http://127.0.0.1:${ports[0]}`, token: "remote-token" }, + languageFeatures: false, + languageFeaturesDetail: "language features need the same Whiteboard version on wb-test-a: it runs aaaaaaa, this Desktop bbbbbbb", + }); + assert.equal(ssh.of("wb-test-a", "forward").length, 1); + assert.equal(await host.languageEndpoint(FAKE_SERVER_ID), undefined); +}); + +test("a dev Desktop, with no commit, accepts any VS Code server; a release Desktop only its own", () => { + assert.equal(languageCommitMismatch("a", COMMIT, undefined), undefined); + assert.equal(languageCommitMismatch("a", COMMIT, COMMIT), undefined); + assert.match(languageCommitMismatch("a", COMMIT, "b".repeat(40))!, /on a: it runs aaaaaaa, this Desktop bbbbbbb$/); +}); + +test("a VS Code server that does not answer through its forward is unavailable, and its forward is cancelled", async (t) => { + const ports = [await healthServer(t), await versionServer(t, "c".repeat(40))]; + const languageServer = { port: 45678, connectionToken: "vscode-token", commit: COMMIT }; + const { host, ssh, last } = hostFor(t, { attach: { code: 0, stdout: attachOutput(41234, "remote-token", { languageServer }) } }, ports); + + host.start(); + await until(() => last()?.endpoint !== undefined); + + assert.equal(last()?.languageFeatures, false); + assert.match(last()!.languageFeaturesDetail!, /did not answer through the forward: it reports c{40}, not a{40}\.$/); + const cancels = ssh.of("wb-test-a", "cancel"); + assert.equal(cancels.length, 1); + assert.ok(cancels[0].args.includes(`127.0.0.1:${ports[1]}:127.0.0.1:45678`)); + assert.equal(await host.languageEndpoint(FAKE_SERVER_ID), undefined); +}); + +test("a reattach drops both old forwards", async (t) => { + const ports = [await healthServer(t), await versionServer(t, COMMIT), await healthServer(t), await versionServer(t, COMMIT)]; + const { host, ssh, last } = hostFor( + t, + { attach: (call) => ({ code: 0, stdout: attachOutput(41234 + call, `token-${call}`, { languageServer: { port: 45678 + call, connectionToken: `vscode-${call}`, commit: COMMIT } }) }) }, + ports, + ); + + host.start(); + await until(() => last()?.endpoint !== undefined); + await host.reattach(); + + assert.equal(last()?.endpoint?.token, "token-2"); + assert.equal(ssh.of("wb-test-a", "forward").length, 4); + assert.deepEqual( + ssh.of("wb-test-a", "cancel").map((call) => call.args.find((arg) => arg.startsWith("127.0.0.1:"))), + [`127.0.0.1:${ports[0]}:127.0.0.1:41235`, `127.0.0.1:${ports[1]}:127.0.0.1:45679`], + ); + assert.deepEqual(await host.languageEndpoint(FAKE_SERVER_ID), { host: "127.0.0.1", port: ports[3], connectionToken: "vscode-2" }); +}); + +const PENDING_DETAIL = "Installing the language extensions on this host; they will be available on the next connection."; + +const pendingOutput = (port: number) => + `WHITEBOARD-REMOTE-BEGIN\n${JSON.stringify({ event: "remote.attach", version: "0.1.6", commit: "abc", serverId: FAKE_SERVER_ID, url: `http://127.0.0.1:${port}`, token: "remote-token", startedServer: false, languageServer: null, languageServerDetail: PENDING_DETAIL, languageServerPending: true })}\nWHITEBOARD-REMOTE-END\n`; + +test("a VS Code server that stopped answering is not handed out, and the host attaches again", async (t) => { + const stopped: Server = createServer((request, response) => response.end(request.url === "/version" ? COMMIT : "")); + await new Promise((resolve) => stopped.listen(0, "127.0.0.1", resolve)); + t.after(() => new Promise((resolve) => stopped.close(() => resolve(undefined)))); + const ports = [await healthServer(t), (stopped.address() as AddressInfo).port, await versionServer(t, COMMIT)]; + const { host, ssh, last } = hostFor( + t, + { attach: (call) => ({ code: 0, stdout: attachOutput(41234, "remote-token", { languageServer: { port: 45677 + call, connectionToken: `vscode-${call}`, commit: COMMIT } }) }) }, + ports, + ); + + host.start(); + await until(() => last()?.languageFeatures === true); + await new Promise((resolve) => stopped.close(resolve)); + + assert.equal(await host.languageEndpoint(FAKE_SERVER_ID), undefined); + await until(() => ssh.of("wb-test-a", "exec").length === 2 && ssh.of("wb-test-a", "cancel").length === 1); + assert.deepEqual(await host.languageEndpoint(FAKE_SERVER_ID), { host: "127.0.0.1", port: ports[2], connectionToken: "vscode-2" }); + assert.equal(last()?.endpoint?.url, `http://127.0.0.1:${ports[0]}`); + assert.deepEqual( + ssh.of("wb-test-a", "cancel").map((call) => call.args.find((arg) => arg.startsWith("127.0.0.1:"))), + [`127.0.0.1:${ports[1]}:127.0.0.1:45678`], + ); +}); + +test("a reattach the gateway asks for opens a new review forward, even to the same server", async (t) => { + const ports = [await healthServer(t), await healthServer(t)]; + const { host, ssh, last } = hostFor(t, {}, ports); + + host.start(); + await until(() => last()?.endpoint !== undefined); + await host.reattach(); + + assert.equal(last()?.endpoint?.url, `http://127.0.0.1:${ports[1]}`); + assert.equal(last()?.endpoint?.token, "remote-token"); + assert.equal(ssh.of("wb-test-a", "forward").length, 2); + assert.ok(ssh.of("wb-test-a", "cancel")[0].args.includes(`127.0.0.1:${ports[0]}:127.0.0.1:41234`)); +}); + +test("a reattach the gateway asks for during a pending attach runs after it, with new forwards", async (t) => { + const ports = [await healthServer(t), await versionServer(t, COMMIT), await healthServer(t), await versionServer(t, COMMIT)]; + const { host, ssh, clock, last } = hostFor( + t, + { attach: (call) => ({ code: 0, stdout: call === 1 ? pendingOutput(41234) : attachOutput(41234, "remote-token", { languageServer: { port: 45678, connectionToken: `vscode-${call}`, commit: COMMIT } }) }) }, + ports, + ); + + host.start(); + await until(() => last()?.endpoint !== undefined); + assert.ok(clock.next()); + await host.reattach(); + await until(() => ssh.of("wb-test-a", "exec").length === 3 && last()?.endpoint?.url === `http://127.0.0.1:${ports[2]}`); + + assert.equal(last()?.languageFeatures, true); + assert.deepEqual(await host.languageEndpoint(FAKE_SERVER_ID), { host: "127.0.0.1", port: ports[3], connectionToken: "vscode-3" }); +}); + +test("a Retry after ten pending attaches attaches again on the same schedule", async (t) => { + const port = await healthServer(t); + const { host, ssh, clock, last } = hostFor(t, { attach: { code: 0, stdout: pendingOutput(41234) } }, port); + + host.start(); + await until(() => last()?.endpoint !== undefined); + for (let attaches = 1; attaches < 10; attaches++) { + await until(() => clock.pending === 1); + clock.next(); + await until(() => ssh.of("wb-test-a", "exec").length === attaches + 1); + } + await new Promise((resolve) => setTimeout(resolve, 100)); + assert.equal(clock.pending, 0); + + host.retry(); + await until(() => ssh.of("wb-test-a", "exec").length === 11 && last()?.endpoint !== undefined); + await until(() => clock.pending === 1); + assert.equal(clock.delays.at(-1), 60_000); +}); + +test("a remote still installing its extensions is attached again after a minute, until its VS Code server is reported", async (t) => { + const ports = [await healthServer(t), await versionServer(t, COMMIT)]; + const { host, ssh, clock, last } = hostFor( + t, + { attach: (call) => ({ code: 0, stdout: call === 1 ? pendingOutput(41234) : attachOutput(41234, "remote-token", { languageServer: { port: 45678, connectionToken: "vscode-token", commit: COMMIT } }) }) }, + ports, + ); + + host.start(); + await until(() => last()?.endpoint !== undefined); + + assert.deepEqual(last(), { + alias: "wb-test-a", + endpoint: { url: `http://127.0.0.1:${ports[0]}`, token: "remote-token" }, + languageFeatures: false, + languageFeaturesDetail: PENDING_DETAIL, + }); + assert.equal(clock.pending, 1); + assert.equal(clock.delays.at(-1), 60_000); + assert.ok(clock.next()); + await until(() => last()?.languageFeatures === true); + + assert.equal(last()?.endpoint?.url, `http://127.0.0.1:${ports[0]}`); + assert.equal(ssh.of("wb-test-a", "exec").length, 2); + assert.equal(clock.pending, 0); +}); + +test("attaching again for a pending install stops after ten attaches in a row", async (t) => { + const port = await healthServer(t); + const { host, ssh, clock, last } = hostFor(t, { attach: { code: 0, stdout: pendingOutput(41234) } }, port); + + host.start(); + await until(() => last()?.endpoint !== undefined); + for (let attaches = 1; attaches < 10; attaches++) { + await until(() => clock.pending === 1); + clock.next(); + await until(() => ssh.of("wb-test-a", "exec").length === attaches + 1); + } + await new Promise((resolve) => setTimeout(resolve, 100)); + assert.equal(clock.pending, 0); + + assert.equal(ssh.of("wb-test-a", "exec").length, 10); + assert.equal(last()?.languageFeaturesDetail, PENDING_DETAIL); + assert.equal(last()?.endpoint?.url, `http://127.0.0.1:${port}`); +}); + +test("output with a banner before the first sentinel still parses", async (t) => { + const port = await healthServer(t); + const banner = "Welcome to Ubuntu 22.04\n\nLast login: yesterday\nnvm: using node 24"; + const { host, last } = hostFor(t, { attach: { code: 0, stdout: `${banner}${attachOutput(41234, "t2")}bye\n` } }, port); + + host.start(); + await until(() => last()?.endpoint !== undefined); + + assert.equal(last()?.endpoint?.token, "t2"); +}); + +test("exit 127 from the script is not-installed, naming the version to install", async (t) => { + const { host, clock, last } = hostFor(t, { attach: { code: 127 } }, 1); + + host.start(); + await until(() => last()?.problem !== undefined); + + assert.equal(last()?.problem?.state, "not-installed"); + assert.match(last()!.problem!.detail, /Install Whiteboard 0\.1\.6 there/); + assert.doesNotMatch(last()!.problem!.detail, /npm install/); + assert.match(last()!.problem!.detail, /Node 24/); + assert.equal(clock.pending, 0); +}); + +test("a cancelled prompt is auth-failed, and there is no second attempt", async (t) => { + const { host, ssh, clock, last } = hostFor(t, { master: "hang" }, 1); + + host.start(); + await until(() => ssh.master("wb-test-a") !== undefined); + host.promptOpened(); + host.promptClosed(false); + ssh.master("wb-test-a")!.finish(255, { + stderr: "Warning: Permanently added '[127.0.0.1]:2222' (ED25519) to the list of known hosts.\ndev@127.0.0.1: Permission denied (publickey,password).\n", + }); + await until(() => last()?.problem !== undefined); + + assert.equal(last()?.problem?.state, "auth-failed"); + assert.equal(last()!.problem!.detail, "dev@127.0.0.1: Permission denied (publickey,password)."); + assert.equal(clock.pending, 0); + assert.equal(ssh.of("wb-test-a", "master").length, 1); +}); + +test("OpenSSH's authentication and host key refusals are auth-failed; the rest unreachable", () => { + assert.equal(classifySshFailure("u@h: Permission denied (publickey).", false), "auth-failed"); + assert.equal(classifySshFailure("Host key verification failed.", false), "auth-failed"); + assert.equal(classifySshFailure("@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @", false), "auth-failed"); + assert.equal(classifySshFailure("Connection closed by 10.0.0.1 port 22", true), "auth-failed"); + assert.equal(classifySshFailure("ssh: connect to host h port 22: Connection refused", false), "unreachable"); +}); + +test("the master exits and the host reconnects after the backoff", async (t) => { + const port = await healthServer(t); + const { host, ssh, clock, last } = hostFor(t, {}, port); + + host.start(); + await until(() => last()?.endpoint !== undefined); + ssh.master("wb-test-a")!.finish(255, { stderr: "Connection to 127.0.0.1 closed by remote host.\n" }); + await until(() => last()?.problem !== undefined); + + assert.equal(last()?.problem?.state, "unreachable"); + assert.match(last()!.problem!.detail, /closed by remote host/); + assert.equal(ssh.of("wb-test-a", "master").length, 1); + assert.ok(clock.next()); + await until(() => last()?.endpoint !== undefined); + assert.equal(ssh.of("wb-test-a", "master").length, 2); + assert.ok(clock.delays[0] >= 1000 && clock.delays[0] <= 1250); +}); + +test("a master killed by a signal leaves its socket, and the next master does not find it", async (t) => { + const port = await healthServer(t); + const dir = await mkdtemp(join(tmpdir(), "wb-ssh-")); + t.after(() => rm(dir, { recursive: true, force: true })); + const { host, ssh, clock, last } = hostFor(t, {}, port, "wb-test-a", dir); + const socket = reviewSshSession("wb-test-a", dir).controlPath; + + host.start(); + await until(() => last()?.endpoint !== undefined); + await writeFile(socket, ""); + ssh.master("wb-test-a")!.kill("SIGKILL"); + await until(() => last()?.problem !== undefined); + assert.ok(clock.next()); + await until(() => last()?.endpoint !== undefined); + + assert.equal(ssh.of("wb-test-a", "master").length, 2); + assert.equal(existsSync(socket), false); +}); + +test("a resume reconnects at once, without waiting for the backoff", async (t) => { + let masters = 0; + const port = await healthServer(t); + const { host, ssh, clock, last } = hostFor( + t, + { master: () => (++masters === 1 ? { code: 255, stderr: "Network is unreachable\n" } : "up") }, + port, + ); + + host.start(); + await until(() => clock.pending === 1); + await host.resume(); + await until(() => last()?.endpoint !== undefined); + + assert.equal(ssh.of("wb-test-a", "master").length, 2); + assert.equal(clock.pending, 0); +}); + +test("a resume replaces a master whose forward no longer answers", async (t) => { + const servers: Server[] = []; + const ports = [await healthServer(t, servers), await healthServer(t, servers)]; + const { host, ssh, clock, last } = hostFor(t, {}, ports); + + host.start(); + await until(() => last()?.endpoint !== undefined); + const first = ssh.master("wb-test-a")!; + servers[0].closeAllConnections(); + await new Promise((resolve) => servers[0].close(resolve)); + await host.resume(); + + await until(() => last()?.endpoint?.url === `http://127.0.0.1:${ports[1]}`); + assert.equal(ssh.of("wb-test-a", "master").length, 2); + assert.equal(first.alive, false); + assert.equal(clock.pending, 0); +}); + +test("a resume leaves a host whose forward answers alone", async (t) => { + const port = await healthServer(t); + const { host, ssh, last } = hostFor(t, {}, port); + + host.start(); + await until(() => last()?.endpoint !== undefined); + await host.resume(); + + assert.equal(ssh.of("wb-test-a", "master").length, 1); +}); + +test("an authenticated master that ends is unreachable and retried, whatever its prompts left in stderr", async (t) => { + const port = await healthServer(t); + const { host, ssh, clock, last } = hostFor(t, { masterStderr: "Permission denied, please try again.\n" }, port); + + host.start(); + await until(() => last()?.endpoint !== undefined); + ssh.master("wb-test-a")!.finish(255, { stderr: "Connection reset by peer\n" }); + await until(() => last()?.problem !== undefined); + + assert.equal(last()?.problem?.state, "unreachable"); + assert.equal(last()!.problem!.detail, "The SSH connection to wb-test-a ended: Connection reset by peer"); + assert.equal(clock.pending, 1); +}); + +test("a restarted remote server is attached again over the same master", async (t) => { + const ports = [await healthServer(t), await healthServer(t)]; + const { host, ssh, last } = hostFor( + t, + { attach: (call) => ({ code: 0, stdout: attachOutput(41234 + call, `token-${call}`) }) }, + ports, + ); + + host.start(); + await until(() => last()?.endpoint !== undefined); + await host.reattach(); + + assert.deepEqual(last(), { alias: "wb-test-a", endpoint: { url: `http://127.0.0.1:${ports[1]}`, token: "token-2" }, ...NO_SERVER }); + assert.equal(ssh.of("wb-test-a", "master").length, 1); + assert.equal(ssh.of("wb-test-a", "forward").length, 2); + const cancels = ssh.of("wb-test-a", "cancel"); + assert.equal(cancels.length, 1); + assert.ok(cancels[0].args.includes(`127.0.0.1:${ports[0]}:127.0.0.1:41235`)); +}); + +test("a server that keeps restarting is attached again with growing delays, until a stable period", async (t) => { + const ports = [await healthServer(t), await healthServer(t)]; + const { host, ssh, clock, last } = hostFor( + t, + { attach: (call) => ({ code: 0, stdout: attachOutput(41234 + call, `token-${call}`) }) }, + ports, + ); + const execs = () => ssh.of("wb-test-a", "exec"); + + host.start(); + await until(() => last()?.endpoint !== undefined); + await host.reattach(); + assert.equal(execs().length, 2); + for (let call = 3; call <= 4; call++) { + await host.reattach(); + assert.equal(clock.pending, 1); + assert.ok(clock.next()); + await until(() => last()?.endpoint?.token === `token-${call}`); + } + + const [, first, second, third] = execs().map((c) => c.at); + assert.ok(second - first >= 1000, `${second - first} ms`); + assert.ok(third - second > second - first, `${third - second} ms after ${second - first} ms`); + + clock.advance(30_000); + await host.reattach(); + assert.equal(execs().length, 5); + assert.equal(clock.pending, 0); + assert.equal(last()?.endpoint?.token, "token-5"); +}); + +test("a retry drops a reattach waiting for its delay, and the next reattach runs at once", async (t) => { + const port = await healthServer(t); + const { host, ssh, clock, last } = hostFor(t, {}, port); + + host.start(); + await until(() => last()?.endpoint !== undefined); + await host.reattach(); + await host.reattach(); + assert.equal(clock.pending, 1); + host.retry(); + await until(() => last()?.endpoint !== undefined); + + assert.equal(clock.pending, 0); + const execs = ssh.of("wb-test-a", "exec").length; + await host.reattach(); + assert.equal(clock.pending, 0); + assert.equal(ssh.of("wb-test-a", "exec").length, execs + 1); +}); + +test("a resume while a reattach waits for its delay closes the old master before connecting again", async (t) => { + const port = await healthServer(t); + const { host, ssh, clock, last } = hostFor(t, {}, port); + + host.start(); + await until(() => last()?.endpoint !== undefined); + const first = ssh.master("wb-test-a")!; + await host.reattach(); + await host.reattach(); + assert.equal(clock.pending, 1); + await host.resume(); + await until(() => ssh.of("wb-test-a", "master").length === 2 && last()?.endpoint !== undefined); + + assert.equal(first.alive, false); + assert.equal(clock.pending, 0); +}); + +test("a retry starts the new master only after the old one has exited", async (t) => { + const port = await healthServer(t); + const { host, ssh, last } = hostFor(t, { exitDelayMs: 150 }, port); + + host.start(); + await until(() => last()?.endpoint !== undefined); + const first = ssh.master("wb-test-a")!; + host.retry(); + await until(() => ssh.of("wb-test-a", "master").length === 2); + + assert.ok(first.exitedAt !== undefined && ssh.of("wb-test-a", "master")[1].wall >= first.exitedAt); +}); + +test("the sentinels are found after a banner longer than the output bound", async (t) => { + const port = await healthServer(t); + const banner = `${"motd ".repeat(20 * 1024)}\n`; + const { host, last } = hostFor(t, { attach: { code: 0, stdout: `${banner}${attachOutput(41234, "late")}` } }, port); + + host.start(); + await until(() => last()?.endpoint !== undefined); + + assert.equal(last()?.endpoint?.token, "late"); +}); + +test("ssh missing from PATH is unreachable and says OpenSSH is needed", async (t) => { + const { host, last } = hostFor(t, { master: "missing" }, 1); + + host.start(); + await until(() => last()?.problem !== undefined); + + assert.equal(last()?.problem?.state, "unreachable"); + assert.match(last()!.problem!.detail, /OpenSSH is needed/); +}); + +test("dispose closes the master with -O exit", async (t) => { + const port = await healthServer(t); + const { host, ssh, last } = hostFor(t, {}, port); + + host.start(); + await until(() => last()?.endpoint !== undefined); + await host.dispose(); + + assert.equal(ssh.of("wb-test-a", "exit").length, 1); + assert.equal(ssh.alive(), 0); +}); + +test("a dispose while -O check is pending starts no attach, even if the check then succeeds", async (t) => { + const port = await healthServer(t); + const checked = Promise.withResolvers(); + const { host, ssh } = hostFor(t, { checkAnswered: (call) => (call === 2 ? checked.promise : undefined) }, port); + + host.start(); + await until(() => ssh.of("wb-test-a", "check").length === 2); + await host.dispose(); + checked.resolve(); + await new Promise((resolve) => setTimeout(resolve, 20)); + + assert.deepEqual( + ssh.calls.map((c) => c.kind), + ["master", "check", "check", "exit"], + ); +}); + +test("a dispose while the forward's port is chosen starts no forward", async (t) => { + const port = await healthServer(t); + const chosen = Promise.withResolvers(); + const { host, ssh } = hostFor(t, {}, () => chosen.promise); + + host.start(); + await until(() => ssh.of("wb-test-a", "exec").length === 1); + await new Promise((resolve) => setTimeout(resolve, 20)); + await host.dispose(); + chosen.resolve(port); + await new Promise((resolve) => setTimeout(resolve, 20)); + + assert.equal(ssh.of("wb-test-a", "forward").length, 0); +}); + +test("a dispose while the language forward's port is chosen starts no language forward, probe or endpoint", async (t) => { + const port = await healthServer(t); + let probes = 0; + const vscode: Server = createServer((_request, response) => { + probes++; + response.end(COMMIT); + }); + await new Promise((resolve) => vscode.listen(0, "127.0.0.1", resolve)); + t.after(() => new Promise((resolve) => vscode.close(() => resolve(undefined)))); + const chosen = Promise.withResolvers(); + const asked = Promise.withResolvers(); + let calls = 0; + const languageServer = { port: 45678, connectionToken: "vscode-token", commit: COMMIT }; + const { host, ssh, reports } = hostFor(t, { attach: { code: 0, stdout: attachOutput(41234, "remote-token", { languageServer }) } }, async () => { + if (++calls === 1) return port; + asked.resolve(); + return chosen.promise; + }); + + host.start(); + await asked.promise; + await host.dispose(); + chosen.resolve((vscode.address() as AddressInfo).port); + await new Promise((resolve) => setTimeout(resolve, 20)); + + assert.equal(ssh.of("wb-test-a", "forward").length, 1); + assert.equal(probes, 0); + assert.ok(reports.every((report) => !report.endpoint)); +}); + +test("installs off: no probe, stage 1's attach through PATH, and not-installed with the version to install", async (t) => { + const { flow, prompts, runs } = await installFlow(t, "never"); + const { host, ssh, clock, last } = hostFor(t, { attach: { code: 127 } }, 1, "wb-test-a", "/tmp/wb-ssh-test", flow); + + host.start(); + await until(() => last()?.problem !== undefined); + + assert.deepEqual(last(), { + alias: "wb-test-a", + problem: { state: "not-installed", detail: "Whiteboard is not installed on wb-test-a. Install Whiteboard 0.1.6 there; Node 24 is needed." }, + }); + assert.equal(ssh.of("wb-test-a", "probe").length, 0); + assert.match(ssh.of("wb-test-a", "exec")[0].input!, /command -v whiteboard/); + assert.deepEqual([prompts.length, runs.length, clock.pending], [0, 0, 0]); +}); + +test("the version present: no prompt and no install shown; the installed CLI attaches by its path, with --replace", async (t) => { + const port = await healthServer(t); + const { flow, prompts, runs } = await installFlow(t, "ask"); + const { host, ssh, reports, last } = hostFor(t, { probe: { installed: [at("0.1.5"), at("0.1.6")] } }, port, "wb-test-a", "/tmp/wb-ssh-test", flow); + + host.start(); + await until(() => last()?.endpoint !== undefined); + + assert.equal(prompts.length, 0); + assert.equal(runs.length, 1); + assert.ok(reports.every((report) => !report.installing)); + assert.equal(ssh.of("wb-test-a", "probe").length, 1); + assert.equal( + ssh.of("wb-test-a", "exec")[0].input, + `exec '${INSTALLED.nodePath}' '${INSTALLED.cliPath}' remote attach --json --replace --groups go\n`, + ); +}); + +test("an installed host attaches again for its pending extensions by the installed CLI, with --replace and the groups enabled then", async (t) => { + const ports = [await healthServer(t), await versionServer(t, COMMIT)]; + const { flow } = await installFlow(t, "ask"); + let groups = ["go"]; + const { host, ssh, clock, last } = hostFor( + t, + { + probe: { installed: [at("0.1.6")] }, + attach: (call) => ({ code: 0, stdout: call === 1 ? pendingOutput(41234) : attachOutput(41234, "remote-token", { languageServer: { port: 45678, connectionToken: "vscode-token", commit: COMMIT } }) }), + }, + ports, + "wb-test-a", + "/tmp/wb-ssh-test", + flow, + undefined, + { groups: async () => groups }, + ); + + host.start(); + await until(() => last()?.endpoint !== undefined); + groups = ["go", "rust"]; + assert.ok(clock.next()); + await until(() => last()?.languageFeatures === true); + + const installed = `exec '${INSTALLED.nodePath}' '${INSTALLED.cliPath}' remote attach --json --replace`; + assert.deepEqual( + ssh.of("wb-test-a", "exec").map((call) => call.input), + [`${installed} --groups go\n`, `${installed} --groups go,rust\n`], + ); +}); + +test("the version present with another integrity is not installed: the user is asked and the install is shown", async (t) => { + const port = await healthServer(t); + const { flow, prompts, runs } = await installFlow(t, "ask", { answers: [true], steps: STEPS }); + const { host, reports, last } = hostFor(t, { probe: { installed: [at("0.1.5"), at("0.1.6", `sha512-${"B".repeat(86)}==`)] } }, port, "wb-test-a", "/tmp/wb-ssh-test", flow); + + host.start(); + await until(() => last()?.endpoint !== undefined); + + assert.equal(prompts.length, 1); + assert.equal(runs.length, 1); + assert.deepEqual(reports.find((report) => report.installing)?.installing, { step: "preparing" }); + assert.equal(await flow.consent.get("wb-test-a"), "allow"); +}); + +test("the version absent with installs always: each step is reported, then the host attaches", async (t) => { + const port = await healthServer(t); + const { flow, prompts, runs } = await installFlow(t, "always", { steps: STEPS }); + const { host, reports, last } = hostFor(t, {}, port, "wb-test-a", "/tmp/wb-ssh-test", flow); + + host.start(); + await until(() => last()?.endpoint !== undefined); + + assert.equal(prompts.length, 0); + assert.equal(runs.length, 1); + assert.equal(runs[0].version, "0.1.6"); + assert.equal(runs[0].target, "linux-arm64"); + assert.deepEqual( + reports.filter((report) => report.installing).map((report) => report.installing), + [ + { step: "preparing" }, + { step: "node", detail: "uploaded from this computer" }, + { step: "package", detail: "downloaded on the host" }, + { step: "verifying" }, + { step: "done" }, + ], + ); + assert.ok(reports.every((report) => !(report.installing && report.endpoint))); +}); + +test("the version absent, asked and declined: not-installed and declined, and no second prompt on reconnect", async (t) => { + const { flow, prompts, runs, consentFile } = await installFlow(t, "ask", { answers: [false] }); + const { host, ssh, clock, last } = hostFor(t, { attach: { code: 127 } }, 1, "wb-test-a", "/tmp/wb-ssh-test", flow); + + host.start(); + await until(() => last()?.problem !== undefined); + + assert.equal(prompts.length, 1); + assert.match(prompts[0].text, /Whiteboard 0\.1\.6 is not installed on wb-test-a/); + assert.match(prompts[0].text, /Install it in ~\/\.dev\/whiteboard-remote\?/); + assert.doesNotMatch(prompts[0].text, /\/home\/dev/); + assert.deepEqual(last(), { + alias: "wb-test-a", + problem: { state: "not-installed", detail: "Whiteboard is not installed on wb-test-a. Install Whiteboard 0.1.6 there; Node 24 is needed." }, + declined: true, + }); + assert.equal(clock.pending, 0); + assert.deepEqual(JSON.parse(await readFile(consentFile, "utf8")).aliases, { "wb-test-a": "deny" }); + + host.retry(); + await until(() => ssh.of("wb-test-a", "exec").length === 2 && last()?.problem !== undefined); + assert.equal(prompts.length, 1); + assert.equal(runs.length, 0); + assert.equal(last()?.declined, true); +}); + +test("a declined host with another version's CLI on PATH attaches it, and still offers the install", async (t) => { + const port = await healthServer(t); + const { flow, runs } = await installFlow(t, "ask", { answers: [false] }); + const { host, ssh, last } = hostFor(t, {}, port, "wb-test-a", "/tmp/wb-ssh-test", flow, "0.1.7"); + + host.start(); + await until(() => last()?.endpoint !== undefined); + + assert.equal(last()?.declined, true); + assert.equal(runs.length, 0); + assert.match(ssh.of("wb-test-a", "exec")[0].input!, /command -v whiteboard/); + assert.doesNotMatch(ssh.of("wb-test-a", "exec")[0].input!, /--replace/); + assert.equal(await flow.consent.get("wb-test-a"), "deny"); +}); + +test("this version's CLI on PATH counts as installed: no prompt, no install, and it attaches through PATH with --replace", async (t) => { + const port = await healthServer(t); + const { flow, prompts, runs } = await installFlow(t, "ask"); + const { host, ssh, last } = hostFor(t, { probe: { pathCli: { path: "/usr/local/bin/whiteboard", version: "0.1.6" } } }, port, "wb-test-a", "/tmp/wb-ssh-test", flow); + + host.start(); + await until(() => last()?.endpoint !== undefined); + + assert.deepEqual([prompts.length, runs.length], [0, 0]); + assert.equal(last()?.declined, undefined); + assert.match(ssh.of("wb-test-a", "exec")[0].input!, /command -v whiteboard[\s\S]*remote attach --json --replace/); +}); + +test("another version's CLI on PATH does not count: the user is asked", async (t) => { + const port = await healthServer(t); + const { flow, prompts } = await installFlow(t, "ask", { answers: [false] }); + const { host, last } = hostFor(t, { probe: { pathCli: { path: "/usr/local/bin/whiteboard", version: "0.1.5" } } }, port, "wb-test-a", "/tmp/wb-ssh-test", flow); + + host.start(); + await until(() => last()?.endpoint !== undefined); + + assert.equal(prompts.length, 1); + assert.equal(await flow.consent.get("wb-test-a"), "deny"); +}); + +test("an open install question is reported until it is answered", async (t) => { + const { flow } = await installFlow(t, "ask"); + const answer = Promise.withResolvers(); + flow.confirm = () => answer.promise; + const { host, reports, last } = hostFor(t, { attach: { code: 127 } }, 1, "wb-test-a", "/tmp/wb-ssh-test", flow); + + host.start(); + await until(() => last()?.asking !== undefined); + assert.deepEqual(last(), { alias: "wb-test-a", asking: "0.1.6" }); + + answer.resolve(false); + await until(() => last()?.problem !== undefined); + assert.deepEqual(reports.at(-2), { alias: "wb-test-a" }); +}); + +test("a quiesce closes an open install question, and one it lands before", async (t) => { + const { flow } = await installFlow(t, "ask"); + const answer = Promise.withResolvers(); + flow.confirm = () => answer.promise; + const { host, last } = hostFor(t, { attach: { code: 127 } }, 1, "wb-test-a", "/tmp/wb-ssh-test", flow); + + host.start(); + await until(() => last()?.asking !== undefined); + host.quiesce(); + answer.resolve(undefined); + await new Promise((resolve) => setTimeout(resolve, 20)); + + assert.deepEqual(last(), { alias: "wb-test-a" }); + + const reading = await installFlow(t, "ask"); + const read = Promise.withResolvers(); + const release = Promise.withResolvers(); + const get = reading.flow.consent.get.bind(reading.flow.consent); + reading.flow.consent.get = async (alias) => { + read.resolve(); + await release.promise; + return get(alias); + }; + const other = hostFor(t, { attach: { code: 127 } }, 1, "wb-test-a", "/tmp/wb-ssh-test", reading.flow); + + other.host.start(); + await read.promise; + other.host.quiesce(); + release.resolve(); + await new Promise((resolve) => setTimeout(resolve, 20)); + + assert.equal(reading.prompts.length, 0); + assert.equal(other.last()?.asking, undefined); +}); + +test("a prompt nobody answered is not remembered, offers Install, and the next connect asks again", async (t) => { + const { flow, prompts, consentFile } = await installFlow(t, "ask", { answers: [undefined, undefined] }); + const { host, ssh, last } = hostFor(t, { attach: { code: 127 } }, 1, "wb-test-a", "/tmp/wb-ssh-test", flow); + + host.start(); + await until(() => last()?.problem !== undefined); + assert.equal(last()?.problem?.state, "not-installed"); + assert.equal(last()?.declined, true); + await assert.rejects(readFile(consentFile), { code: "ENOENT" }); + host.retry(); + await until(() => ssh.of("wb-test-a", "exec").length === 2 && last()?.problem !== undefined); + + assert.equal(prompts.length, 2); +}); + +test("a host the user agreed to is remembered by its server id, so a Desktop update installs without asking", async (t) => { + const port = await healthServer(t); + const first = await installFlow(t, "ask", { answers: [true], steps: STEPS }); + const one = hostFor(t, {}, port, "wb-test-a", "/tmp/wb-ssh-test", first.flow); + + one.host.start(); + await until(() => one.last()?.endpoint !== undefined); + await one.host.dispose(); + assert.equal(await first.flow.consent.get("wb-test-a"), "allow"); + assert.equal(first.prompts.length, 1); + assert.deepEqual(JSON.parse(await readFile(first.consentFile, "utf8")), { servers: { [FAKE_SERVER_ID]: { consent: "allow", alias: "wb-test-a" } }, aliases: {} }); + + const second = await installFlow(t, "ask", { steps: STEPS, consentFile: first.consentFile }); + const two = hostFor(t, { probe: { installed: [at("0.1.6")] } }, port, "wb-test-a", "/tmp/wb-ssh-test", second.flow, "0.1.7"); + + two.host.start(); + await until(() => two.last()?.endpoint !== undefined); + assert.equal(second.prompts.length, 0); + assert.equal(second.runs[0].version, "0.1.7"); + assert.equal(await second.flow.consent.get("wb-test-a"), "allow"); +}); + +test("an unsupported host is reported with the reason, and nothing is installed or attached", async (t) => { + const { flow, prompts, runs } = await installFlow(t, "always"); + const { host, ssh, clock, last } = hostFor(t, { probe: { glibc: "2.31" } }, 1, "wb-test-a", "/tmp/wb-ssh-test", flow); + + host.start(); + await until(() => last()?.problem !== undefined); + + assert.deepEqual(last()?.problem, { state: "unsupported", detail: "This host runs glibc 2.31; Whiteboard needs 2.34 or newer." }); + assert.deepEqual([prompts.length, runs.length, ssh.of("wb-test-a", "exec").length, clock.pending], [0, 0, 0, 0]); +}); + +test("a server of another version the CLI started is replaced, and the host comes online", async (t) => { + const port = await healthServer(t); + const { flow } = await installFlow(t, "always"); + const { host, last } = hostFor( + t, + { attach: { code: 0, stdout: attachOutput(41234, "new-token", { replaced: true, previousVersion: "0.1.5", startedServer: true }) } }, + port, + "wb-test-a", + "/tmp/wb-ssh-test", + flow, + ); + + host.start(); + await until(() => last()?.endpoint !== undefined); + + assert.equal(last()?.endpoint?.token, "new-token"); +}); + +test("a server of another version a user started is left, and the host is incompatible until a retry", async (t) => { + const port = await healthServer(t); + const { flow } = await installFlow(t, "always"); + const { host, ssh, clock, last } = hostFor( + t, + { + attach: (call) => ({ + code: 0, + stdout: attachOutput(41234, "old-token", call === 1 ? { version: "0.1.5", startedServer: false, incompatibleRunning: { version: "0.1.5", pid: 4242, startedBy: "user" } } : {}), + }), + }, + port, + "wb-test-a", + "/tmp/wb-ssh-test", + flow, + ); + + host.start(); + await until(() => last()?.problem !== undefined); + + assert.deepEqual(last(), { + alias: "wb-test-a", + problem: { + state: "incompatible", + detail: "A Whiteboard server 0.1.5 started by a user is running on wb-test-a; stop it to use this Desktop's version.", + }, + }); + assert.equal(clock.pending, 0); + assert.equal(ssh.of("wb-test-a", "exec").length, 1); + host.retry(); + await until(() => last()?.endpoint !== undefined); + assert.equal(ssh.of("wb-test-a", "exec").length, 2); +}); + +test("a failed install names the step and the reason, is not retried alone, and Retry installs again", async (t) => { + const port = await healthServer(t); + const reason = "Installing on wb-test-a failed while installing the package: exit 1: \x1b[31mnpm ERR!\x1b[0m 404\r."; + const { flow, runs } = await installFlow(t, "always", { + steps: [{ step: "package", via: "upload" }], + fails: (call) => (call === 1 ? new Error(reason) : undefined), + }); + const { host, clock, last } = hostFor(t, {}, port, "wb-test-a", "/tmp/wb-ssh-test", flow); + + host.start(); + await until(() => last()?.problem !== undefined); + + assert.deepEqual(last()?.problem, { + state: "not-installed", + detail: "Installing Whiteboard 0.1.6 on wb-test-a failed while installing the package: exit 1: npm ERR! 404 .", + }); + assert.equal(clock.pending, 0); + host.retry(); + await until(() => last()?.endpoint !== undefined); + assert.equal(runs.length, 2); +}); + +test("an install whose connection dropped is unreachable, and is tried again after the backoff", async (t) => { + const { flow } = await installFlow(t, "always", { + steps: [{ step: "node", via: "remote-download" }], + fails: () => new Error("Installing on wb-test-a failed while unpacking Node: exit 255: Connection to 127.0.0.1 closed by remote host."), + }); + const { host, clock, last } = hostFor(t, {}, 1, "wb-test-a", "/tmp/wb-ssh-test", flow); + + host.start(); + await until(() => last()?.problem !== undefined); + + assert.deepEqual(last()?.problem, { + state: "unreachable", + detail: "The connection to wb-test-a dropped while installing Whiteboard 0.1.6 (installing Node).", + }); + assert.equal(clock.pending, 1); +}); + +test("a retry during an install aborts it", async (t) => { + const port = await healthServer(t); + const signals: AbortSignal[] = []; + const { flow } = await installFlow(t, "always"); + const run = flow.run; + flow.run = async (input) => { + signals.push(input.signal); + if (signals.length === 1) await new Promise((resolve) => input.signal.addEventListener("abort", resolve)); + input.signal.throwIfAborted(); + return run(input); + }; + const { host, last } = hostFor(t, {}, port, "wb-test-a", "/tmp/wb-ssh-test", flow); + + host.start(); + await until(() => signals.length === 1); + host.retry(); + await until(() => last()?.endpoint !== undefined); + + assert.equal(signals[0].aborted, true); + assert.equal(signals.length, 2); +}); + +test("a failed upgrade attaches the older version, which reports why, and Retry installs again", async (t) => { + const port = await healthServer(t); + const { flow, runs } = await installFlow(t, "always", { + steps: [{ step: "verifying" }], + fails: (call) => (call === 1 ? new Error("The package installed on wb-test-a reports version 0.1.5, not 0.1.6.") : undefined), + }); + const { host, ssh, clock, last } = hostFor(t, { probe: { installed: [at("0.1.5")] } }, port, "wb-test-a", "/tmp/wb-ssh-test", flow); + + host.start(); + await until(() => last()?.endpoint !== undefined); + + assert.deepEqual(last(), { + alias: "wb-test-a", + endpoint: { url: `http://127.0.0.1:${port}`, token: "remote-token" }, + ...NO_SERVER, + installFailure: "Installing Whiteboard 0.1.6 on wb-test-a failed while checking the install: The package installed on wb-test-a reports version 0.1.5, not 0.1.6.", + }); + assert.match(ssh.of("wb-test-a", "exec")[0].input!, /command -v whiteboard/); + assert.equal(clock.pending, 0); + host.retry(); + await until(() => last()?.endpoint !== undefined && !last()?.installFailure); + assert.equal(runs.length, 2); + assert.match(ssh.of("wb-test-a", "exec")[1].input!, /--replace/); +}); + +test("a newer server another Desktop started is left, and this Desktop is told to update", async (t) => { + const { flow } = await installFlow(t, "always"); + const { host, clock, last } = hostFor( + t, + { attach: { code: 0, stdout: attachOutput(41234, "t", { version: "0.1.7", startedServer: false, incompatibleRunning: { version: "0.1.7", pid: 77, startedBy: "desktop" } }) } }, + 1, + "wb-test-a", + "/tmp/wb-ssh-test", + flow, + ); + + host.start(); + await until(() => last()?.problem !== undefined); + + assert.deepEqual(last()?.problem, { + state: "incompatible", + detail: "A newer Whiteboard 0.1.7 is running on wb-test-a, started by another Desktop; update this Desktop to use it.", + }); + assert.equal(clock.pending, 0); +}); + +test("a server id that is not a UUID is not used for the install consent", async (t) => { + const port = await healthServer(t); + const { flow, consentFile } = await installFlow(t, "ask", { answers: [true] }); + const { host, last } = hostFor(t, { attach: { code: 0, stdout: attachOutput(41234, "t", { serverId: "__proto__" }) } }, port, "wb-test-a", "/tmp/wb-ssh-test", flow); + + host.start(); + await until(() => last()?.endpoint !== undefined); + + assert.equal(await flow.consent.get("wb-test-a"), "allow"); + assert.deepEqual(JSON.parse(await readFile(consentFile, "utf8")), { servers: {}, aliases: { "wb-test-a": "allow" } }); +}); + +test("a newer server the CLI started says so", async (t) => { + const { flow } = await installFlow(t, "always"); + const { host, last } = hostFor( + t, + { attach: { code: 0, stdout: attachOutput(41234, "t", { version: "0.1.7", incompatibleRunning: { version: "0.1.7", pid: 77, startedBy: "cli" } }) } }, + 1, + "wb-test-a", + "/tmp/wb-ssh-test", + flow, + ); + + host.start(); + await until(() => last()?.problem !== undefined); + + assert.equal(last()?.problem?.detail, "A newer Whiteboard 0.1.7 is running on wb-test-a, started by the CLI; update this Desktop to use it."); +}); + +test("agents are detected once after the first attach, with the installed CLI, and kept", async (t) => { + const port = await healthServer(t); + const { flow } = await installFlow(t, "always"); + const attached = new Set(); + const firstAttach = (key: string) => !attached.has(key) && !!attached.add(key); + const remote: FakeRemote = { detect: { code: 0, stdout: `noise\n${detectOutput([{ id: "pi", name: "Pi", connected: false }])}` } }; + const { host, ssh, last } = hostFor(t, remote, port, "wb-test-a", "/tmp/wb-ssh-test", flow, "0.1.6", { firstAttach }); + + host.start(); + await until(() => ssh.of("wb-test-a", "detect").length === 1); + + assert.ok(last()?.endpoint, "online before agents are read"); + const [detect] = ssh.of("wb-test-a", "detect"); + assert.match(detect.input!, new RegExp(`exec '${INSTALLED.nodePath}' '${INSTALLED.cliPath}' 'connect' '--detect' '--json'\n$`)); + assert.deepEqual(await host.detectAgents(), [{ id: "pi", connected: false }]); + assert.equal(ssh.of("wb-test-a", "detect").length, 1, "the kept answer"); + assert.equal(ssh.of("wb-test-a", "connect").length, 0, "nothing changed on the host"); + assert.deepEqual([...attached], [FAKE_SERVER_ID]); + + host.retry(); + await until(() => ssh.of("wb-test-a", "exec").length === 2); + await until(() => last()?.endpoint !== undefined); + await new Promise((resolve) => setTimeout(resolve, 20)); + assert.equal(ssh.of("wb-test-a", "detect").length, 1); +}); + +test("a hand-installed host detects with the CLI on its PATH", async (t) => { + const port = await healthServer(t); + const { host, ssh, last } = hostFor(t, {}, port, "wb-test-a", "/tmp/wb-ssh-test", undefined, "0.1.6", { firstAttach: () => true }); + + host.start(); + await until(() => last()?.endpoint !== undefined && ssh.of("wb-test-a", "detect").length === 1); + + assert.match(ssh.of("wb-test-a", "detect")[0].input!, /exec "\$wb" 'connect' '--detect' '--json'\n$/); + assert.deepEqual(await host.detectAgents(), []); +}); + +test("connecting runs the agents' commands with the installed CLI, and refuses agents it did not find or cannot connect alone", async (t) => { + const port = await healthServer(t); + const { flow } = await installFlow(t, "always"); + const remote: FakeRemote = { + detect: { + code: 0, + stdout: detectOutput([ + { id: "pi", connected: false }, + { id: "codex", connected: false, manual: true }, + ]), + }, + }; + const { host, ssh, last } = hostFor(t, remote, port, "wb-test-a", "/tmp/wb-ssh-test", flow); + + host.start(); + await until(() => last()?.endpoint !== undefined); + + for (const ids of [["cursor"], ["codex"], ["claude"], ["__proto__"], ["pi", "x".repeat(10)]]) { + await assert.rejects(host.connectAgents(ids), /Whiteboard cannot connect .* on wb-test-a/); + } + assert.equal(ssh.of("wb-test-a", "connect").length, 0); + + assert.deepEqual(await host.connectAgents(["pi", "pi"]), [{ id: "pi", connected: true, output: "" }]); + const [connect] = ssh.of("wb-test-a", "connect"); + assert.match(connect.input!, new RegExp(`exec '${INSTALLED.nodePath}' '${INSTALLED.cliPath}' 'connect' '--yes' '--json' 'pi'\n$`)); + assert.deepEqual(await host.detectAgents(), [ + { id: "codex", connected: false, manual: true }, + { id: "pi", connected: true }, + ]); +}); + +test("a connect that prints no result fails with ssh's words, as plain text", async (t) => { + const port = await healthServer(t); + const remote: FakeRemote = { + detect: { code: 0, stdout: detectOutput([{ id: "pi", connected: false }]) }, + connect: { code: 1, stderr: "\u001b[31mboom\u001b[0m\r\nmore\n" }, + }; + const { host, last } = hostFor(t, remote, port, "wb-test-a", "/tmp/wb-ssh-test", undefined, "0.1.6", { firstAttach: () => true }); + + host.start(); + await until(() => last()?.endpoint !== undefined); + + await assert.rejects(host.connectAgents(["pi"]), (error: Error) => { + assert.equal(error.message, "Connecting agents on wb-test-a failed: boom more"); + return true; + }); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteHost.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteHost.ts new file mode 100644 index 000000000..2fe6c0e06 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteHost.ts @@ -0,0 +1,838 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { rm } from "node:fs/promises"; +import { get } from "node:http"; +import type { Readable, Writable } from "node:stream"; +import type { ReviewGatewayHost, ReviewRemoteAgent, ReviewRemoteAgentResult } from "../../common/reviewProtocol.js"; +import { isRemoteAgentId, parseRemoteAgents, parseRemoteConnect, plainText as plain, remoteConnectScript, type ReviewRemoteCli } from "./reviewRemoteAgents.js"; +import { installedAttachScript, parseRemoteAttach, reviewRemoteAttachScript, type ReviewRemoteAttach } from "./reviewRemoteAttachScript.js"; +import type { ReviewRemoteInstallConsent } from "./reviewRemoteInstallConsent.js"; +import type { ReviewRemoteInstallInput, ReviewRemoteInstallResult } from "./reviewRemoteInstaller.js"; +import { judgeRemote, probeRemote, type ReviewRemoteProbe, type ReviewRemoteTarget } from "./reviewRemoteProbe.js"; +import { + sshCancelForwardArgs, + sshCheckArgs, + sshCloseArgs, + sshExecArgs, + sshForwardArgs, + sshMasterArgs, + type ReviewSshSession, +} from "./reviewSshCommand.js"; + +export interface SshChildProcess { + readonly pid?: number; + readonly exitCode: number | null; + readonly signalCode: NodeJS.Signals | null; + readonly stdin: Writable | null; + readonly stdout: Readable | null; + readonly stderr: Readable | null; + kill(signal?: NodeJS.Signals): boolean; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + once(event: string, listener: (...args: any[]) => void): unknown; +} + +/** Spawns `ssh` with `args`. Every call is detached: without a terminal, ssh can only prompt through askpass. */ +export type SpawnSsh = ( + args: string[], + options: { env: NodeJS.ProcessEnv; detached: true; stdio: ["pipe" | "ignore", "pipe" | "ignore", "pipe"] }, +) => SshChildProcess; + +export interface ReviewRemoteClock { + now(): number; + schedule(ms: number, run: () => void): () => void; +} + +export const systemClock: ReviewRemoteClock = { + now: () => Date.now(), + schedule(ms, run) { + const timer = setTimeout(run, ms); + return () => clearTimeout(timer); + }, +}; + +export const REVIEW_REMOTE_TIMEOUTS = { + poll: 250, + connect: 30_000, + attach: 60_000, + operation: 10_000, + close: 2_000, + agents: 10_000, + agentConnect: 130_000, + stable: 30_000, +}; + +const FIRST_DELAY_MS = 1_000; +const PENDING_REATTACH_MS = 60_000; +const PENDING_ATTACHES = 10; +const MAX_DELAY_MS = 60_000; +const OUTPUT_LIMIT = 64 * 1024; + +export function reconnectDelay(failures: number, random: () => number = Math.random): number { + const base = Math.min(FIRST_DELAY_MS * 2 ** failures, MAX_DELAY_MS); + return Math.round(Math.min(MAX_DELAY_MS, Math.max(FIRST_DELAY_MS, base * (0.75 + 0.5 * random())))); +} + +export function classifySshFailure(stderr: string, promptCancelled: boolean): "auth-failed" | "unreachable" { + return promptCancelled || /Permission denied|Host key verification failed|REMOTE HOST IDENTIFICATION HAS CHANGED/.test(stderr) + ? "auth-failed" + : "unreachable"; +} + +export const OPENSSH_NEEDED = "OpenSSH is needed: no `ssh` command was found on PATH."; + +type Problem = NonNullable; + +class HostFailure extends Error { + constructor(readonly problem: Problem) { + super(problem.detail); + } +} + +const unreachable = (detail: string) => new HostFailure({ state: "unreachable", detail }); + +function meaningfulLines(text: string): string[] { + return ( + text + .split("\n") + .map((line) => line.trim()) + .filter((line) => line && !line.startsWith("Warning: Permanently added")) + ); +} + +function firstLines(text: string, count = 6): string { + return meaningfulLines(text).slice(0, count).join("\n").slice(0, 1000); +} + +function lastLines(text: string, count = 6): string { + return meaningfulLines(text).slice(-count).join("\n").slice(-1000); +} + +const gone = (child: SshChildProcess) => child.exitCode !== null || child.signalCode !== null; + +function exitedWithin(child: SshChildProcess, ms: number): Promise { + if (gone(child)) return Promise.resolve(true); + return new Promise((resolve) => { + const timer = setTimeout(() => resolve(false), ms); + child.once("exit", () => { + clearTimeout(timer); + resolve(true); + }); + }); +} + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +function probeHealth(port: number, timeout: number): Promise { + return new Promise((resolve, reject) => { + const request = get({ host: "127.0.0.1", port, path: "/health", timeout }, (response) => { + response.resume(); + if (response.statusCode === 200) resolve(); + else reject(new Error(`/health answered ${response.statusCode}`)); + }); + request.on("timeout", () => request.destroy(new Error(`/health did not answer within ${timeout / 1000} seconds`))); + request.on("error", reject); + }); +} + +function probeVersion(port: number, timeout: number): Promise { + return new Promise((resolve, reject) => { + const request = get({ host: "127.0.0.1", port, path: "/version", timeout }, (response) => { + let body = ""; + response.setEncoding("utf8"); + response.on("data", (chunk: string) => (body = (body + chunk).slice(0, 200))); + response.on("end", () => (response.statusCode === 200 ? resolve(body.trim()) : reject(new Error(`/version answered ${response.statusCode}`)))); + }); + request.on("timeout", () => request.destroy(new Error(`/version did not answer within ${timeout / 1000} seconds`))); + request.on("error", reject); + }); +} + +export function languageCommitMismatch(alias: string, serverCommit: string, desktopCommit: string | undefined): string | undefined { + if (!desktopCommit || desktopCommit === serverCommit) return undefined; + return `language features need the same Whiteboard version on ${alias}: it runs ${serverCommit.slice(0, 7)}, this Desktop ${desktopCommit.slice(0, 7)}`; +} + +type LanguageFeatures = Pick; + +const groupsOf = ({ languageGroups }: ReviewRemoteAttach): Pick => + languageGroups.length > 0 ? { languageGroups: [...languageGroups] } : {}; + +export interface RunResult { + readonly code: number | null; + readonly stdout: string; + readonly stderr: string; + readonly error?: NodeJS.ErrnoException; + readonly timedOut: boolean; +} + +export function runSsh(spawn: SpawnSsh, env: NodeJS.ProcessEnv, args: string[], timeout: number, input?: string): Promise { + return new Promise((resolve) => { + let child: SshChildProcess; + try { + child = spawn(args, { env, detached: true, stdio: [input === undefined ? "ignore" : "pipe", "pipe", "pipe"] }); + } catch (error) { + return resolve({ code: null, stdout: "", stderr: "", error: error as NodeJS.ErrnoException, timedOut: false }); + } + let stdout = ""; + let stderr = ""; + let timedOut = false; + const done = (result: Omit) => { + clearTimeout(timer); + resolve({ ...result, stdout, stderr, timedOut }); + }; + const timer = setTimeout(() => { + timedOut = true; + child.kill("SIGKILL"); + done({ code: null }); + }, timeout); + child.stdout?.setEncoding("utf8"); + child.stdout?.on("data", (chunk: string) => (stdout = (stdout + chunk).slice(-OUTPUT_LIMIT))); + child.stderr?.setEncoding("utf8"); + child.stderr?.on("data", (chunk: string) => (stderr = (stderr + chunk).slice(0, OUTPUT_LIMIT))); + child.once("error", (error: NodeJS.ErrnoException) => done({ code: null, error })); + child.once("close", (code: number | null) => done({ code })); + if (input !== undefined) { + child.stdin?.on("error", () => {}); + child.stdin?.end(input); + } + }); +} + +export type ReviewRemoteInstallMode = "ask" | "always" | "never"; + +export type ReviewRemoteInstallRunInput = Pick; + +export interface ReviewRemoteInstallFlow { + mode(): ReviewRemoteInstallMode; + readonly consent: ReviewRemoteInstallConsent; + confirm(request: { alias: string; text: string; signal?: AbortSignal }): Promise; + cancel?(alias: string): void; + run(input: ReviewRemoteInstallRunInput): Promise; + integrity(): Promise; +} + +type InstallStep = NonNullable["step"]; + +type AttachScript = (groups: readonly string[]) => string; + +interface PreparedAttach { + readonly script: AttachScript; + readonly cli?: ReviewRemoteCli; +} + +const STEP_WORDS: Record = { + preparing: "preparing", + "waiting-for-lock": "waiting for another install", + node: "installing Node", + package: "installing the package", + verifying: "checking the install", + done: "finishing", +}; + +const VIA = { "remote-download": "downloaded on the host", upload: "uploaded from this computer" } as const; + +export function installPromptText(alias: string, version: string, probe: ReviewRemoteProbe): string { + const node = probe.node || probe.managedNode ? "" : `, and about 200 MB for Node 24, which ${alias} does not have`; + const where = probe.root === `${probe.home}/.dev/whiteboard-remote` ? "~/.dev/whiteboard-remote" : "whiteboard-remote under DEV_REVIEW_HOME"; + return `Whiteboard ${version} is not installed on ${alias}. Install it in ${where}? It takes about 140 MB${node}. Whiteboard also adds ~/.local/bin/whiteboard if that path is free.`; +} + +export function installFailureText(alias: string, version: string, step: InstallStep, message: string): string { + const said = /^Installing on .+? failed( while .+?)?: ([\s\S]*)$/.exec(message); + const what = said ? (said[1] ?? "") : ` while ${STEP_WORDS[step]}`; + return plain(`Installing Whiteboard ${version} on ${alias} failed${what}: ${said ? said[2] : message}`); +} + +export interface ReviewRemoteHostOptions { + readonly session: ReviewSshSession; + readonly spawn: SpawnSsh; + environment(): Promise; + desktopVersion(): Promise; + readonly desktopCommit?: string; + groups?(): Promise; + freePort(): Promise; + report(host: ReviewGatewayHost): void; + log(message: string): void; + readonly clock?: ReviewRemoteClock; + readonly timeouts?: Partial; + readonly random?: () => number; + readonly install?: ReviewRemoteInstallFlow; + firstAttach?(key: string): boolean; +} + +export class ReviewRemoteHost { + private readonly alias: string; + private readonly clock: ReviewRemoteClock; + private readonly timeouts: typeof REVIEW_REMOTE_TIMEOUTS; + private reported: ReviewGatewayHost; + private generation = 0; + private master: SshChildProcess | undefined; + private readonly closing = new Map>(); + private forwarded: { local: number; remote: number } | undefined; + private language: { local: number; remote: number; connectionToken: string; commit: string } | undefined; + private pendingAttaches = 0; + private cancelPending: (() => void) | undefined; + private reattaching = false; + private queuedReattach = false; + private masterStderr = ""; + private env: NodeJS.ProcessEnv | undefined; + private connectedAt: number | undefined; + private failures = 0; + private cancelTimer: (() => void) | undefined; + private prompts = 0; + private promptCancelled = false; + private disposed = false; + private attachScript: AttachScript = reviewRemoteAttachScript; + private cli: ReviewRemoteCli; + private agents: ReviewRemoteAgent[] | undefined; + private detecting: Promise | undefined; + private declined = false; + private installFailure: string | undefined; + private installing: AbortController | undefined; + private removing = false; + serverId: string | undefined; + + constructor(private readonly options: ReviewRemoteHostOptions) { + this.alias = options.session.alias; + this.clock = options.clock ?? systemClock; + this.timeouts = { ...REVIEW_REMOTE_TIMEOUTS, ...options.timeouts }; + this.reported = { alias: this.alias }; + } + + get state(): ReviewGatewayHost { + return this.reported; + } + + start(): void { + void this.connect(); + } + + retry(): void { + if (this.disposed || this.removing) return; + this.failures = 0; + this.pendingAttaches = 0; + this.declined = false; + this.installFailure = undefined; + this.set({ alias: this.alias }); + void this.connect(); + } + + async resume(): Promise { + if (this.disposed || this.removing) return; + if (this.cancelTimer) return void this.connect(); + const master = this.master; + const port = this.forwarded?.local; + if (!master || this.connectedAt === undefined || port === undefined) return; + try { + await probeHealth(port, this.timeouts.operation); + return; + } catch (error) { + if (master !== this.master || this.disposed) return; + this.options.log(`${this.alias}: no answer through the forward after resume (${(error as Error).message}); reconnecting.`); + } + void this.connect(); + } + + reattach(): Promise { + if (this.disposed || this.removing || this.cancelTimer || !this.master || this.connectedAt === undefined) return Promise.resolve(); + if (this.reattaching) { + this.queuedReattach = true; + return Promise.resolve(); + } + const attempt = this.nextAttempt(); + if (attempt === 0) return this.attachAgain("its server restarted", false); + const delay = reconnectDelay(attempt - 1, this.options.random); + this.options.log(`${this.alias}: its server restarted again; attaching again in ${Math.round(delay / 1000)} s.`); + this.cancelTimer = this.clock.schedule(delay, () => { + this.cancelTimer = undefined; + void this.attachAgain("its server restarted", false); + }); + return Promise.resolve(); + } + + private async attachAgain(reason: string, reuse: boolean): Promise { + const env = this.env; + if (this.disposed || this.removing || this.reattaching || !this.master || this.connectedAt === undefined || !env || !this.forwarded) return; + const generation = this.generation; + const stale = () => generation !== this.generation || this.disposed; + this.reattaching = true; + this.options.log(`${this.alias}: ${reason}; attaching again.`); + try { + const check = await this.run(sshCheckArgs(this.options.session, env), this.timeouts.operation); + if (stale()) return; + if (check.code !== 0) throw unreachable(`The SSH connection to ${this.alias} did not answer. ${firstLines(check.stderr)}`.trim()); + await this.establish(env, stale, { script: this.attachScript, cli: this.cli }, reuse); + } catch (error) { + if (stale()) return; + this.fail(error instanceof HostFailure ? error.problem : { state: "unreachable", detail: (error as Error).message }); + } finally { + this.reattaching = false; + if (this.queuedReattach) { + this.queuedReattach = false; + void this.reattach(); + } + } + } + + quiesce(): void { + this.removing = true; + this.generation++; + this.cancelTimer?.(); + this.cancelTimer = undefined; + this.cancelPending?.(); + this.cancelPending = undefined; + this.installing?.abort(); + this.options.install?.cancel?.(this.alias); + if (this.reported.asking) this.set({ alias: this.alias }); + } + + get quiesced(): boolean { + return this.removing; + } + + unquiesce(): void { + this.removing = false; + this.retry(); + } + + promptOpened(): void { + this.prompts++; + } + + promptClosed(answered: boolean): void { + this.prompts = Math.max(0, this.prompts - 1); + if (!answered) this.promptCancelled = true; + } + + async dispose(): Promise { + this.disposed = true; + this.generation++; + this.options.install?.cancel?.(this.alias); + this.cancelTimer?.(); + this.cancelTimer = undefined; + this.cancelPending?.(); + this.cancelPending = undefined; + this.installing?.abort(); + const master = this.master; + this.master = undefined; + await this.close(master); + } + + async languageEndpoint(serverId: string): Promise<{ host: "127.0.0.1"; port: number; connectionToken: string } | undefined> { + const language = this.language; + if (!language || this.connectedAt === undefined || this.serverId !== serverId || this.reported.languageFeatures !== true) return undefined; + const commit = await probeVersion(language.local, this.timeouts.operation).catch(() => undefined); + if (commit === language.commit) return { host: "127.0.0.1", port: language.local, connectionToken: language.connectionToken }; + if (language === this.language) { + void this.attachAgain("its VS Code server did not answer", true); + } + return undefined; + } + + private whilePending(attach: ReviewRemoteAttach): void { + this.cancelPending?.(); + this.cancelPending = undefined; + if (!attach.languageServerPending) { + this.pendingAttaches = 0; + return; + } + if (++this.pendingAttaches >= PENDING_ATTACHES) return; + this.cancelPending = this.clock.schedule(PENDING_REATTACH_MS, () => { + this.cancelPending = undefined; + void this.attachAgain("its language extensions were installing", true); + }); + } + + killNow(): void { + for (const master of [this.master, ...this.closing.keys()]) master?.kill(); + } + + private set(state: ReviewGatewayHost): void { + if (JSON.stringify(state) === JSON.stringify(this.reported)) return; + this.reported = state; + this.options.log( + `${this.alias}: ${state.endpoint ? `online through ${state.endpoint.url}` : state.problem ? `${state.problem.state}: ${state.problem.detail}` : "connecting"}`, + ); + this.options.report(state); + } + + private async connect(): Promise { + if (this.removing) return; + const generation = ++this.generation; + const stale = () => generation !== this.generation || this.disposed; + this.cancelTimer?.(); + this.cancelTimer = undefined; + this.cancelPending?.(); + this.cancelPending = undefined; + this.queuedReattach = false; + this.dropMaster(); + this.connectedAt = undefined; + this.promptCancelled = false; + this.declined = false; + this.installFailure = undefined; + this.agents = undefined; + try { + const env = await this.options.environment(); + if (stale()) return; + this.env = env; + await Promise.all(this.closing.values()); + // None of this host's masters runs now, but one killed by a signal left its socket, and ssh does not multiplex on a path that exists. + await rm(this.options.session.controlPath, { force: true }); + if (stale()) return; + const master = this.startMaster(env); + await this.waitForMaster(master, env, stale); + const prepared = await this.prepareAttach(env, stale); + if (prepared === undefined) return; + await this.establish(env, stale, prepared); + if (stale()) return; + const serverId = this.serverId; + if (serverId) void this.options.install?.consent.attached(this.alias, serverId).catch((error: Error) => this.options.log(`${this.alias}: could not keep its install consent: ${error.message}`)); + if (this.options.firstAttach?.(serverId ?? `alias:${this.alias}`)) void this.detectAgents(); + } catch (error) { + if (stale()) return; + this.fail(error instanceof HostFailure ? error.problem : { state: "unreachable", detail: (error as Error).message }); + } + } + + private async establish(env: NodeJS.ProcessEnv, stale: () => boolean, prepared: PreparedAttach, reuse = false): Promise { + const old = this.forwarded; + const oldLanguage = this.language; + const attach = await this.attach(env, prepared.script); + if (stale()) return; + const kept = reuse && old !== undefined && attach.port === old.remote && (await probeHealth(old.local, this.timeouts.operation).then(() => true, () => false)); + if (stale()) return; + if (kept) this.forwarded = old; + const url = kept && old ? `http://127.0.0.1:${old.local}` : await this.forward(env, attach, stale); + if (stale()) return; + const language = await this.forwardLanguage(env, attach, stale); + if (stale()) return; + for (const forward of [kept ? undefined : old, oldLanguage]) { + if (forward) await this.run(sshCancelForwardArgs(this.options.session, forward.local, forward.remote, env), this.timeouts.operation); + if (stale()) return; + } + this.attachScript = prepared.script; + this.cli = prepared.cli; + this.connectedAt = this.clock.now(); + this.serverId = attach.serverId ?? undefined; + this.masterStderr = ""; + this.set({ alias: this.alias, endpoint: { url, token: attach.token }, ...language, ...groupsOf(attach), ...this.facts() }); + this.whilePending(attach); + } + + private nextAttempt(): number { + if (this.connectedAt !== undefined && this.clock.now() - this.connectedAt >= this.timeouts.stable) this.failures = 0; + return this.failures++; + } + + private startMaster(env: NodeJS.ProcessEnv): SshChildProcess { + let master: SshChildProcess; + try { + master = this.options.spawn(sshMasterArgs(this.options.session, env), { + env, + detached: true, + stdio: ["ignore", "ignore", "pipe"], + }); + } catch (error) { + throw (error as NodeJS.ErrnoException).code === "ENOENT" ? unreachable(OPENSSH_NEEDED) : error; + } + this.master = master; + this.masterStderr = ""; + master.stderr?.setEncoding("utf8"); + master.stderr?.on("data", (chunk: string) => (this.masterStderr = (this.masterStderr + chunk).slice(-OUTPUT_LIMIT))); + master.once("error", (error: NodeJS.ErrnoException) => this.masterGone(master, null, error)); + // "close", not "exit": OpenSSH's last lines can arrive after the exit. + master.once("close", (code: number | null) => this.masterGone(master, code)); + this.options.log(`${this.alias}: connecting, ssh pid ${master.pid}`); + return master; + } + + private masterGone(master: SshChildProcess, code: number | null, error?: NodeJS.ErrnoException): void { + if (master !== this.master) return; + this.master = undefined; + this.options.install?.cancel?.(this.alias); + if (error?.code === "ENOENT") return this.fail({ state: "unreachable", detail: OPENSSH_NEEDED }); + const exited = error?.message || `ssh exited with code ${code ?? "none"}.`; + if (this.connectedAt !== undefined) { + return this.fail({ + state: "unreachable", + detail: `The SSH connection to ${this.alias} ended: ${lastLines(this.masterStderr) || exited}`, + }); + } + this.fail({ + state: classifySshFailure(this.masterStderr, this.promptCancelled), + detail: firstLines(this.masterStderr) || exited, + }); + } + + private async waitForMaster(master: SshChildProcess, env: NodeJS.ProcessEnv, stale: () => boolean): Promise { + let since = Date.now(); + for (;;) { + if (stale() || gone(master)) throw unreachable("The SSH connection ended."); + const check = await this.run(sshCheckArgs(this.options.session, env), this.timeouts.operation); + if (stale()) throw unreachable("The SSH connection ended."); + if (check.code === 0) return; + if (this.prompts > 0) since = Date.now(); + if (Date.now() - since > this.timeouts.connect) { + throw unreachable(`${this.alias} did not connect within ${this.timeouts.connect / 1000} seconds. ${firstLines(this.masterStderr)}`.trim()); + } + await sleep(this.timeouts.poll); + } + } + + private async prepareAttach(env: NodeJS.ProcessEnv, stale: () => boolean): Promise { + const onPath = { script: reviewRemoteAttachScript }; + const flow = this.options.install; + const mode = flow?.mode() ?? "never"; + if (!flow || mode === "never") return onPath; + const probed = await probeRemote({ session: this.options.session, spawn: this.options.spawn, env }); + if (stale()) return; + if ("error" in probed) throw unreachable(probed.error); + const support = judgeRemote(probed.probe); + if (!support.supported) throw new HostFailure({ state: "unsupported", detail: support.reason }); + const version = await this.options.desktopVersion(); + const integrity = probed.probe.installed.some((entry) => entry.version === version) + ? await flow.integrity().catch((error: Error) => void this.options.log(`${this.alias}: no package integrity to compare: ${error.message}`)) + : undefined; + if (stale()) return; + const present = probed.probe.installed.some((entry) => entry.version === version && entry.integrity === integrity); + if (!present && probed.probe.pathCli?.version === version) return { script: (groups) => reviewRemoteAttachScript(groups, true) }; + if (!present && mode === "ask" && !(await this.agreed(flow, probed.probe, version, stale))) { + this.declined = true; + return stale() ? undefined : onPath; + } + if (stale()) return; + const installed = await this.install(flow, env, probed.probe, support.target, version, present, stale); + if (!installed) return undefined; + if ("path" in installed) { + const { nodePath, cliPath } = installed.path; + return { script: (groups) => installedAttachScript(nodePath, cliPath, groups), cli: { nodePath, cliPath } }; + } + if (!probed.probe.installed.some((other) => other.version !== version)) throw new HostFailure({ state: "not-installed", detail: installed.failed }); + this.installFailure = installed.failed; + return onPath; + } + + private async agreed(flow: ReviewRemoteInstallFlow, probe: ReviewRemoteProbe, version: string, stale: () => boolean): Promise { + const log = (error: Error) => this.options.log(`${this.alias}: install consent: ${error.message}`); + const stored = await flow.consent.get(this.alias).catch(log); + if (stored) return stored === "allow"; + if (stale()) return false; + this.set({ alias: this.alias, asking: version }); + const answer = await flow.confirm({ alias: this.alias, text: installPromptText(this.alias, version, probe) }); + if (!stale()) this.set({ alias: this.alias }); + if (answer !== undefined && !stale()) await flow.consent.set(this.alias, answer ? "allow" : "deny").catch(log); + return answer === true; + } + + private async install( + flow: ReviewRemoteInstallFlow, + env: NodeJS.ProcessEnv, + probe: ReviewRemoteProbe, + target: ReviewRemoteTarget, + version: string, + present: boolean, + stale: () => boolean, + ): Promise<{ path: ReviewRemoteInstallResult } | { failed: string } | undefined> { + const abort = new AbortController(); + this.installing = abort; + let step: InstallStep = "preparing"; + if (!present) this.set({ alias: this.alias, installing: { step } }); + try { + const path = await flow.run({ + session: this.options.session, + probe, + target, + version, + spawn: this.options.spawn, + env, + signal: abort.signal, + onProgress: (progress) => { + if (stale()) return; + step = progress.step; + if (step === "done" && !this.reported.installing) return; + this.set({ alias: this.alias, installing: { step, ...("via" in progress && { detail: VIA[progress.via] }) } }); + }, + }); + return { path }; + } catch (error) { + if (stale()) return undefined; + const message = (error as Error).message; + // ssh exits 255 when the connection drops. + if (/\bexit 255\b/.test(message) || !this.master || gone(this.master)) { + throw unreachable(`The connection to ${this.alias} dropped while installing Whiteboard ${version} (${STEP_WORDS[step]}).`); + } + return { failed: installFailureText(this.alias, version, step, message) }; + } finally { + if (this.installing === abort) this.installing = undefined; + } + } + + detectAgents(): Promise { + if (this.agents) return Promise.resolve(this.agents); + return (this.detecting ??= this.readAgents().finally(() => (this.detecting = undefined))); + } + + private async readAgents(): Promise { + const env = this.env; + if (this.disposed || !this.master || this.connectedAt === undefined || !env) return undefined; + const generation = this.generation; + const result = await this.run(sshExecArgs(this.options.session, env), this.timeouts.agents, remoteConnectScript(this.cli, ["--detect", "--json"])); + if (generation !== this.generation || this.disposed) return undefined; + const agents = parseRemoteAgents(result.stdout); + if (!agents) { + this.options.log(`${this.alias}: could not read its agents: ${result.timedOut ? `no answer within ${this.timeouts.agents / 1000} seconds` : plain(firstLines(result.stderr)) || `exit ${result.code ?? "none"}`}.`); + return undefined; + } + return (this.agents = agents); + } + + async connectAgents(ids: readonly unknown[]): Promise { + const known = await this.detectAgents(); + if (!known) throw new Error(`${this.alias} is not connected, or its agents could not be read.`); + const wanted = [...new Set(ids)]; + for (const id of wanted) { + if (!isRemoteAgentId(id) || !known.some((agent) => agent.id === id && !agent.manual)) { + throw new Error(`Whiteboard cannot connect ${isRemoteAgentId(id) ? id : "that agent"} on ${this.alias}.`); + } + } + const env = this.env; + if (!wanted.length) return []; + if (!this.master || !env) throw new Error(`The connection to ${this.alias} dropped; connect again to connect its agents.`); + const generation = this.generation; + const result = await this.run(sshExecArgs(this.options.session, env), this.timeouts.agentConnect * wanted.length, remoteConnectScript(this.cli, ["--yes", "--json", ...(wanted as string[])])); + const results = parseRemoteConnect(result.stdout); + if (!results) { + throw new Error( + `Connecting agents on ${this.alias} failed: ${result.timedOut ? "it did not finish in time" : plain(firstLines(result.stderr)) || `exit ${result.code ?? "none"}`}`, + ); + } + if (generation === this.generation && this.agents) { + this.agents = this.agents.map((agent) => ({ ...agent, connected: results.find((done) => done.id === agent.id)?.connected ?? agent.connected })); + } + return results; + } + + private async attach(env: NodeJS.ProcessEnv, script: AttachScript): Promise { + const groups = (await this.options.groups?.()) ?? []; + const result = await this.run(sshExecArgs(this.options.session, env), this.timeouts.attach, script(groups)); + const parsed = parseRemoteAttach(result.stdout); + if (parsed && "attach" in parsed) { + const { incompatibleRunning, replaced } = parsed.attach; + if (incompatibleRunning) { + throw new HostFailure({ + state: "incompatible", + detail: + incompatibleRunning.startedBy === "user" + ? `A Whiteboard server ${incompatibleRunning.version} started by a user is running on ${this.alias}; stop it to use this Desktop's version.` + : `A newer Whiteboard ${incompatibleRunning.version} is running on ${this.alias}, started by ${incompatibleRunning.startedBy === "cli" ? "the CLI" : "another Desktop"}; update this Desktop to use it.`, + }); + } + if (replaced) this.options.log(`${this.alias}: replaced its Whiteboard server ${replaced} with ${parsed.attach.version}.`); + const languageGroups = parsed.attach.languageGroups.filter( + (entry, index, all) => groups.includes(entry.group) && all.findIndex((other) => other.group === entry.group) === index, + ); + return { ...parsed.attach, languageGroups }; + } + if (parsed) throw unreachable(`whiteboard remote attach failed on ${this.alias}: ${parsed.error}`); + if (result.timedOut) throw unreachable(`whiteboard remote attach on ${this.alias} did not finish within ${this.timeouts.attach / 1000} seconds.`); + if (result.code === 127) { + const version = await this.options.desktopVersion(); + throw new HostFailure({ + state: "not-installed", + detail: this.installFailure ?? `Whiteboard is not installed on ${this.alias}. Install Whiteboard ${version} there; Node 24 is needed.`, + }); + } + throw unreachable(firstLines(result.stderr) || `whiteboard remote attach on ${this.alias} exited with code ${result.code ?? "none"}.`); + } + + private async forward(env: NodeJS.ProcessEnv, attach: ReviewRemoteAttach, stale: () => boolean): Promise { + this.forwarded = undefined; + const port = await this.options.freePort(); + if (stale()) throw unreachable("The SSH connection ended."); + const forward = await this.run(sshForwardArgs(this.options.session, port, attach.port, env), this.timeouts.operation); + if (forward.code !== 0) throw unreachable(`Could not forward a local port to ${this.alias}: ${firstLines(forward.stderr) || `ssh exited with code ${forward.code}`}`); + try { + await probeHealth(port, this.timeouts.operation); + } catch (error) { + const refused = /^.*administratively prohibited.*$/m.exec(this.masterStderr)?.[0]; + throw unreachable(refused ? `${this.alias} refused the port forward: ${refused}` : `The Whiteboard server on ${this.alias} did not answer through the forward: ${(error as Error).message}.`); + } + this.forwarded = { local: port, remote: attach.port }; + return `http://127.0.0.1:${port}`; + } + + private async forwardLanguage(env: NodeJS.ProcessEnv, attach: ReviewRemoteAttach, stale: () => boolean): Promise { + this.language = undefined; + const server = attach.languageServer; + const unavailable = (detail: string): LanguageFeatures => ({ languageFeatures: false, languageFeaturesDetail: detail }); + if (!server) return unavailable(attach.languageServerDetail ?? "This host has no VS Code server."); + const mismatch = languageCommitMismatch(this.alias, server.commit, this.options.desktopCommit); + if (mismatch) return unavailable(mismatch); + const port = await this.options.freePort(); + if (stale()) throw unreachable("The SSH connection ended."); + const forward = await this.run(sshForwardArgs(this.options.session, port, server.port, env), this.timeouts.operation); + if (forward.code !== 0) return unavailable(`Could not forward a local port to the VS Code server on ${this.alias}: ${firstLines(forward.stderr) || `ssh exited with code ${forward.code}`}`); + try { + const commit = await probeVersion(port, this.timeouts.operation); + if (commit !== server.commit) throw new Error(`it reports ${commit.slice(0, 40)}, not ${server.commit}`); + } catch (error) { + await this.run(sshCancelForwardArgs(this.options.session, port, server.port, env), this.timeouts.operation); + return unavailable(`The VS Code server on ${this.alias} did not answer through the forward: ${(error as Error).message}.`); + } + this.language = { local: port, remote: server.port, connectionToken: server.connectionToken, commit: server.commit }; + return { languageFeatures: true }; + } + + private facts(): Pick { + return { ...(this.declined && { declined: true as const }), ...(this.installFailure && { installFailure: this.installFailure }) }; + } + + private fail(problem: Problem): void { + const attempt = this.nextAttempt(); + this.generation++; + this.queuedReattach = false; + this.cancelTimer?.(); + this.cancelTimer = undefined; + this.cancelPending?.(); + this.cancelPending = undefined; + this.connectedAt = undefined; + this.dropMaster(); + this.set({ alias: this.alias, problem, ...(this.declined && { declined: true as const }) }); + if (problem.state !== "unreachable" || this.disposed) return; + const delay = reconnectDelay(attempt, this.options.random); + this.cancelTimer = this.clock.schedule(delay, () => { + this.cancelTimer = undefined; + void this.connect(); + }); + } + + private dropMaster(): void { + const master = this.master; + this.master = undefined; + this.forwarded = undefined; + this.language = undefined; + this.installing?.abort(); + void this.close(master); + } + + private close(master: SshChildProcess | undefined): Promise { + if (!master || gone(master)) return Promise.resolve(); + const pending = this.closing.get(master); + if (pending) return pending; + const closed = (async () => { + // A master still connecting has no socket to take -O exit. + const asked = this.env && (await this.run(sshCloseArgs(this.options.session, this.env), this.timeouts.close)).code === 0; + if (!asked || !(await exitedWithin(master, this.timeouts.close))) master.kill("SIGTERM"); + await exitedWithin(master, this.timeouts.close); + })().finally(() => this.closing.delete(master)); + this.closing.set(master, closed); + return closed; + } + + private run(args: string[], timeout: number, input?: string): Promise { + return runSsh(this.options.spawn, this.env ?? {}, args, timeout, input); + } +} diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteHosts.test.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteHosts.test.ts new file mode 100644 index 000000000..eca67c766 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteHosts.test.ts @@ -0,0 +1,624 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import type { ReviewGatewayHost } from "../../common/reviewProtocol.js"; +import { attachOutput, detectOutput, FAKE_SERVER_ID, fakeClock, fakeSsh, until, type FakeRemote } from "./test/fakeSsh.js"; +import type { ReviewRemoteInstallFlow } from "./reviewRemoteHost.js"; +import { closeRemoteHostWindows, remoteHostState, ReviewRemoteHosts, type ReviewRemoteHostsOptions } from "./reviewRemoteHosts.js"; +import { openRemoteInstallConsent } from "./reviewRemoteInstallConsent.js"; +import type { SshPromptRequest } from "./reviewSshAskpass.js"; +import { reviewSshInstancePrefix } from "./reviewSshCommand.js"; + +const COMMIT = "a".repeat(40); + +async function healthServer(t: test.TestContext): Promise { + const server: Server = createServer((request, response) => response.end(request.url === "/version" ? COMMIT : '{"ok":true}')); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + t.after(() => new Promise((resolve) => server.close(resolve))); + return (server.address() as AddressInfo).port; +} + +async function managerFor( + t: test.TestContext, + remotes: Record, + answer?: string, + before?: (directory: string) => Promise, + install?: ReviewRemoteInstallFlow, + removed?: ReviewRemoteHostsOptions["removed"], +) { + const port = await healthServer(t); + const directory = await mkdtemp(join(tmpdir(), "wb-hosts-")); + await before?.(directory); + const clock = fakeClock(); + const ssh = fakeSsh(remotes, clock); + const sent: { at: number; hosts: ReviewGatewayHost[] }[] = []; + let prompt: ((request: SshPromptRequest) => Promise) | undefined; + const manager = new ReviewRemoteHosts({ + spawn: ssh.spawn, + controlDirectory: directory, + instance: "/user-data", + environment: async () => ({ PATH: "/usr/bin" }), + createAskpass: async (input) => { + prompt = input.prompt; + return { env: (alias) => ({ SSH_ASKPASS: "askpass", ALIAS: alias }), dispose() {} }; + }, + prompt: async () => answer, + desktopVersion: async () => "0.1.6", + freePort: async () => port, + send: (hosts) => sent.push({ at: clock.now(), hosts }), + log: () => {}, + clock, + timeouts: { poll: 1 }, + install, + removed, + }); + t.after(async () => { + await manager.dispose(); + await rm(directory, { recursive: true, force: true }); + }); + const sentUntil = async (condition: (hosts: ReviewGatewayHost[]) => boolean) => + until(() => { + clock.next(); + const last = sent.at(-1); + return last !== undefined && condition(last.hosts); + }); + return { manager, ssh, clock, sent, sentUntil, port, prompt: () => prompt! }; +} + +const byAlias = (hosts: ReviewGatewayHost[], alias: string) => hosts.find((h) => h.alias === alias); + +test("a host whose ssh never returns does not hold another, and update returns at once", async (t) => { + const { manager, ssh, sentUntil } = await managerFor(t, { "wb-test-slow": { master: "hang" }, "wb-test-fast": {} }); + + manager.update(true, ["wb-test-slow", "wb-test-fast"]); + assert.equal(ssh.calls.length, 0); + + await sentUntil((hosts) => byAlias(hosts, "wb-test-fast")?.endpoint !== undefined); + const slow = ssh.of("wb-test-slow", "master"); + assert.equal(slow.length, 1); + assert.ok(ssh.master("wb-test-slow")?.alive); +}); + +test("hosts are sent in the setting's order, at most once a second", async (t) => { + const { manager, sent, sentUntil } = await managerFor(t, { "wb-test-a": {}, "wb-test-b": {} }); + + manager.update(true, ["wb-test-b", "wb-test-a"]); + await sentUntil((hosts) => hosts.every((h) => h.endpoint)); + + assert.deepEqual(sent.at(-1)!.hosts.map((h) => h.alias), ["wb-test-b", "wb-test-a"]); + for (let i = 1; i < sent.length; i++) assert.ok(sent[i].at - sent[i - 1].at >= 1000); +}); + +test("removing a host from the setting closes its connection", async (t) => { + const { manager, ssh, sentUntil } = await managerFor(t, { "wb-test-a": {}, "wb-test-b": {} }); + + manager.update(true, ["wb-test-a", "wb-test-b"]); + await sentUntil((hosts) => hosts.every((h) => h.endpoint)); + manager.update(true, ["wb-test-b"]); + await sentUntil((hosts) => hosts.length === 1); + + await until(() => !ssh.master("wb-test-a")!.alive); + assert.equal(ssh.of("wb-test-a", "exit").length, 1); + assert.ok(ssh.master("wb-test-b")!.alive); +}); + +test("removing a machine's last alias closes the windows bound to it, and no other window", async (t) => { + const closed: string[] = []; + const windows = [`whiteboard+${FAKE_SERVER_ID}`, "whiteboard+wb-test-b-server", "ssh-remote+x", undefined].map((remoteAuthority) => ({ + remoteAuthority, + close: () => closed.push(String(remoteAuthority)), + })); + const b = { attach: { code: 0, stdout: attachOutput(41234, "remote-token", { serverId: "wb-test-b-server" }) } }; + const { manager, sentUntil } = await managerFor(t, { "wb-test-a": {}, "wb-test-a2": {}, "wb-test-b": b }, undefined, undefined, undefined, (serverId) => + closeRemoteHostWindows({ getWindows: () => windows }, serverId), + ); + + manager.update(true, ["wb-test-a", "wb-test-a2", "wb-test-b"]); + await sentUntil((hosts) => hosts.length === 3 && hosts.every((h) => h.endpoint)); + manager.update(true, ["wb-test-a", "wb-test-b"]); + assert.deepEqual(closed, []); + manager.update(true, ["wb-test-b"]); + assert.deepEqual(closed, [`whiteboard+${FAKE_SERVER_ID}`]); +}); + +test("a window waiting on a machine sees its alias and state, or connecting while an alias has not reported its id", () => { + assert.deepEqual(remoteHostState("s1", [ + { alias: "copy", serverId: "s1", state: "duplicate" }, + { alias: "devbox", serverId: "s1", state: "online" }, + ]), { alias: "devbox", state: "online" }); + assert.deepEqual(remoteHostState("s1", [{ alias: "devbox", serverId: "s1", state: "offline" }]), { alias: "devbox", state: "offline" }); + assert.deepEqual(remoteHostState("s1", [{ alias: "new", state: "connecting" }]), { state: "connecting" }); + assert.equal(remoteHostState("s1", [{ alias: "gone", state: "unreachable" }, { alias: "other", serverId: "s2", state: "connecting" }]), undefined); +}); + +test("a machine whose entry lost its id while unreachable is known through its alias until that alias leaves the setting", async (t) => { + const { manager, sentUntil } = await managerFor(t, { "wb-test-a": {} }); + manager.update(true, ["wb-test-a"]); + await sentUntil((hosts) => byAlias(hosts, "wb-test-a")?.endpoint !== undefined); + + const unreachable = [{ alias: "wb-test-a", state: "unreachable" as const }]; + assert.deepEqual(manager.hostState(FAKE_SERVER_ID, unreachable), { alias: "wb-test-a", state: "unreachable" }); + assert.deepEqual(manager.hostState(FAKE_SERVER_ID, []), { alias: "wb-test-a", state: "connecting" }); + assert.equal(manager.hostState("other-machine", unreachable), undefined); + + manager.update(true, []); + assert.equal(manager.hostState(FAKE_SERVER_ID, unreachable), undefined); +}); + +test("an alias added again while its old host closes starts only after the old master exited", async (t) => { + const { manager, ssh, sentUntil } = await managerFor(t, { "wb-test-a": { exitDelayMs: 150 } }); + + manager.update(true, ["wb-test-a"]); + await sentUntil((hosts) => byAlias(hosts, "wb-test-a")?.endpoint !== undefined); + const first = ssh.master("wb-test-a")!; + manager.update(true, []); + manager.update(true, ["wb-test-a"]); + await until(() => ssh.of("wb-test-a", "master").length === 2); + + assert.ok(first.exitedAt !== undefined && ssh.of("wb-test-a", "master")[1].wall >= first.exitedAt); +}); + +test("removing a host whose master is still connecting ends that ssh at once", async (t) => { + const { manager, ssh } = await managerFor(t, { "wb-test-slow": { master: "hang" } }); + + manager.update(true, ["wb-test-slow"]); + await until(() => ssh.master("wb-test-slow") !== undefined); + manager.update(true, []); + + await until(() => !ssh.master("wb-test-slow")!.alive, 500); + assert.equal(ssh.master("wb-test-slow")!.signalCode, "SIGTERM"); +}); + +test("with the experimental setting off, nothing connects and no hosts are sent", async (t) => { + const { manager, ssh, sent, clock } = await managerFor(t, { "wb-test-a": {} }); + + manager.update(false, ["wb-test-a"]); + await new Promise((resolve) => setTimeout(resolve, 20)); + while (clock.next()); + + assert.equal(ssh.calls.length, 0); + assert.equal(sent.length, 0); +}); + +test("an invalid alias is unreachable with the reason and never reaches ssh", async (t) => { + const { manager, ssh, sentUntil } = await managerFor(t, {}); + + manager.update(true, ["-oProxyCommand=x"]); + await sentUntil((hosts) => hosts[0]?.problem !== undefined); + + assert.equal(ssh.calls.length, 0); +}); + +test("a prompt cancelled through askpass is auth-failed, until a retry", async (t) => { + const { manager, ssh, sent, sentUntil, prompt } = await managerFor(t, { "wb-test-c": { master: "hang" } }, undefined); + + manager.update(true, ["wb-test-c"]); + await until(() => ssh.master("wb-test-c") !== undefined); + assert.equal(await prompt()({ alias: "wb-test-c", text: "dev@h's password: ", kind: "secret" }), undefined); + ssh.master("wb-test-c")!.finish(255, { stderr: "dev@h: Permission denied (publickey,password).\n" }); + await sentUntil((hosts) => hosts[0]?.problem?.state === "auth-failed"); + const count = sent.length; + + await new Promise((resolve) => setTimeout(resolve, 20)); + assert.equal(ssh.of("wb-test-c", "master").length, 1); + assert.equal(sent.length, count); + + manager.retry("wb-test-c"); + await until(() => ssh.of("wb-test-c", "master").length === 2); +}); + +test("at start, sockets an earlier run of this Desktop left are closed and removed; others are kept", async (t) => { + const mine = `${reviewSshInstancePrefix("/user-data")}0123456789ab`; + const theirs = `${reviewSshInstancePrefix("/other-user-data")}0123456789ab`; + let directory = ""; + const { manager, ssh, sentUntil } = await managerFor(t, { "wb-test-a": {} }, undefined, async (dir) => { + directory = dir; + await writeFile(join(dir, mine), ""); + await writeFile(join(dir, theirs), ""); + }); + + manager.update(true, ["wb-test-a"]); + await sentUntil((hosts) => hosts[0]?.endpoint !== undefined); + + const exits = ssh.calls.filter((c) => c.kind === "exit"); + assert.equal(exits.length, 1); + assert.equal(exits[0].args[exits[0].args.indexOf("-S") + 1], join(directory, mine)); + assert.ok(ssh.calls.indexOf(exits[0]) < ssh.calls.findIndex((c) => c.kind === "master")); + assert.deepEqual(await readdir(directory), [theirs]); +}); + +test("an entry the sweep cannot remove does not stop the sweep or the hosts", async (t) => { + const prefix = reviewSshInstancePrefix("/user-data"); + let directory = ""; + const { manager, ssh, sentUntil } = await managerFor(t, { "wb-test-a": {} }, undefined, async (dir) => { + directory = dir; + await mkdir(join(dir, `${prefix}000000000000`)); + await writeFile(join(dir, `${prefix}111111111111`), ""); + }); + + manager.update(true, ["wb-test-a"]); + await sentUntil((hosts) => hosts[0]?.endpoint !== undefined); + + assert.equal(ssh.calls.filter((c) => c.kind === "exit").length, 2); + assert.deepEqual(await readdir(directory), [`${prefix}000000000000`]); +}); + +test("a window gets the VS Code server of a machine only while the gateway has it online", async (t) => { + const attach = { code: 0, stdout: attachOutput(41234, "remote-token", { languageServer: { port: 45678, connectionToken: "vscode-token", commit: COMMIT } }) }; + const { manager, sentUntil, port } = await managerFor(t, { "wb-test-a": { attach } }); + + manager.update(true, ["wb-test-a"]); + await sentUntil((hosts) => byAlias(hosts, "wb-test-a")?.languageFeatures === true); + + const online = [{ alias: "wb-test-a", serverId: FAKE_SERVER_ID, state: "online" as const }]; + assert.deepEqual(await manager.languageEndpoint(FAKE_SERVER_ID, online), { host: "127.0.0.1", port, connectionToken: "vscode-token" }); + assert.equal(await manager.languageEndpoint(FAKE_SERVER_ID, [{ ...online[0], state: "duplicate" }]), undefined); + assert.equal(await manager.languageEndpoint(FAKE_SERVER_ID, [{ ...online[0], serverId: "s2" }]), undefined); + assert.equal(await manager.languageEndpoint(FAKE_SERVER_ID, []), undefined); +}); + +const INSTALLED_INTEGRITY = `sha512-${"A".repeat(86)}==`; +const INSTALLED = { nodePath: "/n/bin/node", cliPath: "/v/cli.js" }; + +test("install progress is sent at most once a second, the latest step only", async (t) => { + const gate = Promise.withResolvers(); + let reported = false; + const dir = await mkdtemp(join(tmpdir(), "wb-consent-")); + t.after(() => rm(dir, { recursive: true, force: true })); + const flow: ReviewRemoteInstallFlow = { + mode: () => "always", + consent: openRemoteInstallConsent(join(dir, "c.json")), + confirm: async () => true, + run: async (input) => { + for (let i = 0; i < 40; i++) input.onProgress(i % 2 ? { step: "package", via: "upload" } : { step: "node", via: "upload" }); + input.onProgress({ step: "verifying" }); + reported = true; + await gate.promise; + return INSTALLED; + }, + integrity: async () => INSTALLED_INTEGRITY, + }; + const { manager, sent, sentUntil } = await managerFor(t, { "wb-test-a": {} }, undefined, undefined, flow); + + manager.update(true, ["wb-test-a"]); + await until(() => reported); + await sentUntil((hosts) => hosts[0].installing !== undefined); + gate.resolve(); + await sentUntil((hosts) => hosts[0].endpoint !== undefined); + + assert.deepEqual( + sent.filter((send) => send.hosts[0].installing).map((send) => send.hosts[0].installing), + [{ step: "verifying" }], + ); + for (let i = 1; i < sent.length; i++) assert.ok(sent[i].at - sent[i - 1].at >= 1000); +}); + +test("Install on a declined host stores the agreement, installs and attaches", async (t) => { + const dir = await mkdtemp(join(tmpdir(), "wb-consent-")); + t.after(() => rm(dir, { recursive: true, force: true })); + const consentFile = join(dir, "c.json"); + const runs: string[] = []; + const flow: ReviewRemoteInstallFlow = { + mode: () => "ask", + consent: openRemoteInstallConsent(consentFile), + confirm: async () => false, + run: async (input) => { + runs.push(input.version); + return INSTALLED; + }, + integrity: async () => INSTALLED_INTEGRITY, + }; + const { manager, sentUntil } = await managerFor( + t, + { "wb-test-a": { attach: (call) => (call === 1 ? { code: 127 } : { code: 0, stdout: attachOutput(41234) }) } }, + undefined, + undefined, + flow, + ); + + manager.update(true, ["wb-test-a"]); + await sentUntil((hosts) => hosts[0].declined === true); + await manager.install("wb-test-a"); + await sentUntil((hosts) => hosts[0].endpoint !== undefined); + + assert.deepEqual(runs, ["0.1.6"]); + assert.equal(await flow.consent.get("wb-test-a"), "allow"); +}); + +test("two hosts asking at once are asked one after the other", async (t) => { + const dir = await mkdtemp(join(tmpdir(), "wb-consent-")); + t.after(() => rm(dir, { recursive: true, force: true })); + let open = 0; + const asked: string[] = []; + const flow: ReviewRemoteInstallFlow = { + mode: () => "ask", + consent: openRemoteInstallConsent(join(dir, "c.json")), + confirm: async ({ alias }) => { + assert.equal(open++, 0, `${alias} was asked while another prompt was open`); + asked.push(alias); + await new Promise((resolve) => setTimeout(resolve, 30)); + open--; + return true; + }, + run: async () => INSTALLED, + integrity: async () => INSTALLED_INTEGRITY, + }; + const other = { code: 0, stdout: attachOutput(41234, "remote-token", { serverId: "wb-test-b-server" }) }; + const { manager, sentUntil } = await managerFor(t, { "wb-test-a": {}, "wb-test-b": { attach: other } }, undefined, undefined, flow); + + manager.update(true, ["wb-test-a", "wb-test-b"]); + await sentUntil((hosts) => hosts.every((host) => host.endpoint)); + + assert.deepEqual(asked.sort(), ["wb-test-a", "wb-test-b"]); + assert.equal(await flow.consent.get("wb-test-a"), "allow"); + assert.equal(await flow.consent.get("wb-test-b"), "allow"); +}); + +async function promptingFlow(t: test.TestContext) { + const dir = await mkdtemp(join(tmpdir(), "wb-consent-")); + t.after(() => rm(dir, { recursive: true, force: true })); + const consentFile = join(dir, "c.json"); + const asked: { alias: string; signal?: AbortSignal; answer(value: boolean | undefined): void }[] = []; + const runs: string[] = []; + const flow: ReviewRemoteInstallFlow = { + mode: () => "ask", + consent: openRemoteInstallConsent(consentFile), + confirm: (request) => + new Promise((resolve) => { + asked.push({ alias: request.alias, signal: request.signal, answer: resolve }); + request.signal?.addEventListener("abort", () => resolve(undefined)); + }), + run: async (input) => { + runs.push(input.version); + return INSTALLED; + }, + integrity: async () => INSTALLED_INTEGRITY, + }; + return { flow, asked, runs, consentFile }; +} + +test("a retry while the install prompt is open joins it: one prompt, one answer applied", async (t) => { + const { flow, asked, runs } = await promptingFlow(t); + const { manager, sentUntil } = await managerFor(t, { "wb-test-a": {} }, undefined, undefined, flow); + + manager.update(true, ["wb-test-a"]); + await until(() => asked.length === 1); + manager.retry("wb-test-a"); + await new Promise((resolve) => setTimeout(resolve, 100)); + assert.equal(asked.length, 1); + assert.equal(asked[0].signal?.aborted, false); + asked[0].answer(true); + await sentUntil((hosts) => hosts[0].endpoint !== undefined); + + assert.equal(asked.length, 1); + assert.deepEqual(runs, ["0.1.6"]); + assert.equal(await flow.consent.get("wb-test-a"), "allow"); +}); + +test("removing a host while its prompt is open closes the prompt and writes nothing", async (t) => { + const { flow, asked, consentFile } = await promptingFlow(t); + const { manager } = await managerFor(t, { "wb-test-a": {} }, undefined, undefined, flow); + + manager.update(true, ["wb-test-a"]); + await until(() => asked.length === 1); + manager.update(true, []); + + await until(() => asked[0].signal?.aborted === true); + await new Promise((resolve) => setTimeout(resolve, 50)); + await assert.rejects(readFile(consentFile), { code: "ENOENT" }); +}); + +test("a dropped connection closes the host's prompt, and the next connection asks again", async (t) => { + const { flow, asked, consentFile } = await promptingFlow(t); + const { manager, ssh, clock } = await managerFor(t, { "wb-test-a": {} }, undefined, undefined, flow); + + manager.update(true, ["wb-test-a"]); + await until(() => asked.length === 1); + ssh.master("wb-test-a")!.finish(255, { stderr: "Connection reset by peer\n" }); + await until(() => asked[0].signal?.aborted === true); + await assert.rejects(readFile(consentFile), { code: "ENOENT" }); + + await until(() => { + clock.next(); + return asked.length === 2; + }); + assert.equal(asked[1].signal?.aborted, false); +}); + +test("quitting while a prompt is open closes it without an unhandled rejection", async (t) => { + const rejections: unknown[] = []; + const record = (reason: unknown) => rejections.push(reason); + process.on("unhandledRejection", record); + t.after(() => void process.off("unhandledRejection", record)); + const { flow, asked } = await promptingFlow(t); + const { manager } = await managerFor(t, { "wb-test-a": {} }, undefined, undefined, flow); + + manager.update(true, ["wb-test-a"]); + await until(() => asked.length === 1); + await manager.dispose(); + await new Promise((resolve) => setTimeout(resolve, 50)); + + assert.equal(asked[0].signal?.aborted, true); + assert.deepEqual(rejections, []); +}); + +test("a question cancelled while still queued is not joined: the next connection is asked", async (t) => { + const { flow, asked } = await promptingFlow(t); + const { manager, ssh, clock } = await managerFor(t, { "wb-test-a": {}, "wb-test-b": {} }, undefined, undefined, flow); + + manager.update(true, ["wb-test-a", "wb-test-b"]); + await until(() => asked.length === 1); + const other = asked[0].alias === "wb-test-a" ? "wb-test-b" : "wb-test-a"; + await until(() => ssh.of(other, "probe").length === 1); + await new Promise((resolve) => setTimeout(resolve, 20)); + ssh.master(other)!.finish(255, { stderr: "Connection reset by peer\n" }); + await until(() => { + clock.next(); + return ssh.of(other, "probe").length === 2; + }); + await new Promise((resolve) => setTimeout(resolve, 20)); + asked[0].answer(undefined); + + await until(() => asked.length === 2); + assert.equal(asked[1].alias, other); + assert.equal(asked[1].signal?.aborted, false); + asked[1].answer(undefined); +}); + +test("removing Whiteboard runs while nothing reconnects, then forgets the answer and closes the host; adding it again asks", async (t) => { + const { flow, asked, runs } = await promptingFlow(t); + let finish!: () => void; + const after = new Promise((resolve) => (finish = resolve)); + t.after(() => finish()); + const { manager, ssh, clock, sentUntil } = await managerFor(t, { "wb-test-a": { uninstall: { ok: true, after } } }, undefined, undefined, flow); + + manager.update(true, ["wb-test-a"]); + await until(() => asked.length === 1); + asked[0].answer(true); + await sentUntil((hosts) => hosts[0]?.endpoint !== undefined); + assert.equal(await flow.consent.get("wb-test-a"), "allow"); + + const before = ssh.calls.length; + const removed = manager.uninstall("wb-test-a"); + assert.equal(manager.uninstall("wb-test-a"), removed); + await until(() => ssh.of("wb-test-a", "uninstall").length === 2); + manager.reattach("wb-test-a"); + manager.resume(); + manager.retry("wb-test-a"); + while (clock.next()); + await new Promise((resolve) => setTimeout(resolve, 50)); + assert.deepEqual(ssh.calls.slice(before).map((call) => call.kind), ["uninstall", "uninstall"]); + assert.ok(ssh.master("wb-test-a")!.alive); + finish(); + await removed; + + assert.equal(await flow.consent.get("wb-test-a"), undefined); + await until(() => !ssh.master("wb-test-a")!.alive); + await sentUntil((hosts) => hosts.length === 0); + manager.update(true, ["wb-test-a"]); + await new Promise((resolve) => setTimeout(resolve, 20)); + assert.equal(ssh.of("wb-test-a", "master").length, 1); + + manager.update(true, []); + manager.update(true, ["wb-test-a"]); + await until(() => asked.length === 2); + assert.deepEqual(runs, ["0.1.6"]); +}); + +test("a host removed without removing Whiteboard keeps the answer; a failed uninstall keeps it, and the host waits for the setting", async (t) => { + const { flow, asked } = await promptingFlow(t); + const { manager, ssh, sentUntil } = await managerFor(t, { "wb-test-a": { uninstall: { ok: false } } }, undefined, undefined, flow); + + manager.update(true, ["wb-test-a"]); + await until(() => asked.length === 1); + asked[0].answer(true); + await sentUntil((hosts) => hosts[0]?.endpoint !== undefined); + + await assert.rejects(manager.uninstall("wb-test-a"), /Could not remove Whiteboard from wb-test-a: refused/); + assert.equal(await flow.consent.get("wb-test-a"), "allow"); + await new Promise((resolve) => setTimeout(resolve, 20)); + assert.equal(ssh.of("wb-test-a", "master").length, 1); + manager.update(true, ["wb-test-a"]); + await until(() => ssh.of("wb-test-a", "master").length === 2); + await sentUntil((hosts) => hosts[0]?.endpoint !== undefined); + + manager.update(true, []); + await until(() => !ssh.master("wb-test-a")!.alive); + assert.equal(await flow.consent.get("wb-test-a"), "allow"); + assert.equal(asked.length, 1); +}); + +test("a setting change that keeps the alias during an uninstall changes nothing until the uninstall ends", async (t) => { + const { flow, asked } = await promptingFlow(t); + let finish!: () => void; + const after = new Promise((resolve) => (finish = resolve)); + t.after(() => finish()); + const { manager, ssh, sentUntil } = await managerFor(t, { "wb-test-a": { uninstall: { ok: false, after } } }, undefined, undefined, flow); + + manager.update(true, ["wb-test-a"]); + await until(() => asked.length === 1); + asked[0].answer(true); + await sentUntil((hosts) => hosts[0]?.endpoint !== undefined); + const removed = manager.uninstall("wb-test-a"); + await until(() => ssh.of("wb-test-a", "uninstall").length === 2); + manager.update(true, ["wb-test-a"]); + await new Promise((resolve) => setTimeout(resolve, 20)); + + assert.equal(ssh.of("wb-test-a", "master").length, 1); + assert.equal(ssh.of("wb-test-a", "exit").length, 0); + assert.ok(ssh.master("wb-test-a")!.alive); + finish(); + await assert.rejects(removed, /refused/); + assert.equal(ssh.of("wb-test-a", "master").length, 1); + manager.update(true, ["wb-test-a"]); + await until(() => ssh.of("wb-test-a", "master").length === 2); +}); + +test("an alias removed during a successful uninstall is closed once", async (t) => { + const { flow, asked } = await promptingFlow(t); + let finish!: () => void; + const after = new Promise((resolve) => (finish = resolve)); + t.after(() => finish()); + const { manager, ssh, sentUntil } = await managerFor(t, { "wb-test-a": { uninstall: { ok: true, after }, exitDelayMs: 50 } }, undefined, undefined, flow); + + manager.update(true, ["wb-test-a"]); + await until(() => asked.length === 1); + asked[0].answer(true); + await sentUntil((hosts) => hosts[0]?.endpoint !== undefined); + const removed = manager.uninstall("wb-test-a"); + await until(() => ssh.of("wb-test-a", "uninstall").length === 2); + manager.update(true, []); + finish(); + await removed; + + assert.equal(ssh.master("wb-test-a")!.alive, false); + assert.equal(ssh.of("wb-test-a", "exit").length, 1); + await manager.dispose(); +}); + +test("quitting while a removed host's master closes waits for it", async (t) => { + const { flow, asked } = await promptingFlow(t); + const { manager, ssh, sentUntil } = await managerFor(t, { "wb-test-a": { uninstall: { ok: true }, exitDelayMs: 150 } }, undefined, undefined, flow); + + manager.update(true, ["wb-test-a"]); + await until(() => asked.length === 1); + asked[0].answer(true); + await sentUntil((hosts) => hosts[0]?.endpoint !== undefined); + const removed = manager.uninstall("wb-test-a"); + await until(() => ssh.of("wb-test-a", "exit").length === 1); + await manager.dispose(); + + assert.equal(ssh.master("wb-test-a")!.alive, false); + await removed; +}); + +test("a server's agents are read on their own once a session, and again only when asked", async (t) => { + const { manager, ssh, sentUntil } = await managerFor(t, { "wb-test-a": { detect: { code: 0, stdout: detectOutput([{ id: "pi", connected: false }]) } } }); + + manager.update(true, ["wb-test-a"]); + await sentUntil((hosts) => byAlias(hosts, "wb-test-a")?.endpoint !== undefined); + await until(() => ssh.of("wb-test-a", "detect").length === 1); + + manager.update(true, []); + manager.update(true, ["wb-test-a"]); + await sentUntil((hosts) => byAlias(hosts, "wb-test-a")?.endpoint !== undefined); + await until(() => ssh.of("wb-test-a", "exec").length === 2); + await new Promise((resolve) => setTimeout(resolve, 20)); + assert.equal(ssh.of("wb-test-a", "detect").length, 1); + + assert.deepEqual(await manager.detectAgents("wb-test-a"), [{ id: "pi", connected: false }]); + assert.equal(ssh.of("wb-test-a", "detect").length, 2); + assert.equal(await manager.detectAgents("wb-test-b"), undefined); + assert.deepEqual(await manager.connectAgents("wb-test-a", ["pi"]), [{ id: "pi", connected: true, output: "" }]); + await assert.rejects(manager.connectAgents("wb-test-b", ["pi"]), /wb-test-b is not a remote host/); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteHosts.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteHosts.ts new file mode 100644 index 000000000..ef9d22f98 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteHosts.ts @@ -0,0 +1,336 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { readdir, rm } from "node:fs/promises"; +import { createServer, type AddressInfo } from "node:net"; +import { join } from "node:path"; +import type { ReviewGatewayHost, ReviewGatewayHostState, ReviewRemoteAgent, ReviewRemoteAgentResult } from "../../common/reviewProtocol.js"; +import { + REVIEW_REMOTE_TIMEOUTS, + ReviewRemoteHost, + runSsh, + systemClock, + type ReviewRemoteClock, + type ReviewRemoteHostOptions, + type ReviewRemoteInstallFlow, + type SpawnSsh, +} from "./reviewRemoteHost.js"; +import type { ReviewSshAskpass, SshPromptRequest } from "./reviewSshAskpass.js"; +import { uninstallRemote } from "./reviewRemoteUninstall.js"; +import { + prepareSshControlDirectory, + reviewSshInstancePrefix, + reviewSshSession, + sshCloseArgs, + validateSshAlias, +} from "./reviewSshCommand.js"; + +const SEND_INTERVAL_MS = 1_000; + +export interface ReviewRemoteHostsOptions { + readonly spawn: SpawnSsh; + readonly controlDirectory: string; + readonly instance?: string; + environment(): Promise; + createAskpass(input: { + directory: string; + prompt(request: SshPromptRequest): Promise; + log(message: string): void; + }): Promise; + prompt(request: SshPromptRequest): Promise; + desktopVersion(): Promise; + readonly desktopCommit?: string; + groups?(): Promise; + freePort?(): Promise; + send(hosts: ReviewGatewayHost[]): void; + log(message: string): void; + readonly clock?: ReviewRemoteClock; + readonly timeouts?: ReviewRemoteHostOptions["timeouts"]; + readonly install?: ReviewRemoteInstallFlow; + /** No host in the setting serves this machine any more. */ + removed?(serverId: string): void; +} + +/** The gateway's state of the machine `serverId`; `configured` is the alias that served it. */ +export function remoteHostState( + serverId: string, + states: readonly ReviewGatewayHostState[], + configured?: string, +): { alias?: string; state: ReviewGatewayHostState["state"] } | undefined { + const own = states.filter((state) => state.serverId === serverId); + const best = own.find((state) => state.state === "online") ?? own.find((state) => state.state === "connecting") ?? own[0]; + if (best) return { alias: best.alias, state: best.state }; + if (configured) return { alias: configured, state: states.find((state) => state.alias === configured)?.state ?? "connecting" }; + return states.some((state) => state.serverId === undefined && state.state === "connecting") ? { state: "connecting" } : undefined; +} + +export function closeRemoteHostWindows(windows: { getWindows(): readonly { readonly remoteAuthority?: string; close(): void }[] }, serverId: string): void { + const authority = `whiteboard+${serverId.toLowerCase()}`; + for (const window of windows.getWindows()) if (window.remoteAuthority === authority) window.close(); +} + +export function freeLoopbackPort(): Promise { + return new Promise((resolve, reject) => { + const server = createServer(); + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const { port } = server.address() as AddressInfo; + server.close(() => resolve(port)); + }); + }); +} + +export class ReviewRemoteHosts { + private readonly clock: ReviewRemoteClock; + private readonly hosts = new Map(); + private readonly closing = new Map>(); + private readonly uninstalling = new Map>(); + private readonly removed = new Set(); + private readonly refused = new Map(); + private order: string[] = []; + private prepared: Promise | undefined; + private cancelSend: (() => void) | undefined; + private lastSent = -Infinity; + private sentAny = false; + private disposed = false; + private disposing: Promise | undefined; + private readonly agentsRead = new Set(); + + private readonly flow: ReviewRemoteInstallFlow | undefined; + + constructor(private readonly options: ReviewRemoteHostsOptions) { + this.clock = options.clock ?? systemClock; + const flow = options.install; + let asking: Promise = Promise.resolve(); + const questions = new Map; abort: AbortController }>(); + this.flow = flow && { + ...flow, + confirm: (request) => { + const open = questions.get(request.alias); + if (open) return open.answer; + const abort = new AbortController(); + const answer = asking + .then(() => (abort.signal.aborted ? undefined : flow.confirm({ ...request, signal: abort.signal }))) + .finally(() => questions.get(request.alias)?.abort === abort && questions.delete(request.alias)); + asking = answer.catch(() => undefined); + questions.set(request.alias, { answer, abort }); + return answer; + }, + cancel: (alias) => { + questions.get(alias)?.abort.abort(); + questions.delete(alias); + }, + }; + } + + update(enabled: boolean, aliases: readonly string[]): void { + if (this.disposed) return; + const wanted = enabled ? [...new Set(aliases)] : []; + for (const [alias, host] of this.hosts) { + if (wanted.includes(alias)) continue; + this.hosts.delete(alias); + this.options.log(`${alias}: removed from the setting; closing its connection.`); + this.closing.set(host, host.dispose().finally(() => this.closing.delete(host))); + this.forget(host); + } + this.order = wanted; + this.refused.clear(); + for (const alias of this.removed) if (!wanted.includes(alias)) this.removed.delete(alias); + for (const alias of wanted) { + if (this.removed.has(alias)) continue; + const valid = validateSshAlias(alias); + if (!valid.ok) { + this.refused.set(alias, { alias, problem: { state: "unreachable", detail: `The SSH alias ${JSON.stringify(alias)} ${valid.reason}.` } }); + continue; + } + const existing = this.hosts.get(alias); + if (existing) { + if (existing.quiesced) { + if (!this.uninstalling.has(alias)) existing.unquiesce(); + } else if (existing.state.problem && existing.state.problem.state !== "unreachable") existing.retry(); + continue; + } + const host = this.createHost(alias); + this.hosts.set(alias, host); + // The same alias still closing uses the same socket path, which the new host would unlink under it. + const previous = [...this.closing].filter(([old]) => old.state.alias === alias).map(([, closed]) => closed); + if (previous.length) void Promise.all(previous).then(() => this.hosts.get(alias) === host && host.start()); + else host.start(); + } + this.publish(); + } + + retry(alias: string): void { + const host = this.hosts.get(alias); + if (host?.quiesced && !this.uninstalling.has(alias)) host.unquiesce(); + else host?.retry(); + } + + async install(alias: string): Promise { + await this.options.install?.consent.set(alias, "allow"); + this.hosts.get(alias)?.retry(); + } + + detectAgents(alias: string): Promise { + return this.hosts.get(alias)?.detectAgents() ?? Promise.resolve(undefined); + } + + connectAgents(alias: string, ids: readonly unknown[]): Promise { + const host = this.hosts.get(alias); + if (!host) return Promise.reject(new Error(`${alias} is not a remote host in Settings.`)); + return host.connectAgents(ids); + } + + uninstall(alias: string): Promise { + const running = this.uninstalling.get(alias); + if (running) return running; + const done = this.uninstallOnce(alias).finally(() => this.uninstalling.delete(alias)); + this.uninstalling.set(alias, done); + return done; + } + + private async uninstallOnce(alias: string): Promise { + const valid = validateSshAlias(alias); + if (!valid.ok) throw new Error(`The SSH alias ${JSON.stringify(alias)} ${valid.reason}.`); + const host = this.hosts.get(alias); + host?.quiesce(); + try { + const env = { ...(await this.options.environment()), ...(await this.askpass()).env(alias) }; + await uninstallRemote({ session: reviewSshSession(alias, this.options.controlDirectory, this.options.instance), spawn: this.options.spawn, env }); + } catch (error) { + this.options.log(`${alias}: Whiteboard was not removed; the host waits for the setting.`); + throw error; + } + await this.options.install?.consent.forget(alias, host?.serverId); + if (host && this.hosts.get(alias) === host) { + this.hosts.delete(alias); + this.removed.add(alias); + this.forget(host); + this.publish(); + } + if (!host) return; + let closed = this.closing.get(host); + if (!closed) { + closed = host.dispose().finally(() => this.closing.delete(host)); + this.closing.set(host, closed); + } + await closed; + } + + reattach(alias: string): void { + void this.hosts.get(alias)?.reattach(); + } + + async languageEndpoint(serverId: string, states: readonly ReviewGatewayHostState[]): ReturnType { + const online = states.find((state) => state.serverId === serverId && state.state === "online"); + return online && this.hosts.get(online.alias)?.languageEndpoint(serverId); + } + + /** The state of the machine `serverId`, through the alias in the setting that served it this session. */ + hostState(serverId: string, states: readonly ReviewGatewayHostState[]): ReturnType { + const configured = [...this.hosts.values()].find((host) => host.serverId === serverId)?.state.alias; + return remoteHostState(serverId, states, configured); + } + + resume(): void { + for (const host of this.hosts.values()) void host.resume(); + } + + dispose(): Promise { + return (this.disposing ??= (async () => { + this.disposed = true; + this.cancelSend?.(); + const hosts = [...this.hosts.values()]; + this.hosts.clear(); + await Promise.all([...hosts.map((host) => host.dispose()), ...this.closing.values()]); + (await this.prepared?.catch(() => undefined))?.dispose(); + })()); + } + + killNow(): void { + for (const host of [...this.hosts.values(), ...this.closing.keys()]) host.killNow(); + } + + private createHost(alias: string): ReviewRemoteHost { + return new ReviewRemoteHost({ + session: reviewSshSession(alias, this.options.controlDirectory, this.options.instance), + spawn: this.options.spawn, + environment: async () => ({ ...(await this.options.environment()), ...(await this.askpass()).env(alias) }), + desktopVersion: () => this.options.desktopVersion(), + desktopCommit: this.options.desktopCommit, + groups: this.options.groups, + freePort: this.options.freePort ?? freeLoopbackPort, + report: () => this.publish(), + log: this.options.log, + clock: this.clock, + timeouts: this.options.timeouts, + install: this.flow, + firstAttach: (key) => !this.agentsRead.has(key) && !!this.agentsRead.add(key), + }); + } + + private forget(host: ReviewRemoteHost): void { + const { serverId } = host; + if (serverId && ![...this.hosts.values()].some((other) => other.serverId === serverId)) this.options.removed?.(serverId); + } + + private askpass(): Promise { + return (this.prepared ??= (async () => { + await prepareSshControlDirectory(this.options.controlDirectory); + await this.sweepOrphans(); + return this.options.createAskpass({ + directory: this.options.controlDirectory, + prompt: (request) => this.prompt(request), + log: this.options.log, + }); + })()); + } + + private async sweepOrphans(): Promise { + if (this.options.instance === undefined) return; + const prefix = reviewSshInstancePrefix(this.options.instance); + try { + const env = await this.options.environment(); + for (const name of await readdir(this.options.controlDirectory)) { + if (!name.startsWith(prefix)) continue; + const controlPath = join(this.options.controlDirectory, name); + try { + // -O exit works on any socket; the alias after -- is only a placeholder. + const closed = await runSsh(this.options.spawn, env, sshCloseArgs({ alias: "orphan", controlPath }, env), REVIEW_REMOTE_TIMEOUTS.close); + if (closed.code === 0) this.options.log(`closed an SSH connection left by an earlier run (${name}).`); + await rm(controlPath, { force: true }); + } catch (error) { + this.options.log(`could not remove ${name} from the SSH control directory: ${(error as Error).message}`); + } + } + } catch (error) { + this.options.log(`could not sweep the SSH control directory: ${(error as Error).message}`); + } + } + + private async prompt(request: SshPromptRequest): Promise { + const host = this.hosts.get(request.alias); + host?.promptOpened(); + let answered = true; + try { + const answer = await this.options.prompt(request); + answered = answer !== undefined; + return answer; + } finally { + host?.promptClosed(answered); + } + } + + private publish(): void { + if (this.disposed || this.cancelSend || (!this.sentAny && !this.order.length)) return; + const wait = Math.max(0, this.lastSent + SEND_INTERVAL_MS - this.clock.now()); + this.cancelSend = this.clock.schedule(wait, () => { + this.cancelSend = undefined; + this.lastSent = this.clock.now(); + this.sentAny = true; + this.options.send(this.order.flatMap((alias) => this.refused.get(alias) ?? this.hosts.get(alias)?.state ?? [])); + }); + } +} diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteInstallConsent.test.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteInstallConsent.test.ts new file mode 100644 index 000000000..2a5d1e446 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteInstallConsent.test.ts @@ -0,0 +1,103 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from "node:assert/strict"; +import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { openRemoteInstallConsent } from "./reviewRemoteInstallConsent.js"; + +async function file(t: test.TestContext) { + const dir = await mkdtemp(join(tmpdir(), "wb-consent-")); + t.after(() => rm(dir, { recursive: true, force: true })); + return join(dir, "remote-install-consent.json"); +} + +test("a consent given before the first attach moves to the server id, and follows a renamed alias", async (t) => { + const path = await file(t); + const consent = openRemoteInstallConsent(path); + + await consent.set("box", "allow"); + assert.equal(await consent.get("box"), "allow"); + await consent.attached("box", "server-1"); + + assert.deepEqual(JSON.parse(await readFile(path, "utf8")), { servers: { "server-1": { consent: "allow", alias: "box" } }, aliases: {} }); + assert.equal((await stat(path)).mode & 0o777, 0o600); + const again = openRemoteInstallConsent(path); + assert.equal(await again.get("box"), "allow"); + await again.attached("renamed", "server-1"); + assert.equal(await again.get("renamed"), "allow"); + assert.equal(await again.get("box"), undefined); +}); + +test("a decline is kept per alias until the user agrees, and an attach without consent stores nothing", async (t) => { + const path = await file(t); + const consent = openRemoteInstallConsent(path); + + await consent.attached("other", "server-2"); + assert.equal(await consent.get("other"), undefined); + await consent.set("box", "deny"); + assert.equal(await openRemoteInstallConsent(path).get("box"), "deny"); + await consent.set("box", "allow"); + assert.equal(await consent.get("box"), "allow"); +}); + +test("an agreement for a host known by its server id is stored under that id", async (t) => { + const path = await file(t); + const consent = openRemoteInstallConsent(path); + + await consent.set("box", "deny"); + await consent.attached("box", "server-1"); + await consent.set("box", "allow"); + + assert.deepEqual(JSON.parse(await readFile(path, "utf8")), { servers: { "server-1": { consent: "allow", alias: "box" } }, aliases: {} }); +}); + +test("an unreadable file is no consent, and is replaced on the next answer", async (t) => { + const path = await file(t); + await writeFile(path, "{not json"); + const consent = openRemoteInstallConsent(path); + + assert.equal(await consent.get("box"), undefined); + await consent.set("box", "allow"); + assert.equal(await consent.get("box"), "allow"); +}); + +test("an unchanged answer is not written again, and __proto__ is an ordinary key", async (t) => { + const path = await file(t); + const consent = openRemoteInstallConsent(path); + + await consent.set("box", "allow"); + await consent.attached("box", "server-1"); + const written = (await stat(path)).mtimeMs; + await new Promise((resolve) => setTimeout(resolve, 20)); + await consent.attached("box", "server-1"); + await consent.set("box", "allow"); + assert.equal((await stat(path)).mtimeMs, written); + + await consent.set("__proto__", "deny"); + assert.equal(await openRemoteInstallConsent(path).get("__proto__"), "deny"); + assert.equal(await consent.get("box"), "allow"); +}); + +test("forgetting a host clears its answer by alias and by server id, and leaves other hosts", async (t) => { + const path = await file(t); + const consent = openRemoteInstallConsent(path); + + await consent.set("box", "allow"); + await consent.attached("box", "server-1"); + await consent.set("pending", "deny"); + await consent.set("other", "allow"); + await consent.attached("other", "server-2"); + await consent.set("renamed", "allow"); + await consent.attached("renamed", "server-3"); + + await consent.forget("box", "server-3"); + await consent.forget("pending"); + + assert.deepEqual(JSON.parse(await readFile(path, "utf8")), { servers: { "server-2": { consent: "allow", alias: "other" } }, aliases: {} }); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteInstallConsent.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteInstallConsent.ts new file mode 100644 index 000000000..61739ae66 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteInstallConsent.ts @@ -0,0 +1,109 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { randomBytes } from "node:crypto"; +import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; + +export type ReviewRemoteInstallAnswer = "allow" | "deny"; + +export interface ReviewRemoteInstallConsent { + get(alias: string): Promise; + set(alias: string, answer: ReviewRemoteInstallAnswer): Promise; + attached(alias: string, serverId: string): Promise; + forget(alias: string, serverId?: string): Promise; +} + +interface Stored { + servers: Record; + aliases: Record; +} + +export const reviewRemoteInstallConsentPath = (userDataPath: string) => join(userDataPath, "remote-install-consent.json"); + +const answer = (value: unknown): ReviewRemoteInstallAnswer | undefined => (value === "allow" || value === "deny" ? value : undefined); + +export function openRemoteInstallConsent(path: string): ReviewRemoteInstallConsent { + let queue: Promise = Promise.resolve(); + const serial = (run: () => Promise): Promise => { + const next = queue.then(run, run); + queue = next.catch(() => undefined); + return next; + }; + + async function read(): Promise { + // No prototype: a key such as `__proto__` is a plain entry. + const stored: Stored = { servers: Object.create(null), aliases: Object.create(null) }; + let value: { servers?: Record; aliases?: Record }; + try { + value = JSON.parse(await readFile(path, "utf8")); + } catch { + return stored; + } + for (const [serverId, entry] of Object.entries(value?.servers ?? {})) { + const consent = answer(entry?.consent); + if (consent && typeof entry.alias === "string") stored.servers[serverId] = { consent, alias: entry.alias }; + } + for (const [alias, value_] of Object.entries(value?.aliases ?? {})) { + const consent = answer(value_); + if (consent) stored.aliases[alias] = consent; + } + return stored; + } + + async function write(stored: Stored, before: string): Promise { + if (JSON.stringify(stored) === before) return; + await mkdir(dirname(path), { recursive: true }); + const part = `${path}.${randomBytes(4).toString("hex")}.part`; + try { + await writeFile(part, JSON.stringify(stored), { mode: 0o600 }); + await rename(part, path); + } finally { + await rm(part, { force: true }); + } + } + + const serverOf = (stored: Stored, alias: string) => Object.entries(stored.servers).find(([, entry]) => entry.alias === alias)?.[0]; + + return { + get: (alias) => + serial(async () => { + const stored = await read(); + const serverId = serverOf(stored, alias); + return stored.aliases[alias] ?? (serverId === undefined ? undefined : stored.servers[serverId].consent); + }), + set: (alias, consent) => + serial(async () => { + const stored = await read(); + const before = JSON.stringify(stored); + const serverId = serverOf(stored, alias); + if (serverId === undefined) stored.aliases[alias] = consent; + else { + stored.servers[serverId] = { consent, alias }; + delete stored.aliases[alias]; + } + await write(stored, before); + }), + attached: (alias, serverId) => + serial(async () => { + const stored = await read(); + const before = JSON.stringify(stored); + const consent = stored.aliases[alias] ?? stored.servers[serverId]?.consent; + if (!consent) return; + for (const entry of Object.values(stored.servers)) if (entry.alias === alias) entry.alias = ""; + stored.servers[serverId] = { consent, alias }; + delete stored.aliases[alias]; + await write(stored, before); + }), + forget: (alias, serverId) => + serial(async () => { + const stored = await read(); + const before = JSON.stringify(stored); + delete stored.aliases[alias]; + for (const [id, entry] of Object.entries(stored.servers)) if (id === serverId || entry.alias === alias) delete stored.servers[id]; + await write(stored, before); + }), + }; +} diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteInstallScript.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteInstallScript.ts new file mode 100644 index 000000000..ccdee0a76 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteInstallScript.ts @@ -0,0 +1,308 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import type { ReviewRemoteTarget } from "./reviewRemoteProbe.js"; +import { REVIEW_REMOTE_INSTALL_LOCK, REVIEW_REMOTE_LOCK_STALE_SECONDS, REVIEW_REMOTE_VERSION, REVIEW_REMOTE_WRAPPER_MARK } from "../../common/reviewProtocol.js"; + +export { REVIEW_REMOTE_LOCK_STALE_SECONDS, REVIEW_REMOTE_VERSION, REVIEW_REMOTE_WRAPPER_MARK }; + +export const REVIEW_REMOTE_INSTALL_SAY = "WHITEBOARD-INSTALL"; +export const REVIEW_REMOTE_INSTALL_MARKER = ".whiteboard-install.json"; + +export function shellQuote(value: string, lines = false): string { + if ((lines ? /[\x00-\x09\x0b-\x1f\x7f-\x9f]/ : /[\x00-\x1f\x7f-\x9f]/).test(value)) { + throw new Error(`${JSON.stringify(value)} holds a control character.`); + } + return `'${value.replace(/'/g, `'\\''`)}'`; +} + +export interface ReviewRemoteInstallContext { + readonly home: string; + readonly root: string; + readonly token: string; +} + +export const REVIEW_REMOTE_ROOT_SCRIPT = `base=$(printf '%s' "\${DEV_REVIEW_HOME-}" | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') +root= +case "$base" in +'') root=$HOME/.dev/whiteboard-remote ;; +*//*|*/./*|*/../*|*/.|*/..|*/./|*/../) ;; +/*) root=\${base%/}/whiteboard-remote ;; +esac +[ "$(printf '%s' "\${DEV_REVIEW_HOME-}" | tr -d '\\001-\\037\\177')" = "\${DEV_REVIEW_HOME-}" ] || root= +`; + +export const reviewRemoteVersionDir = (root: string, version: string) => `${root}/versions/${version}`; +export const reviewRemoteNodeDir = (root: string, nodeVersion: string) => `${root}/node/v${nodeVersion}`; +export const reviewRemoteWrapperPath = (home: string) => `${home}/.local/bin/whiteboard`; + +const versionPart = (context: ReviewRemoteInstallContext, version: string) => `${reviewRemoteVersionDir(context.root, version)}.${context.token}.part`; +const nodePart = (context: ReviewRemoteInstallContext, nodeVersion: string) => `${reviewRemoteNodeDir(context.root, nodeVersion)}.${context.token}.part`; + +function prelude(context: ReviewRemoteInstallContext): string { + if (!/^[0-9a-f]{8,64}$/.test(context.token)) throw new Error("The install token is not hex."); + return `LC_ALL=C +export LC_ALL +umask 022 +trap '' PIPE +exec 3>&1 +root=${shellQuote(context.root)} +lock="$root/${REVIEW_REMOTE_INSTALL_LOCK}" +token=${context.token} +say() { printf '\\n%s %s\\n' ${REVIEW_REMOTE_INSTALL_SAY} "$*" >&3; } +fail() { say FAIL "$*"; exit 3; } +# A rename, so a reader never sees "started" half written. +stamp() { date +%s > "$lock/started.$token" && mv -f "$lock/started.$token" "$lock/started"; } +own() { + [ "$(cat "$lock/token" 2>/dev/null)" = "$token" ] || fail this install no longer holds the install lock + stamp +} +guard() { + if command -v setsid >/dev/null 2>&1; then setsid "$@" & else "$@" & fi + pid=$! + while kill -0 "$pid" 2>/dev/null; do + printf '.\\n' >&3 2>/dev/null || { kill -TERM -"$pid" 2>/dev/null || kill "$pid" 2>/dev/null; exit 3; } + [ "$(cat "$lock/token" 2>/dev/null)" = "$token" ] && stamp + sleep 1 + done + wait "$pid" +} +`; +} + +export function lockScript(context: ReviewRemoteInstallContext, owner: string, staleSeconds = REVIEW_REMOTE_LOCK_STALE_SECONDS): string { + const staleMinutes = Math.ceil(staleSeconds / 60); + return `${prelude(context)}mkdir -p "$root" || fail cannot create "$root" +take() { + mkdir "$lock" 2>/dev/null || return 1 + printf '%s\\n' "$token" > "$lock/token" + printf '%s\\n' ${shellQuote(owner)} > "$lock/owner" + stamp + say LOCKED + exit 0 +} +take +# The token first: a takeover completed between the two reads leaves "started" fresh. +held=$(cat "$lock/token" 2>/dev/null) +started=$(cat "$lock/started" 2>/dev/null) +case "$started" in +''|*[!0-9]*) + # Never written (a holder that died right after mkdir): by the directory's age. Unreadable: fresh. + if [ ! -e "$lock/started" ] && [ -n "$(find "$lock" -prune -mmin +${staleMinutes} 2>/dev/null)" ]; then started=0; else started=$(date +%s); fi ;; +esac +if [ $(( $(date +%s) - started )) -ge ${staleSeconds} ]; then + stale="$lock.$token.stale" + if mv "$lock" "$stale" 2>/dev/null; then + # Another install may have taken it over first: give that one back. + if [ "$(cat "$stale/token" 2>/dev/null)" = "$held" ] || [ -e "$lock" ]; then rm -rf "$stale"; else mv "$stale" "$lock"; fi + fi + take +fi +say BUSY "$(cat "$lock/owner" 2>/dev/null)" +`; +} + +export function refreshScript(context: ReviewRemoteInstallContext): string { + return `${prelude(context)}own +say REFRESHED +`; +} + +export function releaseScript(context: ReviewRemoteInstallContext): string { + return `${prelude(context)}if [ "$(cat "$lock/token" 2>/dev/null)" = "$token" ] && mv "$lock" "$lock.$token.done" 2>/dev/null; then + rm -rf "$lock.$token.done" +fi +say RELEASED +`; +} + +export function prepareScript(context: ReviewRemoteInstallContext, input: { version: string; integrity: string; nodeVersion: string }): string { + const nodeDir = reviewRemoteNodeDir(context.root, input.nodeVersion); + return `${prelude(context)}own +rm -rf "$root"/versions/*.part "$root"/node/*.part "$lock".*.stale "$lock".*.done +mkdir -p "$root/versions" || fail cannot create "$root/versions" +${markerCheck(context, input)} +if [ -n "$complete" ]; then + say COMPLETE + say MARKER "$(cat "$m")" +elif [ -e "$v" ]; then + mv "$v" ${shellQuote(versionPart(context, input.version))} && rm -rf ${shellQuote(versionPart(context, input.version))} || fail cannot remove "$v" +fi +for dir in "$root"/versions/*; do + [ -f "$dir/${REVIEW_REMOTE_INSTALL_MARKER}" ] && say HAVE "\${dir##*/}" && continue + # A version whose removal was cut short: its marker went first. + case "\${dir##*/}" in [0-9]*.[0-9]*.[0-9]*) rm -rf "$dir" ;; esac +done +n=${shellQuote(`${nodeDir}/bin/node`)} +[ -x "$n" ] && [ "$("$n" --version 2>/dev/null)" = v${input.nodeVersion} ] && say MANAGED-NODE +say PREPARED +`; +} + +export const REVIEW_REMOTE_COMPLETE_INTEGRITY = `completeIntegrity() { + ci=$(sed -n 's/.*"integrity":"\\([^"][^"]*\\)".*/\\1/p' "$1" 2>/dev/null) + cn=$(sed -n 's/.*"node":"\\([^"]*\\)".*/\\1/p' "$1" 2>/dev/null) + cc=$(sed -n 's/.*"cli":"\\([^"]*\\)".*/\\1/p' "$1" 2>/dev/null) + [ -n "$ci" ] && [ -x "$cn" ] && [ -f "$cc" ] && printf '%s' "$ci" +}`; + +function markerCheck(context: ReviewRemoteInstallContext, input: { version: string; integrity: string }): string { + return `v=${shellQuote(reviewRemoteVersionDir(context.root, input.version))} +m="$v/${REVIEW_REMOTE_INSTALL_MARKER}" +${REVIEW_REMOTE_COMPLETE_INTEGRITY} +complete= +[ -f "$m" ] && [ "$(completeIntegrity "$m")" = ${shellQuote(input.integrity)} ] && complete=1`; +} + +export function completeScript(context: ReviewRemoteInstallContext, input: { version: string; integrity: string }): string { + return `${prelude(context)}${markerCheck(context, input)} +[ -n "$complete" ] || exit 0 +say COMPLETE +say MARKER "$(cat "$m")" +`; +} + +export function partScript(context: ReviewRemoteInstallContext, part: { node: string } | { package: string }): string { + const dir = "node" in part ? nodePart(context, part.node) : versionPart(context, part.package); + return `${prelude(context)}own +d=${shellQuote(dir)} +rm -rf "$d" && mkdir -p "$d" || fail cannot create "$d" +say READY +`; +} + +export const nodeTarball = (context: ReviewRemoteInstallContext, nodeVersion: string) => `${nodePart(context, nodeVersion)}/node.tar.xz`; +export const packageTarball = (context: ReviewRemoteInstallContext, version: string) => `${versionPart(context, version)}/package.tgz`; + +export function downloadScript(context: ReviewRemoteInstallContext, input: { url: string; file: string; downloader: "curl" | "wget" }): string { + const fetch = + input.downloader === "curl" + ? `guard curl -fsSL --connect-timeout 20 --max-time 900 -o "$f" "$url"` + : `hsts= +wget --help 2>&1 | grep -q -- --no-hsts && hsts=--no-hsts +guard wget -q $hsts -t 2 -T 30 -O "$f" "$url"`; + return `${prelude(context)}own +f=${shellQuote(input.file)} +url=${shellQuote(input.url)} +${fetch} || { rm -f "$f"; fail the download of "$url" failed; } +say DOWNLOADED +`; +} + +export function nodePlaceScript(context: ReviewRemoteInstallContext, input: { nodeVersion: string; sha256: string }): string { + if (!/^[0-9a-f]{64}$/.test(input.sha256)) throw new Error("The Node checksum is not a sha256."); + return `${prelude(context)}own +d=${shellQuote(nodePart(context, input.nodeVersion))} +final=${shellQuote(reviewRemoteNodeDir(context.root, input.nodeVersion))} +f="$d/node.tar.xz" +sum=$(sha256sum "$f" 2>/dev/null) || { rm -rf "$d"; fail cannot read "$f"; } +sum=\${sum%% *} +[ "$sum" = ${input.sha256} ] || { rm -rf "$d"; say MISMATCH "$sum"; exit 3; } +tar -xJf "$f" -C "$d" --strip-components=1 || { rm -rf "$d"; fail cannot unpack Node; } +rm -f "$f" +[ "$("$d/bin/node" --version 2>/dev/null)" = v${input.nodeVersion} ] || { rm -rf "$d"; fail the unpacked Node does not run; } +rm -rf "$final" && mv "$d" "$final" || fail cannot move Node into place +say NODE-OK +`; +} + +export function packageInstallScript( + context: ReviewRemoteInstallContext, + input: { version: string; target: ReviewRemoteTarget; sha512: string; node: string; npm: string; registry?: string }, +): string { + if (!/^[0-9a-f]{128}$/.test(input.sha512)) throw new Error("The package checksum is not a sha512."); + const nodeBin = input.node.slice(0, input.node.lastIndexOf("/")); + const registry = input.registry ? ` --registry=${shellQuote(input.registry)}` : ""; + // --omit=optional skips unused agent binaries, so diffr's platform package is named directly. + return `${prelude(context)}own +p=${shellQuote(versionPart(context, input.version))} +f="$p/package.tgz" +if command -v sha512sum >/dev/null 2>&1; then sum=$(sha512sum "$f" 2>/dev/null) +elif command -v openssl >/dev/null 2>&1; then sum=$(openssl dgst -sha512 -r "$f" 2>/dev/null) +else rm -rf "$p"; fail neither sha512sum nor openssl is installed; fi +sum=\${sum%% *} +[ "$sum" = ${input.sha512} ] || { rm -rf "$p"; say MISMATCH "$sum"; exit 3; } +PATH=${shellQuote(nodeBin)}:$PATH +export PATH +dv=$(tar -xzOf "$f" package/package.json 2>/dev/null | sed -n 's/.*"@dev\\.fast\\/diffr": *"\\([0-9][^"]*\\)".*/\\1/p' | head -n 1) +guard ${shellQuote(input.npm)} install --ignore-scripts --omit=optional --no-audit --no-fund --no-update-notifier --loglevel=error --cache "$p/.npm-cache" --prefix "$p"${registry} "$f" \${dv:+"@dev.fast/diffr-${input.target}@$dv"} > "$p/.npm.log" 2>&1 || { + tail -n 15 "$p/.npm.log" >&3 + rm -rf "$p" + fail npm could not install the package +} +rm -rf "$p/.npm-cache" "$p/.npm.log" "$f" +say INSTALLED +`; +} + +export function verifyScript(context: ReviewRemoteInstallContext, input: { version: string; node: string }): string { + return `${prelude(context)}own +p=${shellQuote(versionPart(context, input.version))} +node=${shellQuote(input.node)} +pkg="$p/node_modules/@dev.fast/whiteboard" +bin=$("$node" -p 'require(process.argv[1]).bin.whiteboard' "$pkg/package.json" 2>/dev/null) +case "$bin" in ''|/*|*..*) fail the package names no whiteboard command ;; esac +bin=\${bin#./} +cd "$p" || fail cannot enter "$p" +out=$(DEV_FAST_REVIEW_CLI_NO_DELEGATE=1 DEV_FAST_REVIEW_TELEMETRY_DISABLED=1 "$node" "$pkg/$bin" version --json /dev/null | tail -n 1) +say BIN "$bin" +say VERSION "$out" +`; +} + +export function finishScript( + context: ReviewRemoteInstallContext, + input: { version: string; launcher: string; marker: string; wrapper?: string }, +): string { + const wrapper = input.wrapper + ? `w=${shellQuote(reviewRemoteWrapperPath(context.home))} +if { [ ! -e "$w" ] && [ ! -L "$w" ]; } || { [ -f "$w" ] && [ ! -L "$w" ] && grep -qxF ${shellQuote(REVIEW_REMOTE_WRAPPER_MARK)} "$w"; }; then + if mkdir -p "\${w%/*}" && printf '%s' ${shellQuote(input.wrapper, true)} > "$w.$token.part" && chmod 755 "$w.$token.part" && mv -f "$w.$token.part" "$w"; then + say WRAPPER written + else + rm -f "$w.$token.part" + say WRAPPER failed + fi +else + say WRAPPER foreign +fi +` + : ""; + return `${prelude(context)}own +p=${shellQuote(versionPart(context, input.version))} +v=${shellQuote(reviewRemoteVersionDir(context.root, input.version))} +printf '%s' ${shellQuote(input.launcher, true)} > "$p/whiteboard" && chmod 755 "$p/whiteboard" || fail cannot write the launcher +printf '%s\\n' ${shellQuote(input.marker)} > "$p/${REVIEW_REMOTE_INSTALL_MARKER}" || fail cannot write the marker +[ -e "$v" ] && { rm -rf "$p"; fail "$v" appeared during the install; } +mv "$p" "$v" || fail cannot move the version into place +say COMPLETE +${wrapper}rm -rf "$root"/versions/*.part "$root"/node/*.part +say FINISHED +`; +} + +export function cleanupScript(context: ReviewRemoteInstallContext, input: { candidates: readonly string[]; room: number }): string { + const names = input.candidates.map((name) => shellQuote(name)).join(" "); + return `${prelude(context)}own +# Every command line, read once: a grep below must not find itself. The +# launchers run the CLI by absolute path, so its version's path is there. +if [ -d /proc/self ]; then + procs=$(for f in /proc/[0-9]*/cmdline; do tr '\\000' ' ' < "$f" 2>/dev/null; echo; done) +else + procs=$(ps -eo args= 2>/dev/null) +fi +running() { printf '%s\\n' "$procs" | grep -qF -- "$root/versions/$1/"; } +kept=0 +for v in ${names}; do running "$v" && kept=$((kept + 1)); done +for v in ${names}; do + if running "$v"; then say IN-USE "$v"; continue; fi + if [ "$kept" -lt ${input.room} ]; then kept=$((kept + 1)); continue; fi + d="$root/versions/$v" + [ -f "$d/${REVIEW_REMOTE_INSTALL_MARKER}" ] || continue + rm -f "$d/${REVIEW_REMOTE_INSTALL_MARKER}" && rm -rf "$d" && say REMOVED "$v" +done +say CLEANED +`; +} diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteInstaller.containers.test.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteInstaller.containers.test.ts new file mode 100644 index 000000000..0107218a6 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteInstaller.containers.test.ts @@ -0,0 +1,287 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from "node:assert/strict"; +import { execFile, execFileSync, spawn } from "node:child_process"; +import { randomBytes, randomUUID } from "node:crypto"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { after, before, test } from "node:test"; +import { promisify } from "node:util"; + +import { remoteArtifacts, type ReviewRemoteArtifact } from "./reviewRemoteArtifacts.js"; +import type { SpawnSsh } from "./reviewRemoteHost.js"; +import { installRemote, type ReviewRemoteInstallInput, type ReviewRemoteInstallProgress } from "./reviewRemoteInstaller.js"; +import { judgeRemote, probeRemote, type ReviewRemoteProbe, type ReviewRemoteTarget } from "./reviewRemoteProbe.js"; +import { REVIEW_SSH_CONFIG_ENV, reviewSshSession, sshCheckArgs, sshCloseArgs, sshMasterArgs, type ReviewSshSession } from "./reviewSshCommand.js"; + +const run = promisify(execFile); +const checkout = resolve(import.meta.dirname, "../../../../../../../.."); +const harness = join(checkout, "apps/review-desktop/scripts/e2e/remote/remote.mjs"); +const VERSION: string = JSON.parse(await readFile(join(checkout, "packages/review/package.json"), "utf8")).version; + +function skipReason(): string | undefined { + if (process.env.WB_TEST_CONTAINERS !== "1") return "set WB_TEST_CONTAINERS=1 to install into containers"; + try { + execFileSync("docker", ["info"], { stdio: "ignore" }); + } catch { + return "Docker is not available"; + } + return undefined; +} + +const skip = skipReason(); +const runId = `s3t3${randomBytes(3).toString("hex")}`; +const hosts = { node: [], bare: ["--node", "none"], sealed: ["--sealed", "--node", "none"] } as const; +type Host = keyof typeof hosts; +const env = { ...process.env, WB_TEST_RUN: runId }; +const sshEnv = { ...process.env, VSCODE_DEV: "1", [REVIEW_SSH_CONFIG_ENV]: `/tmp/wbt.${runId}/ssh_config` }; +const realSsh: SpawnSsh = (args, options) => spawn("ssh", args, options); + +let started = false; +let root = ""; +let controlDirectory = ""; +const masters: ReviewSshSession[] = []; +const found = new Map(); +const artifacts = new Map(); + +const remote = (...args: string[]) => run(process.execPath, [harness, ...args], { env, maxBuffer: 16 << 20 }); +const inContainer = async (host: Host, command: string) => (await run("docker", ["exec", "-u", "dev", `wb-test-${runId}-${host}`, "sh", "-c", command])).stdout.trim(); +const reset = (host: Host) => inContainer(host, "rm -rf ~/.dev/whiteboard-remote ~/.local/bin/whiteboard"); + +async function connect(host: Host): Promise { + const session = reviewSshSession(`wb-test-${host}`, controlDirectory); + spawn("ssh", sshMasterArgs(session, sshEnv), { env: sshEnv, detached: true, stdio: "ignore" }).unref(); + masters.push(session); + for (let i = 0; i < 100; i++) { + if (await run("ssh", sshCheckArgs(session, sshEnv), { env: sshEnv }).then(() => true, () => false)) return session; + await new Promise((r) => setTimeout(r, 200)); + } + throw new Error(`the master for ${host} did not start`); +} + +before( + async () => { + if (skip) return; + root = await mkdtemp(join(process.env.TMPDIR ?? "/tmp", "wb-install-")); + controlDirectory = await mkdtemp("/tmp/wbi-"); + started = true; + for (const [host, flags] of Object.entries(hosts)) await remote("up", host, ...flags); + for (const host of Object.keys(hosts) as Host[]) { + const session = await connect(host); + const probed = await probeRemote({ session, spawn: realSsh, env: sshEnv }); + assert.ok("probe" in probed, "error" in probed ? probed.error : ""); + const judged = judgeRemote(probed.probe); + assert.ok(judged.supported); + found.set(host, { session, probe: probed.probe, target: judged.target }); + if (!artifacts.has(judged.target)) { + artifacts.set(judged.target, await remoteArtifacts(judged.target, { pin: undefined, checkout, cacheDirectory: join(root, "cache") })); + } + } + }, + { timeout: 30 * 60_000 }, +); + +after( + async () => { + for (const session of masters) await run("ssh", sshCloseArgs(session, sshEnv), { env: sshEnv }).catch(() => undefined); + if (started) await remote("down", "--all").catch((error) => console.error(error.stderr ?? error)); + for (const dir of [root, controlDirectory]) if (dir) await rm(dir, { recursive: true, force: true }); + }, + { timeout: 5 * 60_000 }, +); + +function input(host: Host, change: Partial = {}) { + const { session, probe, target } = found.get(host)!; + const progress: ReviewRemoteInstallProgress[] = []; + const value: ReviewRemoteInstallInput = { + session, + probe, + target, + version: VERSION, + onProgress: (step) => progress.push(step), + signal: new AbortController().signal, + artifacts: artifacts.get(target)!, + published: false, + cacheDirectory: join(root, "cache"), + spawn: realSsh, + env: sshEnv, + ...change, + }; + return { value, progress }; +} + +const version = async (host: Host) => JSON.parse(await inContainer(host, "~/.local/bin/whiteboard version --json")).version; +const leftovers = async (host: Host) => (await inContainer(host, "cd ~/.dev/whiteboard-remote && find . -mindepth 1 -maxdepth 2 | sort")).split("\n"); + +test("a host with Node 24 gets the package; whiteboard version prints it", { skip, timeout: 10 * 60_000 }, async () => { + const { value, progress } = input("node"); + const result = await installRemote(value); + + assert.equal(result.nodePath, "/usr/local/bin/node"); + assert.equal(await version("node"), VERSION); + assert.deepEqual( + progress.map((p) => p.step), + ["package", "verifying", "done"], + ); + assert.equal(await inContainer("node", "ls ~/.dev/whiteboard-remote/versions"), VERSION); + + const again = input("node"); + assert.deepEqual(await installRemote(again.value), result); + assert.deepEqual(again.progress.map((p) => p.step), ["done"]); + assert.equal(await inContainer("node", "ls -d ~/.dev/whiteboard-remote/install.lock 2>/dev/null | wc -l"), "0"); +}); + +const FETCH = `const [url, token, method, path, body] = process.argv.slice(1); +const response = await fetch(url + path, { method, headers: { "x-review-token": token, "content-type": "application/json" }, body: method === "GET" ? undefined : body }); +const bytes = Buffer.from(await response.arrayBuffer()); +console.log(JSON.stringify({ status: response.status, type: response.headers.get("content-type"), size: bytes.length, text: bytes.toString() }));`; +const PIXEL = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="; + +test("an install has the adapters but not the bundled agent binaries; images decode and structural diff answers", { skip, timeout: 10 * 60_000 }, async (t) => { + await installRemote(input("node").value); + const modules = `~/.dev/whiteboard-remote/versions/${VERSION}/node_modules`; + + assert.equal(await inContainer("node", `test -f ${modules}/@agentclientprotocol/codex-acp/dist/index.js && echo yes`), "yes"); + assert.equal(await inContainer("node", `test -f ${modules}/@agentclientprotocol/claude-agent-acp/dist/index.js && echo yes`), "yes"); + assert.equal(await inContainer("node", `ls -d ${modules}/@openai/codex-* ${modules}/@anthropic-ai/claude-agent-sdk-* 2>/dev/null | wc -l`), "0"); + assert.equal(await inContainer("node", `test -x ${modules}/@dev.fast/diffr-linux-*/diffr && echo yes`), "yes"); + assert.equal(await version("node"), VERSION); + + await inContainer( + "node", + `git init -q -b main ~/repo && cd ~/repo && printf 'export const a = 1;\\n' > a.ts && git add . && git -c user.name=t -c user.email=t@t commit -qm one && printf 'export const a = 2;\\n' > a.ts && git -c user.name=t -c user.email=t@t commit -qam two`, + ); + t.after(() => inContainer("node", "~/.local/bin/whiteboard server stop; rm -rf ~/repo")); + const attach = JSON.parse((await inContainer("node", "~/.local/bin/whiteboard remote attach --json")).split("\n").find((line) => line.startsWith("{"))!); + const api = async (method: string, path: string, body?: unknown) => + JSON.parse((await run("docker", ["exec", "-u", "dev", `wb-test-${runId}-node`, "node", "--input-type=module", "-e", FETCH, attach.url, attach.token, method, `/reviews-api${path}`, JSON.stringify(body ?? null)])).stdout); + const [base, head] = (await inContainer("node", "cd ~/repo && git rev-parse HEAD~1 HEAD")).split("\n"); + const repositoryId = JSON.parse((await api("POST", "/repositories", { path: "/home/dev/repo" })).text).id; + const { reviewId } = JSON.parse( + (await api("POST", "/commands", { operation: { type: "create", title: "Install check", target: { kind: "commits", repositoryId, base, head } } })).text, + ); + + const id = randomUUID(); + assert.equal((await api("POST", "/resources", { id, repositoryId, kind: "image", base64: PIXEL })).status, 200); + const image = await api("GET", `/${reviewId}/resources/${id}`); + assert.deepEqual([image.status, image.type], [200, "image/png"]); + assert.ok(image.size > 0); + + const diff = await api("GET", `/${reviewId}/structural-diff`); + assert.equal(diff.status, 200); + const events = diff.text.trim().split("\n").map((line: string) => JSON.parse(line)); + assert.ok(events.some((event: { type: string }) => event.type === "file")); + assert.equal(events.at(-1).type, "complete"); + assert.equal(events.at(-1).failed, 0); +}); + +test("a host with no Node gets Node and the package", { skip, timeout: 10 * 60_000 }, async () => { + const { value, progress } = input("bare"); + const result = await installRemote(value); + + assert.match(result.nodePath, /\/\.dev\/whiteboard-remote\/node\/v24\.\d+\.\d+\/bin\/node$/); + assert.deepEqual(progress[0], { step: "node", via: "remote-download" }); + assert.equal(await version("bare"), VERSION); +}); + +test("a sealed host gets Node by upload and the dependencies through the relay", { skip, timeout: 10 * 60_000 }, async () => { + const { value, progress } = input("sealed"); + await installRemote(value); + + assert.deepEqual(progress.slice(0, 2), [ + { step: "node", via: "upload" }, + { step: "package", via: "upload" }, + ]); + assert.equal(await version("sealed"), VERSION); + assert.match(await inContainer("sealed", `~/.dev/whiteboard-remote/versions/${VERSION}/node_modules/@dev.fast/diffr-linux-*/diffr --version`), /^diffr /); + assert.equal(await inContainer("sealed", "ss -Htln | grep -c 127.0.0.1: || true"), "0"); +}); + +test("an abort while npm runs leaves no version; the next install succeeds", { skip, timeout: 10 * 60_000 }, async () => { + await reset("node"); + const abort = new AbortController(); + const npmRunning = () => inContainer("node", "pgrep -f '[n]pm install' || true"); + let sawNpm = false; + const { value } = input("node", { + signal: abort.signal, + onProgress: async (step) => { + if (step.step !== "package") return; + for (let i = 0; i < 300 && !abort.signal.aborted; i++) { + if (await npmRunning()) { + sawNpm = true; + abort.abort(); + } + await new Promise((r) => setTimeout(r, 100)); + } + }, + }); + + await assert.rejects(installRemote(value), { name: "AbortError" }); + assert.ok(sawNpm); + + let npm = ""; + for (let i = 0; i < 50; i++) { + npm = await npmRunning(); + if (!npm) break; + await new Promise((r) => setTimeout(r, 200)); + } + assert.equal(npm, ""); + const left = await leftovers("node"); + assert.ok(!left.includes(`./versions/${VERSION}`) && !left.includes("./install.lock"), left.join(" ")); + + await installRemote(input("node").value); + assert.equal(await version("node"), VERSION); +}); + +test("two installs at once: one waits, the package is sent once, both return the same path", { skip, timeout: 10 * 60_000 }, async () => { + await reset("node"); + const a = input("node"); + const b = input("node", { onProgress: (step) => a.progress.push(step), timeouts: { lockPoll: 500 } }); + + const [first, second] = await Promise.all([installRemote(a.value), installRemote(b.value)]); + + assert.deepEqual(first, second); + assert.equal(a.progress.filter((p) => p.step === "waiting-for-lock").length, 1); + assert.equal(a.progress.filter((p) => p.step === "package").length, 1); +}); + +test("a package with the wrong integrity fails the install and leaves nothing", { skip, timeout: 10 * 60_000 }, async () => { + await reset("node"); + const { value } = input("node"); + const wrong = { ...value.artifacts.package, url: "https://registry.npmjs.org/commander/-/commander-14.0.3.tgz" }; + + await assert.rejects(installRemote({ ...value, published: true, artifacts: { ...value.artifacts, package: wrong } }), /does not match its pinned integrity/); + + assert.deepEqual(await leftovers("node"), ["./versions"]); + assert.equal(await inContainer("node", "ls ~/.local/bin 2>/dev/null | wc -l"), "0"); +}); + +test("an existing ~/.local/bin/whiteboard that Desktop did not write is left alone", { skip, timeout: 10 * 60_000 }, async () => { + await reset("node"); + await inContainer("node", "mkdir -p ~/.local/bin && printf '#!/bin/sh\\necho mine\\n' > ~/.local/bin/whiteboard && chmod 755 ~/.local/bin/whiteboard"); + + await installRemote(input("node").value); + + assert.equal(await inContainer("node", "~/.local/bin/whiteboard"), "mine"); + assert.equal(JSON.parse(await inContainer("node", `~/.dev/whiteboard-remote/versions/${VERSION}/whiteboard version --json`)).version, VERSION); +}); + +test("with three versions installed and one running, an install leaves the running one and the newest", { skip, timeout: 10 * 60_000 }, async (t) => { + await reset("node"); + await installRemote(input("node").value); + await inContainer( + "node", + `cd ~/.dev/whiteboard-remote/versions && for v in ${VERSION}-old.1 ${VERSION}-old.2; do cp -a ${VERSION} $v && sed -i "s#/versions/${VERSION}/#/versions/$v/#g" $v/.whiteboard-install.json; done + setsid nohup node -e 'setInterval(() => {}, 1000)' "$HOME/.dev/whiteboard-remote/versions/${VERSION}-old.1/node_modules/@dev.fast/whiteboard/dist/cli.js" >/dev/null 2>&1 < /dev/null &`, + ); + // The bracket keeps pkill from matching its own shell. + t.after(() => inContainer("node", `pkill -f '[${VERSION[0]}]${VERSION.slice(1)}-old.1/' || true`)); + await inContainer("node", `rm -rf ~/.dev/whiteboard-remote/versions/${VERSION}`); + + await installRemote(input("node").value); + + assert.equal(await inContainer("node", "ls ~/.dev/whiteboard-remote/versions | tr '\\n' ' '"), `${VERSION} ${VERSION}-old.1`); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteInstaller.test.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteInstaller.test.ts new file mode 100644 index 000000000..47639f3db --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteInstaller.test.ts @@ -0,0 +1,520 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from "node:assert/strict"; +import { execFileSync, spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { chmod, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import test from "node:test"; + +import type { SpawnSsh } from "./reviewRemoteHost.js"; +import { probeRemote } from "./reviewRemoteProbe.js"; +import { uninstallRemote } from "./reviewRemoteUninstall.js"; +import { compareVersions, installRemote, type ReviewRemoteInstallInput, type ReviewRemoteInstallProgress } from "./reviewRemoteInstaller.js"; +import { + lockScript, + refreshScript, + nodePlaceScript, + partScript, + REVIEW_REMOTE_INSTALL_MARKER, + REVIEW_REMOTE_WRAPPER_MARK, + shellQuote, +} from "./reviewRemoteInstallScript.js"; +import type { ReviewRemoteProbe } from "./reviewRemoteProbe.js"; +import { reviewSshSession } from "./reviewSshCommand.js"; + +const VERSION = "9.9.9"; + +const localRemote = + (home: string, uploadDelaySeconds = 0, extra: NodeJS.ProcessEnv = {}): SpawnSsh => + (args, options) => { + let command = args.slice(args.indexOf("--") + 2).join(" "); + if (uploadDelaySeconds && command.includes("cat >")) command = `sleep ${uploadDelaySeconds}; ${command}`; + const env = { HOME: home, PATH: `${dirname(process.execPath)}:/usr/bin:/bin:/usr/sbin:/sbin`, ...extra }; + return spawn("/bin/sh", ["-c", `${command}; exit $?`], { ...options, env }); + }; + +async function tarball(dir: string, cli: string): Promise<{ file: string; integrity: string }> { + const source = join(dir, `source-${createHash("sha256").update(cli).digest("hex").slice(0, 8)}`); + await mkdir(join(source, "package", "dist"), { recursive: true }); + await writeFile( + join(source, "package", "package.json"), + JSON.stringify({ name: "@dev.fast/whiteboard", version: VERSION, bin: { whiteboard: "./dist/cli.js" } }), + ); + await writeFile(join(source, "package", "dist", "cli.js"), cli); + const file = join(source, "package.tgz"); + execFileSync("tar", ["-czf", file, "-C", source, "package"]); + const bytes = await readFile(file); + return { file, integrity: `sha512-${createHash("sha512").update(bytes).digest("base64")}` }; +} + +const CLI = `const [a, b] = process.argv.slice(2); +if (a === "version") console.log(JSON.stringify({ event: "version", version: "${VERSION}" })); +else console.log("ran " + process.argv.slice(2).join(" ")); +`; + +async function fixture(t: test.TestContext, cli = CLI) { + const root = await mkdtemp(join(tmpdir(), "wb-install-")); + t.after(() => rm(root, { recursive: true, force: true })); + const home = join(root, "a home"); + const cache = join(root, "cache"); + await mkdir(home); + await mkdir(cache); + const pack = await tarball(root, cli); + const hex = Buffer.from(pack.integrity.slice(7), "base64").toString("hex"); + await writeFile(join(cache, `sha512-${hex}`), await readFile(pack.file)); + const probe: ReviewRemoteProbe = { + os: "Linux", + arch: "aarch64", + glibc: "2.35", + home, + root: join(home, ".dev", "whiteboard-remote"), + homeWritable: true, + freeBytes: 20e9, + node: { path: process.execPath, version: "24.18.0" }, + npm: join(dirname(process.execPath), "npm"), + installed: [], + managedNode: null, + pathCli: null, + downloader: "curl", + registryReachable: true, + tools: ["tar", "xz", "sha256sum", "sha512sum"], + }; + const progress: ReviewRemoteInstallProgress[] = []; + const input = (change: Partial = {}): ReviewRemoteInstallInput => ({ + session: reviewSshSession("devbox", tmpdir()), + probe, + target: "linux-arm64", + version: VERSION, + onProgress: (step) => progress.push(step), + signal: new AbortController().signal, + artifacts: { + package: { name: "dev.fast-whiteboard-9.9.9.tgz", url: "file:///unused", integrity: pack.integrity }, + node: { name: "node-v24.18.0-linux-arm64.tar.xz", url: "https://nodejs.invalid/node.tar.xz", sha256: "0".repeat(64) }, + }, + published: false, + cacheDirectory: cache, + spawn: localRemote(home), + env: {}, + owner: "laptop-test", + timeouts: { lockPoll: 50 }, + ...change, + }); + const remoteRoot = join(home, ".dev", "whiteboard-remote"); + const launcher = join(remoteRoot, "versions", VERSION, "whiteboard"); + return { root, home, cache, pack, probe, progress, input, remoteRoot, launcher, versions: () => readdir(join(remoteRoot, "versions")).catch((): string[] => []) }; +} + +const run = (file: string, ...args: string[]) => execFileSync(file, args, { encoding: "utf8" }).trim(); + +test("a DEV_REVIEW_HOME the remote cannot normalise is refused by the probe and the uninstall, before anything is written", async (t) => { + const f = await fixture(t); + for (const home of ["relative/home", `${f.root}/a/../b`, `${f.root}//b`, `${f.root}/./b`, `${f.root}/a\nb`, `${f.root}/a\tb`]) { + const spawn = localRemote(f.home, 0, { DEV_REVIEW_HOME: home }); + const probed = await probeRemote({ session: reviewSshSession("devbox", tmpdir()), spawn, env: {} }); + assert.match("error" in probed ? probed.error : "", /DEV_REVIEW_HOME there is not an absolute, normalised path/, home); + await assert.rejects(uninstallRemote({ session: reviewSshSession("devbox", tmpdir()), spawn, env: {} }), /DEV_REVIEW_HOME there is not an absolute, normalised path/); + } + assert.deepEqual(await readdir(f.home), []); +}); + +test("DEV_REVIEW_HOME moves the probe's versions, the install, its launcher and the uninstall together", async (t) => { + const uninstalled = `else if (a === "remote" && b === "uninstall") console.log(JSON.stringify({ event: "remote.uninstall", ok: true, removed: [], keptReviews: true }));\n`; + const f = await fixture(t, CLI.replace("else console.log", `${uninstalled}else console.log`)); + const moved = join(f.root, "moved home"); + const spawn = localRemote(f.home, 0, { DEV_REVIEW_HOME: ` ${moved}/ ` }); + const probed = async () => { + const result = await probeRemote({ session: reviewSshSession("devbox", tmpdir()), spawn, env: {} }); + assert.ok("probe" in result, "error" in result ? result.error : ""); + return result.probe; + }; + const root = join(moved, "whiteboard-remote"); + + const before = await probed(); + assert.equal(before.root, root); + const result = await installRemote(f.input({ probe: { ...f.probe, root: before.root }, spawn })); + + assert.ok(result.cliPath.startsWith(`${root}/versions/${VERSION}/`), result.cliPath); + assert.equal(run(`${root}/versions/${VERSION}/whiteboard`, "version", "--json"), `{"event":"version","version":"${VERSION}"}`); + assert.deepEqual((await probed()).installed, [{ version: VERSION, integrity: f.pack.integrity }]); + await uninstallRemote({ session: reviewSshSession("devbox", tmpdir()), spawn, env: {} }); + assert.deepEqual(await readdir(f.home), [".local"]); +}); + +test("installs the package, its launcher and ~/.local/bin/whiteboard; a second call finds it complete", async (t) => { + const f = await fixture(t); + + const result = await installRemote(f.input()); + + const dir = join(f.remoteRoot, "versions", VERSION); + assert.equal(result.cliPath, join(dir, "node_modules/@dev.fast/whiteboard/dist/cli.js")); + assert.equal(result.nodePath, process.execPath); + assert.deepEqual(f.progress, [{ step: "package", via: "upload" }, { step: "verifying" }, { step: "done", cliPath: result.cliPath }]); + assert.equal(run(f.launcher, "version", "--json"), `{"event":"version","version":"${VERSION}"}`); + const wrapper = join(f.home, ".local/bin/whiteboard"); + assert.ok((await readFile(wrapper, "utf8")).includes(REVIEW_REMOTE_WRAPPER_MARK)); + assert.equal(run(wrapper, "a b"), "ran a b"); + const marker = JSON.parse(await readFile(join(dir, REVIEW_REMOTE_INSTALL_MARKER), "utf8")); + assert.deepEqual({ ...marker, installedAt: 0 }, { version: VERSION, integrity: f.pack.integrity, node: process.execPath, cli: result.cliPath, installedAt: 0 }); + assert.deepEqual((await readdir(f.remoteRoot)).sort(), ["versions"]); + assert.deepEqual(await f.versions(), [VERSION]); + + f.progress.length = 0; + assert.deepEqual(await installRemote(f.input()), result); + assert.deepEqual(f.progress, [{ step: "done", cliPath: result.cliPath }]); +}); + +test("two installs at once: one waits, the package is sent once, both get the same path", async (t) => { + const f = await fixture(t); + + const [a, b] = await Promise.all([installRemote(f.input()), installRemote(f.input())]); + + assert.deepEqual(a, b); + assert.equal(f.progress.filter((p) => p.step === "waiting-for-lock").length, 1); + assert.equal(f.progress.filter((p) => p.step === "package").length, 1); + assert.deepEqual(await f.versions(), [VERSION]); +}); + +test("another pack under the same version is installed again", async (t) => { + const f = await fixture(t); + await installRemote(f.input()); + const other = await tarball(f.root, `${CLI}// another build\n`); + const hex = Buffer.from(other.integrity.slice(7), "base64").toString("hex"); + await writeFile(join(f.cache, `sha512-${hex}`), await readFile(other.file)); + f.progress.length = 0; + + const result = await installRemote(f.input({ artifacts: { ...f.input().artifacts, package: { ...f.input().artifacts.package, integrity: other.integrity } } })); + + assert.ok(f.progress.some((p) => p.step === "package")); + assert.match(await readFile(result.cliPath, "utf8"), /another build/); + assert.equal(JSON.parse(await readFile(join(dirname(f.launcher), REVIEW_REMOTE_INSTALL_MARKER), "utf8")).integrity, other.integrity); + assert.deepEqual(await f.versions(), [VERSION]); +}); + +test("a package the remote downloaded that does not match fails the install and leaves nothing", async (t) => { + const f = await fixture(t); + const server = createServer((_req, res) => res.end("not the package")); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + t.after(() => new Promise((resolve) => server.close(resolve))); + const url = `http://127.0.0.1:${(server.address() as AddressInfo).port}/whiteboard-9.9.9.tgz`; + + await assert.rejects( + installRemote(f.input({ published: true, artifacts: { ...f.input().artifacts, package: { ...f.input().artifacts.package, url } } })), + /does not match its pinned integrity; it was removed/, + ); + + assert.deepEqual(f.progress, [{ step: "package", via: "remote-download" }]); + assert.deepEqual(await f.versions(), []); + assert.deepEqual((await readdir(f.remoteRoot)).sort(), ["versions"]); +}); + +test("an existing ~/.local/bin/whiteboard that Desktop did not write is left alone", async (t) => { + const f = await fixture(t); + const wrapper = join(f.home, ".local/bin/whiteboard"); + await mkdir(dirname(wrapper), { recursive: true }); + await writeFile(wrapper, "#!/bin/sh\necho mine\n", { mode: 0o755 }); + + await installRemote(f.input()); + + assert.equal(await readFile(wrapper, "utf8"), "#!/bin/sh\necho mine\n"); + assert.equal(run(f.launcher, "version", "--json"), `{"event":"version","version":"${VERSION}"}`); +}); + +async function earlierVersion(remoteRoot: string, version: string): Promise { + const dir = join(remoteRoot, "versions", version); + await mkdir(join(dir, "node_modules"), { recursive: true }); + await writeFile(join(dir, REVIEW_REMOTE_INSTALL_MARKER), JSON.stringify({ version, integrity: "sha512-old", node: process.execPath, cli: join(dir, "cli.js") })); + return dir; +} + +test("after an install, versions other than the newest two are removed, and a version left without its marker", async (t) => { + const f = await fixture(t); + for (const version of ["1.0.0", "2.0.0", "2.0.0-preview.1", "10.0.0-preview.3"]) await earlierVersion(f.remoteRoot, version); + await mkdir(join(f.remoteRoot, "versions", "notes")); + await mkdir(join(f.remoteRoot, "versions", "3.0.0", "node_modules"), { recursive: true }); + await mkdir(join(f.remoteRoot, "node", "v24.18.0"), { recursive: true }); + + await installRemote(f.input()); + + assert.deepEqual((await f.versions()).sort(), ["10.0.0-preview.3", VERSION, "notes"].sort()); + assert.deepEqual(await readdir(join(f.remoteRoot, "node")), ["v24.18.0"]); +}); + +test("with three versions installed and one running, cleanup leaves the running one and the newest", async (t) => { + const f = await fixture(t); + const running = await earlierVersion(f.remoteRoot, "1.0.0"); + await earlierVersion(f.remoteRoot, "2.0.0"); + const server = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)", join(running, "node_modules/@dev.fast/whiteboard/dist/cli.js"), "server", "start"], { stdio: "ignore" }); + t.after(() => server.kill("SIGKILL")); + + await installRemote(f.input()); + + assert.deepEqual((await f.versions()).sort(), ["1.0.0", VERSION]); + assert.equal(server.exitCode, null); + + server.kill("SIGKILL"); + await new Promise((resolve) => server.once("exit", resolve)); + await earlierVersion(f.remoteRoot, "2.0.0"); + await rm(join(f.remoteRoot, "versions", VERSION), { recursive: true }); + await installRemote(f.input()); + assert.deepEqual((await f.versions()).sort(), ["2.0.0", VERSION]); +}); + +test("an install of an older version keeps itself and the newest", async (t) => { + const f = await fixture(t); + for (const version of ["10.0.0", "11.0.0", "12.0.0"]) await earlierVersion(f.remoteRoot, version); + + await installRemote(f.input()); + + assert.deepEqual((await f.versions()).sort(), ["12.0.0", VERSION]); +}); + +test("an abort while npm runs ends it, leaves no version and no lock, and the next install succeeds", async (t) => { + const f = await fixture(t); + const npm = join(f.root, "slow-npm"); + const seconds = `47.${Math.floor(Math.random() * 1000)}1`; + await writeFile(npm, `#!/bin/sh\ntouch "$HOME/npm-started"\nexec sleep ${seconds}\n`); + await chmod(npm, 0o755); + const abort = new AbortController(); + const started = join(f.home, "npm-started"); + const watch = setInterval(() => void readFile(started).then(() => abort.abort(), () => {}), 50); + t.after(() => clearInterval(watch)); + + await assert.rejects(installRemote(f.input({ probe: { ...f.probe, npm }, signal: abort.signal })), { name: "AbortError" }); + + let alive = ""; + for (let i = 0; i < 50; i++) { + alive = spawnText("pgrep", ["-f", `sleep ${seconds}`]); + if (!alive) break; + await new Promise((resolve) => setTimeout(resolve, 100)); + } + assert.equal(alive, ""); + assert.ok(!(await f.versions()).includes(VERSION)); + assert.ok(!(await readdir(f.remoteRoot)).includes("install.lock")); + + await installRemote(f.input()); + assert.equal(run(f.launcher, "version", "--json"), `{"event":"version","version":"${VERSION}"}`); + assert.deepEqual(await f.versions(), [VERSION]); +}); + +function spawnText(command: string, args: string[]): string { + try { + return execFileSync(command, args, { encoding: "utf8" }).trim(); + } catch { + return ""; + } +} + +test("a stale lock is taken over; a fresh one is waited for, within the bound", async (t) => { + const f = await fixture(t); + const lock = join(f.remoteRoot, "install.lock"); + await mkdir(lock, { recursive: true }); + await writeFile(join(lock, "token"), "0123456789abcdef\n"); + await writeFile(join(lock, "owner"), "gone-laptop\n"); + await writeFile(join(lock, "started"), `${Math.floor(Date.now() / 1000) - 16 * 60}\n`); + + const installed = await installRemote(f.input()); + assert.ok(!(await readdir(f.remoteRoot)).includes("install.lock")); + + await mkdir(lock); + await writeFile(join(lock, "token"), "0123456789abcdef\n"); + await writeFile(join(lock, "started"), `${Math.floor(Date.now() / 1000)}\n`); + f.progress.length = 0; + assert.deepEqual(await installRemote(f.input({ timeouts: { lockPoll: 50, lockWait: 400 } })), installed); + assert.deepEqual(f.progress, [{ step: "done", cliPath: installed.cliPath }]); + await rm(lock, { recursive: true }); + + await rm(join(f.remoteRoot, "versions"), { recursive: true }); + await mkdir(lock); + await writeFile(join(lock, "token"), "0123456789abcdef\n"); + await writeFile(join(lock, "owner"), "busy-laptop\n"); + await writeFile(join(lock, "started"), `${Math.floor(Date.now() / 1000)}\n`); + f.progress.length = 0; + + await assert.rejects(installRemote(f.input({ timeouts: { lockPoll: 50, lockWait: 400 } })), /Another install on devbox \(busy-laptop\) held the lock/); + assert.deepEqual(f.progress, [{ step: "waiting-for-lock" }]); + assert.equal(await readFile(join(lock, "token"), "utf8"), "0123456789abcdef\n"); +}); + +function sh(home: string, script: string): Promise { + return new Promise((resolve, reject) => { + const child = localRemote(home)(["--", "devbox", "sh", "-s"], { env: {}, detached: true, stdio: ["pipe", "pipe", "pipe"] }); + let out = ""; + child.stdout!.setEncoding("utf8").on("data", (chunk: string) => (out += chunk)); + child.once("error", reject); + child.once("close", () => resolve(out)); + child.stdin!.end(script); + }); +} + +test("a Node tarball with the wrong checksum is removed before it is unpacked", async (t) => { + const f = await fixture(t); + const context = { home: f.home, root: f.remoteRoot, token: "00112233aabbccdd" }; + assert.match(await sh(f.home, lockScript(context, "me")), /LOCKED/); + assert.match(await sh(f.home, partScript(context, { node: "24.18.0" })), /READY/); + const part = join(f.remoteRoot, "node", "v24.18.0.00112233aabbccdd.part"); + await writeFile(join(part, "node.tar.xz"), "not node"); + + const out = await sh(f.home, nodePlaceScript(context, { nodeVersion: "24.18.0", sha256: "a".repeat(64) })); + + assert.match(out, /WHITEBOARD-INSTALL MISMATCH [0-9a-f]{64}/); + assert.deepEqual(await readdir(join(f.remoteRoot, "node")), []); +}); + +test("shellQuote survives quotes and refuses control characters", async () => { + for (const value of ["plain", "it's", "a 'b' \"c\" $d `e` \\f", "'", ""]) { + assert.equal(run("/bin/sh", "-c", `printf '%s' ${shellQuote(value)}`), value); + } + assert.throws(() => shellQuote("a\nb"), /control character/); + assert.throws(() => shellQuote("a\u0007b"), /control character/); +}); + +test("versions order as semver", () => { + const sorted = ["0.1.10", "0.1.7", "0.1.7-preview.20261003.10", "0.1.6", "0.1.7-preview.20261003.2", "0.2.0-preview.1"].sort(compareVersions); + assert.deepEqual(sorted, ["0.1.6", "0.1.7-preview.20261003.2", "0.1.7-preview.20261003.10", "0.1.7", "0.1.10", "0.2.0-preview.1"]); +}); + +test("a lock taken over between the stealer's two reads is left to its new holder", async (t) => { + const f = await fixture(t); + const lock = join(f.remoteRoot, "install.lock"); + await mkdir(lock, { recursive: true }); + await writeFile(join(lock, "token"), "0123456789abcdef\n"); + await writeFile(join(lock, "started"), "0\n"); + const bin = join(f.root, "racing-bin"); + await mkdir(bin); + await writeFile( + join(bin, "cat"), + `#!/bin/sh +/bin/cat "$@" +case "$1" in */install.lock/*) + [ -e "$RACED" ] && exit 0 + : > "$RACED" + rm -rf "$LOCK" && mkdir "$LOCK" && echo fedcba9876543210 > "$LOCK/token" && echo winner > "$LOCK/owner" && date +%s > "$LOCK/started" ;; +esac +`, + { mode: 0o755 }, + ); + + const out = await new Promise((resolve, reject) => { + const child = spawn("/bin/sh", ["-s"], { + env: { HOME: f.home, LOCK: lock, RACED: join(f.root, "raced"), PATH: `${bin}:/usr/bin:/bin:/usr/sbin:/sbin` }, + stdio: ["pipe", "pipe", "inherit"], + }); + let text = ""; + child.stdout.setEncoding("utf8").on("data", (chunk: string) => (text += chunk)); + child.once("error", reject); + child.once("close", () => resolve(text)); + child.stdin.end(lockScript({ home: f.home, root: f.remoteRoot, token: "00112233aabbccdd" }, "stealer")); + }); + + assert.match(out, /WHITEBOARD-INSTALL BUSY winner/); + assert.equal(await readFile(join(lock, "token"), "utf8"), "fedcba9876543210\n"); +}); + +test("a long npm step and a long upload keep the lock fresh, so a second install waits", async (t) => { + for (const slow of ["npm", "upload"] as const) { + const f = await fixture(t); + const npm = join(f.root, "slow-npm"); + await writeFile(npm, `#!/bin/sh\nsleep 5\nexec ${shellQuote(join(dirname(process.execPath), "npm"))} "$@"\n`, { mode: 0o755 }); + const timeouts = { lockPoll: 100, lockStale: 3000 }; + const first = installRemote( + f.input({ + timeouts, + ...(slow === "npm" ? { probe: { ...f.probe, npm } } : { spawn: localRemote(f.home, 5) }), + }), + ); + await new Promise((resolve) => setTimeout(resolve, 500)); + const second = installRemote(f.input({ timeouts })); + + const [a, b] = await Promise.all([first, second]); + + assert.deepEqual(a, b, slow); + assert.deepEqual( + f.progress.filter((p) => p.step !== "verifying" && p.step !== "done"), + [{ step: "package", via: "upload" }, { step: "waiting-for-lock" }], + slow, + ); + } +}); + +test("an abort during an upload is an AbortError", async (t) => { + const f = await fixture(t); + const abort = new AbortController(); + + await assert.rejects( + installRemote( + f.input({ + spawn: localRemote(f.home, 5), + signal: abort.signal, + onProgress: (step) => void (step.step === "package" && setTimeout(() => abort.abort(), 300)), + }), + ), + { name: "AbortError" }, + ); + assert.ok(!(await f.versions()).includes(VERSION)); +}); + +test("a marker whose integrity only contains the pin is not complete", async (t) => { + const f = await fixture(t); + await installRemote(f.input()); + const marker = join(dirname(f.launcher), REVIEW_REMOTE_INSTALL_MARKER); + const text = await readFile(marker, "utf8"); + await writeFile(marker, text.replace(f.pack.integrity, `${f.pack.integrity}x`)); + f.progress.length = 0; + + await installRemote(f.input()); + + assert.ok(f.progress.some((p) => p.step === "package")); + assert.equal(JSON.parse(await readFile(marker, "utf8")).integrity, f.pack.integrity); +}); + +test("a Node path with a quote is refused before anything runs", async (t) => { + const f = await fixture(t); + + await assert.rejects( + installRemote(f.input({ probe: { ...f.probe, node: { path: '/opt/no"de/bin/node', version: "24.18.0" } } })), + /Whiteboard cannot install on devbox: .* holds a quote or backslash/, + ); + assert.deepEqual(await readdir(f.home), []); +}); + +test("a read of the lock's start time during refreshes never finds it empty", async (t) => { + const f = await fixture(t); + const context = { home: f.home, root: f.remoteRoot, token: "00112233aabbccdd" }; + assert.match(await sh(f.home, lockScript(context, "me")), /LOCKED/); + const started = join(f.remoteRoot, "install.lock", "started"); + const refreshes = sh(f.home, refreshScript(context).replace("own\n", "i=0\nwhile [ $i -lt 400 ]; do own; i=$((i + 1)); done\n")); + let done = false; + void refreshes.then(() => (done = true)); + + let reads = 0; + const bad: string[] = []; + while (!done) { + const text = await readFile(started, "utf8").catch((error: NodeJS.ErrnoException) => error.code ?? "error"); + reads++; + if (!/^\d+\n$/.test(text)) bad.push(JSON.stringify(text)); + } + + assert.match(await refreshes, /REFRESHED/); + assert.ok(reads > 50, `${reads} reads`); + assert.deepEqual(bad, []); +}); + +test("a lock whose start time cannot be read counts as fresh", async (t) => { + const f = await fixture(t); + const lock = join(f.remoteRoot, "install.lock"); + await mkdir(lock, { recursive: true }); + await writeFile(join(lock, "token"), "0123456789abcdef\n"); + await writeFile(join(lock, "owner"), "busy-laptop\n"); + await writeFile(join(lock, "started"), ""); + execFileSync("touch", ["-t", "202001010000", lock]); + + assert.match(await sh(f.home, lockScript({ home: f.home, root: f.remoteRoot, token: "00112233aabbccdd" }, "me")), /BUSY busy-laptop/); + assert.equal(await readFile(join(lock, "token"), "utf8"), "0123456789abcdef\n"); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteInstaller.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteInstaller.ts new file mode 100644 index 000000000..949eb2564 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteInstaller.ts @@ -0,0 +1,410 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { randomBytes } from "node:crypto"; +import { hostname } from "node:os"; + +import { fetchToLaptopCache, type ReviewRemoteArtifact } from "./reviewRemoteArtifacts.js"; +import { runSsh, type RunResult, type SpawnSsh, type SshChildProcess } from "./reviewRemoteHost.js"; +import { + cleanupScript, + completeScript, + downloadScript, + finishScript, + lockScript, + nodePlaceScript, + nodeTarball, + packageInstallScript, + packageTarball, + partScript, + prepareScript, + refreshScript, + releaseScript, + REVIEW_REMOTE_INSTALL_SAY, + REVIEW_REMOTE_LOCK_STALE_SECONDS, + REVIEW_REMOTE_VERSION, + REVIEW_REMOTE_WRAPPER_MARK, + reviewRemoteNodeDir, + reviewRemoteVersionDir, + shellQuote, + verifyScript, + type ReviewRemoteInstallContext, +} from "./reviewRemoteInstallScript.js"; +import type { ReviewRemoteProbe, ReviewRemoteTarget } from "./reviewRemoteProbe.js"; +import { startRegistryRelay, type ReviewRegistryRelay } from "./reviewRemoteRegistryRelay.js"; +import { uploadFile } from "./reviewRemoteUpload.js"; +import { sshExecArgs, sshRemoteForwardArgs, type ReviewSshSession } from "./reviewSshCommand.js"; + +export type ReviewRemoteInstallProgress = + | { step: "waiting-for-lock" } + | { step: "node"; via: "remote-download" | "upload" } + | { step: "package"; via: "remote-download" | "upload" } + | { step: "verifying" } + | { step: "done"; cliPath: string }; + +export const REVIEW_REMOTE_INSTALL_TIMEOUTS = { + step: 30_000, + download: 15 * 60_000, + npm: 10 * 60_000, + lockWait: REVIEW_REMOTE_LOCK_STALE_SECONDS * 1000, + lockStale: REVIEW_REMOTE_LOCK_STALE_SECONDS * 1000, + lockPoll: 2_000, + release: 15_000, +}; + +export interface ReviewRemoteInstallInput { + readonly session: ReviewSshSession; + readonly probe: ReviewRemoteProbe; + readonly target: ReviewRemoteTarget; + readonly version: string; + onProgress(progress: ReviewRemoteInstallProgress): void; + readonly signal: AbortSignal; + readonly artifacts: { readonly package: ReviewRemoteArtifact; readonly node: ReviewRemoteArtifact }; + readonly published: boolean; + readonly cacheDirectory: string; + readonly spawn: SpawnSsh; + readonly env: NodeJS.ProcessEnv; + readonly owner?: string; + readonly timeouts?: Partial; +} + +export interface ReviewRemoteInstallResult { + readonly cliPath: string; + readonly nodePath: string; +} + +const BIN = /^[\w.-]+(\/[\w.-]+)*$/; + +export async function installRemote(input: ReviewRemoteInstallInput): Promise { + const { session, probe, signal } = input; + const timeouts = { ...REVIEW_REMOTE_INSTALL_TIMEOUTS, ...input.timeouts }; + const alias = session.alias; + signal.throwIfAborted(); + // The marker is JSON read back with sed: no quote or backslash in any path it holds. + for (const path of [probe.home, probe.root, probe.node?.path, probe.npm]) { + if (path && /['"\\]/.test(path)) throw new Error(`Whiteboard cannot install on ${alias}: ${JSON.stringify(path)} holds a quote or backslash.`); + } + if (!REVIEW_REMOTE_VERSION.test(input.version)) throw new Error(`${JSON.stringify(input.version)} is not a version.`); + const { integrity, sha512, nodeVersion, nodeSha256 } = pinned(input.artifacts); + if (!input.artifacts.node.name.endsWith(`-${input.target}.tar.xz`)) throw new Error(`${input.artifacts.node.name} is not the Node for ${input.target}.`); + + const context: ReviewRemoteInstallContext = { home: probe.home, root: probe.root, token: randomBytes(8).toString("hex") }; + const owner = (input.owner ?? hostname()).replace(/[^\w.-]/g, "-").slice(0, 64) || "unknown"; + + const children = new Set(); + const tracked: SpawnSsh = (args, options) => { + const child = input.spawn(args, options); + children.add(child); + child.once("close", () => children.delete(child)); + return child; + }; + const onAbort = () => { + for (const child of children) { + child.kill("SIGKILL"); + child.stdout?.destroy(); + child.stderr?.destroy(); + } + }; + signal.addEventListener("abort", onAbort, { once: true }); + + const run = async (what: string, script: string, timeout = timeouts.step): Promise => { + signal.throwIfAborted(); + const result = await runSsh(tracked, input.env, sshExecArgs(session, input.env), timeout, script); + signal.throwIfAborted(); + return answer(alias, what, result, timeout); + }; + + let relay: { server: ReviewRegistryRelay; forwarded: boolean } | undefined; + let locking = false; + try { + const complete = await alreadyComplete(); + if (complete) return complete; + locking = true; + await takeLock(); + return await install(); + } finally { + signal.removeEventListener("abort", onAbort); + if (relay) await closeRelay(relay); + if (locking) await runSsh(input.spawn, input.env, sshExecArgs(session, input.env), timeouts.release, releaseScript(context)); + } + + async function alreadyComplete(): Promise { + const checked = await run("checking the installed version", completeScript(context, { version: input.version, integrity })).catch((error: unknown) => { + if (signal.aborted) throw error; + return undefined; + }); + const marker = checked?.has("COMPLETE") ? readMarker(checked.get("MARKER")) : undefined; + if (!marker) return undefined; + input.onProgress({ step: "done", cliPath: marker.cli }); + return { nodePath: marker.node, cliPath: marker.cli }; + } + + async function takeLock(): Promise { + const deadline = Date.now() + timeouts.lockWait; + let told = false; + for (;;) { + const says = await run("taking the install lock", lockScript(context, owner, Math.ceil(timeouts.lockStale / 1000))); + if (says.has("LOCKED")) return; + const holder = says.get("BUSY"); + if (holder === undefined) throw says.failure(); + if (Date.now() >= deadline) { + throw new Error(`Another install on ${alias} (${holder || "unknown"}) held the lock for ${timeouts.lockWait / 60_000} minutes.`); + } + if (!told) input.onProgress({ step: "waiting-for-lock" }); + told = true; + await delay(timeouts.lockPoll, signal); + } + } + + async function install(): Promise { + const prepared = await run("preparing", prepareScript(context, { version: input.version, integrity, nodeVersion })); + if (!prepared.has("PREPARED")) throw prepared.failure(); + + if (prepared.has("COMPLETE")) { + const marker = readMarker(prepared.get("MARKER")); + if (marker) return finish({ nodePath: marker.node, cliPath: marker.cli }); + } + + const { node, npm } = await ensureNode(prepared.has("MANAGED-NODE")); + await placePackage(node, npm); + if (relay) await closeRelay(relay); + relay = undefined; + + input.onProgress({ step: "verifying" }); + const verified = await run("verifying", verifyScript(context, { version: input.version, node })); + const bin = verified.get("BIN"); + if (bin === undefined || !BIN.test(bin) || bin.split("/").includes("..")) throw verified.failure(); + const reported = parseVersion(verified.get("VERSION")); + if (reported !== input.version) { + throw new Error(`The package installed on ${alias} reports version ${reported ?? "nothing"}, not ${input.version}.`); + } + + const dir = reviewRemoteVersionDir(probe.root, input.version); + const cliPath = `${dir}/node_modules/@dev.fast/whiteboard/${bin}`; + const launcher = `${dir}/whiteboard`; + const newest = [input.version, ...prepared.all("HAVE").filter((name) => REVIEW_REMOTE_VERSION.test(name))].sort(compareVersions).at(-1); + const finished = await run( + "finishing", + finishScript(context, { + version: input.version, + launcher: `#!/bin/sh\nexec ${shellQuote(node)} ${shellQuote(cliPath)} "$@"\n`, + marker: JSON.stringify({ version: input.version, integrity, node, cli: cliPath, installedAt: Math.floor(Date.now() / 1000) }), + wrapper: newest === input.version ? `#!/bin/sh\n${REVIEW_REMOTE_WRAPPER_MARK}\nexec ${shellQuote(launcher)} "$@"\n` : undefined, + }), + ); + if (!finished.has("COMPLETE")) throw finished.failure(); + await cleanup(prepared.all("HAVE"), newest); + return finish({ nodePath: node, cliPath }); + } + + async function cleanup(have: string[], newest: string | undefined): Promise { + const keep = new Set([input.version, newest]); + const candidates = have.filter((name) => REVIEW_REMOTE_VERSION.test(name) && !keep.has(name)).sort(compareVersions).reverse(); + if (!candidates.length) return; + await run("removing old versions", cleanupScript(context, { candidates, room: 2 - keep.size })).catch((error: unknown) => { + if (signal.aborted) throw error; + }); + } + + function finish(paths: ReviewRemoteInstallResult): ReviewRemoteInstallResult { + input.onProgress({ step: "done", cliPath: paths.cliPath }); + return paths; + } + + async function ensureNode(managed: boolean): Promise<{ node: string; npm: string }> { + if (probe.node && probe.npm) return { node: probe.node.path, npm: probe.npm }; + const bin = `${reviewRemoteNodeDir(probe.root, nodeVersion)}/bin`; + const paths = { node: `${bin}/node`, npm: `${bin}/npm` }; + if (managed) return paths; + + const missing = (["tar", "xz", "sha256sum"] as const).filter((tool) => !probe.tools.includes(tool)); + if (missing.length) throw new Error(`${alias} has no Node 24, and Whiteboard needs ${missing.join(", ")} there to install one.`); + await fetchOnRemote("node", { node: nodeVersion }, input.artifacts.node, nodeTarball(context, nodeVersion), true); + const placed = await run("unpacking Node", nodePlaceScript(context, { nodeVersion: nodeVersion, sha256: nodeSha256 }), timeouts.npm); + if (placed.has("MISMATCH")) throw new Error(`The Node tarball on ${alias} does not match its pinned checksum; it was removed.`); + if (!placed.has("NODE-OK")) throw placed.failure(); + return paths; + } + + async function placePackage(node: string, npm: string): Promise { + if (!probe.tools.includes("sha512sum") && !probe.tools.includes("openssl")) { + throw new Error(`Whiteboard needs sha512sum or openssl on ${alias} to check the package.`); + } + await fetchOnRemote("package", { package: input.version }, input.artifacts.package, packageTarball(context, input.version), input.published); + const registry = probe.registryReachable ? undefined : await startRelay(); + const installed = await run( + "installing the package", + packageInstallScript(context, { + version: input.version, + target: input.target, + sha512, + node, + npm, + registry, + }), + timeouts.npm, + ); + if (installed.has("MISMATCH")) throw new Error(`The package on ${alias} does not match its pinned integrity; it was removed.`); + if (!installed.has("INSTALLED")) throw installed.failure(); + } + + async function fetchOnRemote( + step: "node" | "package", + part: { node: string } | { package: string }, + artifact: ReviewRemoteArtifact, + file: string, + remoteMayFetch: boolean, + ): Promise { + const ready = await run("creating a work directory", partScript(context, part)); + if (!ready.has("READY")) throw ready.failure(); + if (remoteMayFetch && probe.downloader && probe.registryReachable && /^https?:/.test(artifact.url)) { + input.onProgress({ step, via: "remote-download" }); + const downloaded = await run(`downloading ${artifact.name}`, downloadScript(context, { url: artifact.url, file, downloader: probe.downloader }), timeouts.download); + if (downloaded.has("DOWNLOADED")) return; + } + input.onProgress({ step, via: "upload" }); + const local = await abortable(fetchToLaptopCache(artifact, { cacheDirectory: input.cacheDirectory }), signal); + const refresh = setInterval(() => void runSsh(tracked, input.env, sshExecArgs(session, input.env), timeouts.step, refreshScript(context)), timeouts.lockStale / 5); + try { + await uploadFile(session, local, file, { spawn: tracked, env: input.env }); + } catch (error) { + signal.throwIfAborted(); + throw error; + } finally { + clearInterval(refresh); + } + } + + async function startRelay(): Promise { + const server = await startRegistryRelay(); + relay = { server, forwarded: false }; + const result = await runSsh(tracked, input.env, sshRemoteForwardArgs(session, server.port, "forward", input.env), timeouts.step); + signal.throwIfAborted(); + const port = Number(result.stdout.trim()); + if (result.code !== 0 || !Number.isInteger(port) || port < 1 || port > 65535) { + throw new Error(`${alias} cannot reach the npm registry, and forwarding a port to Desktop's relay failed: ${result.stderr.trim().split("\n").at(-1) ?? `exit ${result.code}`}`); + } + relay.forwarded = true; + return `http://127.0.0.1:${port}/`; + } + + async function closeRelay({ server, forwarded }: { server: ReviewRegistryRelay; forwarded: boolean }): Promise { + if (forwarded) await runSsh(input.spawn, input.env, sshRemoteForwardArgs(session, server.port, "cancel", input.env), timeouts.release).catch(() => undefined); + await server.close(); + } +} + +function pinned(artifacts: ReviewRemoteInstallInput["artifacts"]) { + const integrity = artifacts.package.integrity ?? ""; + const base64 = /^sha512-([A-Za-z0-9+/]{86}==)$/.exec(integrity)?.[1]; + if (!base64) throw new Error(`${artifacts.package.name} has no sha512 integrity.`); + const nodeVersion = /^node-v(\d+\.\d+\.\d+)-linux-(?:x64|arm64)\.tar\.xz$/.exec(artifacts.node.name)?.[1]; + const nodeSha256 = artifacts.node.sha256; + if (!nodeVersion || !nodeSha256) throw new Error(`${artifacts.node.name} is not a pinned Linux Node.`); + return { integrity, sha512: Buffer.from(base64, "base64").toString("hex"), nodeVersion, nodeSha256 }; +} + +interface Answer { + readonly lines: string[]; + has(word: string): boolean; + get(word: string): string | undefined; + all(word: string): string[]; + failure(): Error; +} + +function answer(alias: string, what: string, result: RunResult, timeout: number): Answer { + const lines = result.stdout.split("\n"); + const says = lines.flatMap((line) => { + if (!line.startsWith(`${REVIEW_REMOTE_INSTALL_SAY} `)) return []; + const rest = line.slice(REVIEW_REMOTE_INSTALL_SAY.length + 1); + const space = rest.indexOf(" "); + return [space < 0 ? [rest, ""] : [rest.slice(0, space), rest.slice(space + 1)]]; + }); + const all = (word: string) => says.filter(([said]) => said === word).map(([, value]) => value); + return { + lines, + has: (word) => all(word).length > 0, + get: (word) => all(word).at(-1), + all, + failure() { + const reason = all("FAIL").at(-1); + if (reason) return new Error(`Installing on ${alias} failed while ${what}: ${reason}.`); + if (result.timedOut) return new Error(`Installing on ${alias} failed: ${what} took longer than ${timeout / 1000} seconds.`); + if (result.error) return new Error(`Installing on ${alias} failed while ${what}: ${result.error.message}`); + const stderr = result.stderr.trim().split("\n").at(-1)?.slice(0, 300); + return new Error(`Installing on ${alias} failed while ${what}: exit ${result.code}${stderr ? `: ${stderr}` : ""}.`); + }, + }; +} + +const remotePath = (value: unknown): value is string => + typeof value === "string" && value.startsWith("/") && value.length <= 4096 && !/[\x00-\x1f\x7f-\x9f]/.test(value); + +function readMarker(text: string | undefined): { node: string; cli: string } | undefined { + try { + const marker = JSON.parse(text ?? "") as { node?: unknown; cli?: unknown }; + if (remotePath(marker.node) && remotePath(marker.cli)) { + return { node: marker.node, cli: marker.cli }; + } + } catch { + } + return undefined; +} + +function parseVersion(text: string | undefined): string | undefined { + try { + const value = JSON.parse(text ?? "") as { version?: unknown }; + return typeof value.version === "string" ? value.version : undefined; + } catch { + return undefined; + } +} + +export function compareVersions(a: string, b: string): number { + const [coreA, preA] = split(a); + const [coreB, preB] = split(b); + for (let i = 0; i < 3; i++) if (coreA[i] !== coreB[i]) return coreA[i] - coreB[i]; + if (!preA.length || !preB.length) return preB.length - preA.length; + for (let i = 0; i < Math.max(preA.length, preB.length); i++) { + const x = preA[i]; + const y = preB[i]; + if (x === undefined || y === undefined) return x === undefined ? -1 : 1; + if (x === y) continue; + const nx = /^\d+$/.test(x); + const ny = /^\d+$/.test(y); + if (nx && ny) return Number(x) - Number(y); + if (nx !== ny) return nx ? -1 : 1; + return x < y ? -1 : 1; + } + return 0; +} + +function split(version: string): [number[], string[]] { + const dash = version.indexOf("-"); + const core = (dash < 0 ? version : version.slice(0, dash)).split(".").map(Number); + return [core, dash < 0 ? [] : version.slice(dash + 1).split(".")]; +} + +function delay(ms: number, signal: AbortSignal): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + signal.removeEventListener("abort", stop); + resolve(); + }, ms); + const stop = () => { + clearTimeout(timer); + reject(signal.reason); + }; + signal.addEventListener("abort", stop, { once: true }); + }); +} + +function abortable(promise: Promise, signal: AbortSignal): Promise { + return new Promise((resolve, reject) => { + const stop = () => reject(signal.reason); + signal.addEventListener("abort", stop, { once: true }); + promise.then(resolve, reject).finally(() => signal.removeEventListener("abort", stop)); + }); +} diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteProbe.containers.test.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteProbe.containers.test.ts new file mode 100644 index 000000000..9d7d070b4 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteProbe.containers.test.ts @@ -0,0 +1,176 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from "node:assert/strict"; +import { execFile, execFileSync, spawn } from "node:child_process"; +import { randomBytes } from "node:crypto"; +import { mkdtemp, rm } from "node:fs/promises"; +import { arch, tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { after, before, test } from "node:test"; +import { promisify } from "node:util"; + +import type { SpawnSsh } from "./reviewRemoteHost.js"; +import { judgeRemote, parseRemoteProbe, probeRemote, REVIEW_REMOTE_PROBE_TIMEOUT } from "./reviewRemoteProbe.js"; +import { REVIEW_REMOTE_PROBE_SCRIPT } from "./reviewRemoteProbeScript.js"; +import { REVIEW_SSH_CONFIG_ENV, reviewSshSession } from "./reviewSshCommand.js"; + +const run = promisify(execFile); +const harness = resolve(import.meta.dirname, "../../../../../../scripts/e2e/remote/remote.mjs"); + +function skipReason(): string | undefined { + if (process.env.WB_TEST_CONTAINERS !== "1") return "set WB_TEST_CONTAINERS=1 to run the probe against containers"; + try { + execFileSync("docker", ["info"], { stdio: "ignore" }); + } catch { + return "Docker is not available"; + } + return undefined; +} + +const skip = skipReason(); + +const runPrefix = `s3t1${randomBytes(3).toString("hex")}`; +const hosts = { + fish: ["--shell", "fish", "--banner"], + amd64: ["--platform", "linux/amd64"], + old: ["--image", "debian:11", "--sealed"], + musl: ["--image", "alpine:3.20"], + node20: ["--node", "20"], +} as const; +type Host = keyof typeof hosts; +const runOf = (host: Host) => `${runPrefix}-${host}`; +const containerOf = (host: Host) => `wb-test-${runOf(host)}-${host}`; +const aliasOf = (host: Host) => `wb-test-${host}`; +const started = new Set(); +let controlDirectory = ""; + +const remote = (host: Host, args: string[]) => + run(process.execPath, [harness, ...args], { env: { ...process.env, WB_TEST_RUN: runOf(host) }, maxBuffer: 16 << 20 }); + +const sshEnv = (host: Host) => ({ ...process.env, VSCODE_DEV: "1", [REVIEW_SSH_CONFIG_ENV]: `/tmp/wbt.${runOf(host)}/ssh_config` }); + +const realSsh: SpawnSsh = (args, options) => spawn("ssh", args, options); + +async function probe(host: Host) { + const began = Date.now(); + const result = await probeRemote({ session: reviewSshSession(aliasOf(host), controlDirectory), spawn: realSsh, env: sshEnv(host) }); + const elapsed = Date.now() - began; + assert.ok("probe" in result, "error" in result ? result.error : ""); + assert.ok(elapsed < REVIEW_REMOTE_PROBE_TIMEOUT); + return { probe: result.probe, elapsed }; +} + +async function written(host: Host) { + const { stdout } = await remote(host, ["ssh", host, "--", "find", "~", "-newer", "/etc/hostname"]); + return stdout + .split("\n") + .filter((line) => line.startsWith("/")) + .sort(); +} + +before( + async () => { + if (skip) return; + controlDirectory = await mkdtemp(join(tmpdir(), "wb-probe-ssh-")); + const ups = await Promise.allSettled( + (Object.keys(hosts) as Host[]).map(async (host) => { + started.add(host); + await remote(host, ["up", host, "--node", "none", ...hosts[host]]); + }), + ); + for (const up of ups) if (up.status === "rejected") throw up.reason; + }, + { timeout: 20 * 60_000 }, +); + +after( + async () => { + for (const host of started) await remote(host, ["down", "--all"]).catch((error) => console.error(error.stderr ?? error)); + if (controlDirectory) await rm(controlDirectory, { recursive: true, force: true }); + }, + { timeout: 5 * 60_000 }, +); + +test("ubuntu:22.04 under dash, with a fish login shell and a banner, is supported and left as it was", { skip }, async () => { + const before = await written("fish"); + const { probe: found } = await probe("fish"); + + assert.deepEqual(judgeRemote(found), { supported: true, target: arch() === "arm64" ? "linux-arm64" : "linux-x64" }); + assert.equal(found.home, "/home/dev"); + assert.equal(found.node, null); + assert.deepEqual(found.installed, []); + assert.deepEqual(await written("fish"), before); +}); + +test("the script also runs under bash", { skip }, async () => { + const { stdout } = await new Promise<{ stdout: string }>((resolvePromise, reject) => { + const child = execFile( + "ssh", + ["-F", `/tmp/wbt.${runOf("fish")}/ssh_config`, "-T", "--", aliasOf("fish"), "bash", "-s"], + { timeout: REVIEW_REMOTE_PROBE_TIMEOUT }, + (error, out) => (error ? reject(error) : resolvePromise({ stdout: out })), + ); + child.stdin?.end(REVIEW_REMOTE_PROBE_SCRIPT); + }); + const parsed = parseRemoteProbe(stdout); + + assert.ok("probe" in parsed, "error" in parsed ? parsed.error : ""); + assert.equal(judgeRemote(parsed.probe).supported, true); +}); + +test("a home that cannot be written is refused", { skip }, async (t) => { + const container = containerOf("fish"); + await run("docker", ["exec", container, "chmod", "555", "/home/dev"]); + t.after(() => run("docker", ["exec", container, "chmod", "755", "/home/dev"])); + + const { probe: found } = await probe("fish"); + + assert.deepEqual(judgeRemote(found), { + supported: false, + reason: "The home directory /home/dev cannot be written; Whiteboard needs to write under it.", + }); +}); + +test("ubuntu:22.04 on linux/amd64 is linux-x64", { skip }, async () => { + assert.deepEqual(judgeRemote((await probe("amd64")).probe), { supported: true, target: "linux-x64" }); +}); + +test("debian:11 is refused for glibc 2.31, and a sealed host does not reach the registry", { skip }, async () => { + const { probe: found } = await probe("old"); + + assert.equal(found.glibc, "2.31"); + assert.deepEqual(judgeRemote(found), { supported: false, reason: "This host runs glibc 2.31; Whiteboard needs 2.34 or newer." }); + assert.equal(found.downloader, "curl"); + assert.equal(found.registryReachable, false); +}); + +test("alpine is refused for musl", { skip }, async () => { + const { probe: found } = await probe("musl"); + + assert.equal(found.glibc, null); + const judged = judgeRemote(found); + assert.equal(judged.supported, false); + assert.match(judged.supported ? "" : judged.reason, /no glibc .*musl/); +}); + +test("Node 20 on PATH and Node 24 under nvm: the Node 24 is reported", { skip }, async () => { + const nvm = "/home/dev/.nvm/versions/node/v24.18.0/bin"; + await run("docker", [ + "exec", + "-u", + "dev", + containerOf("node20"), + "sh", + "-c", + `mkdir -p ${nvm} && printf '#!/bin/sh\\necho v24.18.0\\n' > ${nvm}/node && chmod +x ${nvm}/node && ln -s node ${nvm}/npm`, + ]); + + const { probe: found } = await probe("node20"); + + assert.deepEqual(found.node, { path: `${nvm}/node`, version: "24.18.0" }); + assert.equal(found.npm, `${nvm}/npm`); + assert.equal(found.registryReachable, true); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteProbe.test.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteProbe.test.ts new file mode 100644 index 000000000..4a73ab475 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteProbe.test.ts @@ -0,0 +1,309 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from "node:assert/strict"; +import { spawn, spawnSync } from "node:child_process"; +import { chmod, mkdir, mkdtemp, readdir, rm, stat, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import type { SpawnSsh } from "./reviewRemoteHost.js"; +import { REVIEW_REMOTE_INSTALL_MARKER, REVIEW_REMOTE_WRAPPER_MARK } from "./reviewRemoteInstallScript.js"; +import { judgeRemote, parseRemoteProbe, probeRemote, type ReviewRemoteProbe } from "./reviewRemoteProbe.js"; +import { REVIEW_REMOTE_PROBE_BEGIN, REVIEW_REMOTE_PROBE_END, REVIEW_REMOTE_PROBE_PATH_CLI } from "./reviewRemoteProbeScript.js"; +import { reviewSshSession } from "./reviewSshCommand.js"; + +const INTEGRITY = `sha512-${"A".repeat(86)}==`; + +const supported: ReviewRemoteProbe = { + os: "Linux", + arch: "x86_64", + glibc: "2.35", + home: "/home/dev", + root: "/home/dev/.dev/whiteboard-remote", + homeWritable: true, + freeBytes: 20e9, + node: { path: "/home/dev/.nvm/versions/node/v24.18.0/bin/node", version: "24.18.0" }, + npm: "/home/dev/.nvm/versions/node/v24.18.0/bin/npm", + installed: [{ version: "0.1.6", integrity: INTEGRITY }], + managedNode: null, + pathCli: { path: "/usr/local/bin/whiteboard", version: "0.1.6" }, + downloader: "curl", + registryReachable: true, + tools: ["tar", "xz", "sha256sum", "sha512sum"], +}; + +const answer = (value: unknown, before = "", after = "") => { + const { pathCli, ...rest } = value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : { pathCli: undefined }; + const body = value && typeof value === "object" && !Array.isArray(value) ? rest : value; + return `${before}${REVIEW_REMOTE_PROBE_BEGIN}\n${JSON.stringify(body)}\n${REVIEW_REMOTE_PROBE_END}\n${pathCli ? `${REVIEW_REMOTE_PROBE_PATH_CLI} ${JSON.stringify(pathCli)}\n` : ""}${after}`; +}; + +test("judgeRemote names the target of a supported host", () => { + assert.deepEqual(judgeRemote(supported), { supported: true, target: "linux-x64" }); + assert.deepEqual(judgeRemote({ ...supported, arch: "aarch64", glibc: "2.34" }), { supported: true, target: "linux-arm64" }); +}); + +test("judgeRemote refuses with the value found and the value needed", () => { + const reason = (change: Partial) => { + const judged = judgeRemote({ ...supported, ...change }); + assert.equal(judged.supported, false); + return judged.supported ? "" : judged.reason; + }; + assert.equal(reason({ os: "Darwin" }), "This host runs Darwin; Whiteboard needs Linux."); + assert.equal(reason({ arch: "armv7l" }), "This host's CPU is armv7l; Whiteboard needs x86_64 or aarch64."); + assert.equal(reason({ glibc: "2.31" }), "This host runs glibc 2.31; Whiteboard needs 2.34 or newer."); + assert.equal(reason({ glibc: "1.99" }), "This host runs glibc 1.99; Whiteboard needs 2.34 or newer."); + assert.match(reason({ glibc: null }), /no glibc .*musl.*needs glibc 2\.34 or newer\.$/); + assert.equal(reason({ homeWritable: false }), "The home directory /home/dev cannot be written; Whiteboard needs to write under it."); + assert.equal(reason({ freeBytes: 999_999_999 }), "The home directory has 0.9 GB free; Whiteboard needs 1 GB."); + assert.equal(judgeRemote({ ...supported, glibc: "3.0" }).supported, true); + assert.equal(judgeRemote({ ...supported, freeBytes: 1e9 }).supported, true); +}); + +test("the answer is read past a login banner, also one that shares the sentinel's line", () => { + assert.deepEqual(parseRemoteProbe(answer(supported, "Welcome\nno newline")), { probe: supported }); +}); + +test("a malformed answer is an error, never an exception", () => { + const malformed = (value: unknown) => { + const parsed = parseRemoteProbe(typeof value === "string" ? value : answer(value)); + assert.ok("error" in parsed, JSON.stringify(value)); + return parsed.error; + }; + assert.match(malformed("banner only\n"), /no answer/); + assert.match(malformed(`${REVIEW_REMOTE_PROBE_BEGIN}\n{"os":\n${REVIEW_REMOTE_PROBE_END}`), /not JSON/); + assert.match(malformed(`${REVIEW_REMOTE_PROBE_BEGIN}\n"x${"y".repeat(70_000)}"\n${REVIEW_REMOTE_PROBE_END}`), /too long/); + for (const change of [ + { os: "Linux; rm -rf" }, + { arch: "" }, + { glibc: "2.35; echo" }, + { home: "home/dev" }, + { root: "relative/whiteboard-remote" }, + { home: "/home/\u0007dev" }, + { home: `/${"a".repeat(5000)}` }, + { homeWritable: "yes" }, + { freeBytes: "20000000000" }, + { freeBytes: -1 }, + { freeBytes: 1.5 }, + { node: { path: "/usr/bin/node", version: "20.11.0" } }, + { node: { path: "node", version: "24.1.0" } }, + { node: "/usr/bin/node" }, + { npm: "npm" }, + { installed: "0.1.6" }, + { installed: Array(300).fill({ version: "0.1.6", integrity: INTEGRITY }) }, + { managedNode: 7 }, + { downloader: "fetch" }, + { registryReachable: 1 }, + { tools: "tar" }, + { npm: undefined }, + ]) { + assert.match(malformed({ ...supported, ...change }), /malformed/); + } + assert.match(malformed([supported]), /malformed/); +}); + +test("tools keeps only the tools asked about", () => { + const parsed = parseRemoteProbe(answer({ ...supported, tools: ["openssl", "rm -rf", 5, "tar"] })); + assert.ok("probe" in parsed); + assert.deepEqual(parsed.probe.tools, ["tar", "openssl"]); +}); + +test("pathCli keeps its version only when it is one, and is null when its line is missing or wrong", () => { + const pathCli = (value: unknown) => { + const parsed = parseRemoteProbe(answer({ ...supported, pathCli: value })); + assert.ok("probe" in parsed); + return parsed.probe.pathCli; + }; + assert.equal(pathCli({ path: "/usr/bin/whiteboard", version: "0.2.0" })?.version, "0.2.0"); + assert.equal(pathCli({ path: "/usr/bin/whiteboard", version: "Whiteboard needs Node.js 24 or newer" })?.version, null); + assert.equal(pathCli({ path: "/usr/bin/whiteboard", version: null })?.version, null); + assert.equal(pathCli({ path: "whiteboard", version: "0.2.0" }), null); + assert.equal(pathCli(null), null); +}); + +test("installed keeps only versions with an npm sha512 integrity", () => { + const entry = (version: unknown, integrity: unknown = INTEGRITY) => ({ version, integrity }); + const parsed = parseRemoteProbe( + answer({ + ...supported, + installed: [ + entry("0.1.6"), + entry("0.1.7-preview.20261003.2"), + entry("0.1.8.part"), + entry("0.1.9-preview.1.part"), + entry("x; rm"), + entry(5), + entry("latest"), + entry("0.2.0", "sha512-abc"), + entry("0.2.1", `sha1-${"A".repeat(26)}=`), + entry("0.2.2", null), + "0.2.3", + null, + ], + }), + ); + assert.ok("probe" in parsed); + assert.deepEqual(parsed.probe.installed, [entry("0.1.6"), entry("0.1.7-preview.20261003.2")]); +}); + +async function executable(path: string, body: string) { + await mkdir(join(path, ".."), { recursive: true }); + await writeFile(path, `#!/bin/sh\n${body}\n`); + await chmod(path, 0o755); +} + +async function fakeRemote(t: test.TestContext) { + const home = await mkdtemp(join(tmpdir(), "wb-probe-")); + t.after(() => rm(home, { recursive: true, force: true })); + const bin = join(home, "bin"); + await executable(join(bin, "node"), "echo v20.11.1"); + await executable(join(bin, "curl"), "exit 7"); + await executable(join(bin, "whiteboard"), 'if [ "$1" = --version ]; then echo 0.2.0; fi'); + await executable(join(home, ".nvm/versions/node/v24.9.0/bin/node"), "echo v24.9.0"); + await executable(join(home, ".nvm/versions/node/v24.10.0/bin/node"), "echo v24.10.0"); + await executable(join(home, ".nvm/versions/node/v24.10.0/bin/npm"), "echo 11.0.0"); + await executable(join(home, ".volta/tools/image/node/24.2.0/bin/node"), "echo v24.2.0"); + await executable(join(home, ".asdf/installs/nodejs/24.99.0/bin/node"), "echo v20.0.0"); + await executable(join(home, ".dev/whiteboard-remote/node/v24.18.0/bin/node"), "echo v24.18.0"); + const versions = join(home, ".dev/whiteboard-remote/versions"); + for (const version of ["0.1.6", "0.1.7.part", "0.1.8", "0.1.9", "0.2.0"]) await mkdir(join(versions, version), { recursive: true }); + const node = join(home, ".dev/whiteboard-remote/node/v24.18.0/bin/node"); + const cli = (version: string) => join(versions, version, "cli.js"); + const marker = (version: string, integrity: string) => writeFile(join(versions, version, REVIEW_REMOTE_INSTALL_MARKER), `${JSON.stringify({ version, integrity, node, cli: cli(version) })}\n`); + await marker("0.1.6", INTEGRITY); + await writeFile(cli("0.1.6"), ""); + await marker("0.1.9", ""); + await writeFile(cli("0.1.9"), ""); + await marker("0.2.0", INTEGRITY); + return { home, env: { HOME: home, PATH: `${bin}:/usr/bin:/bin` } }; +} + +const localShell = + (env: NodeJS.ProcessEnv, command = "/bin/sh", args = ["-s"]): SpawnSsh => + (_args, options) => + spawn(command, args, { ...options, env }); + +const session = reviewSshSession("devbox", tmpdir()); + +async function tree(dir: string): Promise { + const entries = await readdir(dir, { recursive: true }); + return Promise.all(entries.sort().map(async (entry) => `${entry} ${(await stat(join(dir, entry))).mtimeMs}`)); +} + +test("the script finds the highest Node 24, the complete installed versions and the managed Node, and writes nothing", async (t) => { + const { home, env } = await fakeRemote(t); + const before = await tree(home); + + const result = await probeRemote({ session, spawn: localShell(env), env: {} }); + + assert.ok("probe" in result, "error" in result ? result.error : ""); + const { probe } = result; + assert.deepEqual(probe.node, { path: join(home, ".nvm/versions/node/v24.10.0/bin/node"), version: "24.10.0" }); + assert.equal(probe.npm, join(home, ".nvm/versions/node/v24.10.0/bin/npm")); + assert.equal(probe.managedNode, join(home, ".dev/whiteboard-remote/node/v24.18.0/bin/node")); + assert.deepEqual(probe.installed, [{ version: "0.1.6", integrity: INTEGRITY }]); + assert.deepEqual(probe.pathCli, { path: join(home, "bin/whiteboard"), version: "0.2.0" }); + assert.equal(probe.home, home); + assert.equal(probe.root, join(home, ".dev/whiteboard-remote")); + assert.equal(probe.homeWritable, true); + assert.ok(probe.freeBytes > 0); + assert.equal(probe.downloader, "curl"); + assert.equal(probe.registryReachable, false); + assert.ok(probe.tools.includes("tar"), probe.tools.join()); + assert.deepEqual(await tree(home), before); +}); + +test("Desktop's wrapper and a link into the install root are not a CLI on PATH: their markers decide", async (t) => { + const { home, env } = await fakeRemote(t); + const bin = join(home, "bin"); + const local = join(home, ".local/bin"); + await rm(join(bin, "whiteboard")); + await executable(join(local, "whiteboard"), `${REVIEW_REMOTE_WRAPPER_MARK}\necho 0.1.6`); + const probe = async () => { + const result = await probeRemote({ session, spawn: localShell(env), env: {} }); + assert.ok("probe" in result, "error" in result ? result.error : ""); + return result.probe.pathCli; + }; + + assert.equal(await probe(), null); + + const managed = join(home, ".dev/whiteboard-remote/versions/0.1.6/whiteboard"); + await executable(managed, "echo 0.1.6"); + await symlink(managed, join(bin, "whiteboard")); + assert.equal(await probe(), null); +}); + +test("a home with quotes and backslashes reaches the parser intact", async (t) => { + const parent = await mkdtemp(join(tmpdir(), "wb-probe-")); + t.after(() => rm(parent, { recursive: true, force: true })); + const home = join(parent, 'a "quoted" \\ home'); + await mkdir(home); + + const result = await probeRemote({ session, spawn: localShell({ HOME: home, PATH: "/usr/bin:/bin" }), env: {} }); + + assert.ok("probe" in result, "error" in result ? result.error : ""); + assert.equal(result.probe.home, home); +}); + +test("a home that does not exist is refused as unwritable, not as a malformed answer", async (t) => { + const parent = await mkdtemp(join(tmpdir(), "wb-probe-")); + t.after(() => rm(parent, { recursive: true, force: true })); + const home = join(parent, "missing"); + + const result = await probeRemote({ session, spawn: localShell({ HOME: home, PATH: "/usr/bin:/bin" }), env: {} }); + + assert.ok("probe" in result, "error" in result ? result.error : ""); + assert.equal(result.probe.freeBytes, 0); + assert.deepEqual(judgeRemote({ ...result.probe, os: "Linux", arch: "x86_64", glibc: "2.35" }), { + supported: false, + reason: `The home directory ${home} cannot be written; Whiteboard needs to write under it.`, + }); +}); + +test("a probe that does not answer in time is ended and reported", async () => { + const started = Date.now(); + const result = await probeRemote({ session, spawn: localShell({}, "/bin/sleep", ["30"]), env: {}, timeout: 300 }); + + assert.deepEqual(result, { error: "The probe of devbox did not answer within 0.3 seconds." }); + assert.ok(Date.now() - started < 5000); +}); + +test("a probe that runs out of time after its answer has no PATH CLI, and is not an error", async () => { + const printed = answer({ ...supported, pathCli: undefined }); + const result = await probeRemote({ session, spawn: localShell({}, "/bin/sh", ["-c", `cat >/dev/null; printf '%s' '${printed}'; exec sleep 30`]), env: {}, timeout: 500 }); + + assert.deepEqual(result, { probe: { ...supported, pathCli: null } }); +}); + +const timeoutDir = spawnSync("/bin/sh", ["-c", "command -v timeout"], { encoding: "utf8" }).stdout.trim().replace(/\/timeout$/, ""); + +test("a login shell that ignores SIGTERM is killed in time, and finds no CLI", { skip: !timeoutDir && "no timeout on PATH" }, async (t) => { + const home = await mkdtemp(join(tmpdir(), "wb-probe-")); + t.after(() => rm(home, { recursive: true, force: true })); + const shell = join(home, "slow-shell"); + await executable(shell, "trap '' TERM\nsleep 2; sleep 2; sleep 2; echo /opt/bin/whiteboard"); + await executable(join(home, "bin/curl"), "exit 7"); + const started = Date.now(); + + const result = await probeRemote({ session, spawn: localShell({ HOME: home, SHELL: shell, PATH: `${join(home, "bin")}:${timeoutDir}:/usr/bin:/bin` }), env: {} }); + + assert.ok("probe" in result, "error" in result ? result.error : ""); + assert.equal(result.probe.pathCli, null); + assert.ok(Date.now() - started < 6000, `${Date.now() - started} ms`); +}); + +test("an ssh failure is reported with what OpenSSH said", async () => { + const result = await probeRemote({ + session, + spawn: localShell({}, "/bin/sh", ["-c", "cat >/dev/null; echo 'ssh: connect to host devbox port 22: Connection refused' >&2; exit 255"]), + env: {}, + }); + + assert.deepEqual(result, { + error: "The probe of devbox failed: it printed no answer between its sentinels. It exited with 255: ssh: connect to host devbox port 22: Connection refused.", + }); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteProbe.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteProbe.ts new file mode 100644 index 000000000..3db320c48 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteProbe.ts @@ -0,0 +1,187 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { runSsh, type SpawnSsh } from "./reviewRemoteHost.js"; +import { REVIEW_REMOTE_VERSION } from "./reviewRemoteInstallScript.js"; +import { REVIEW_REMOTE_PROBE_BEGIN, REVIEW_REMOTE_PROBE_END, REVIEW_REMOTE_PROBE_PATH_CLI, REVIEW_REMOTE_PROBE_SCRIPT } from "./reviewRemoteProbeScript.js"; +import { sshExecArgs, type ReviewSshSession } from "./reviewSshCommand.js"; + +export interface ReviewRemoteProbe { + os: string; + arch: string; + glibc: string | null; + home: string; + root: string; + homeWritable: boolean; + freeBytes: number; + node: { path: string; version: string } | null; + npm: string | null; + installed: ReviewRemoteInstalled[]; + managedNode: string | null; + pathCli: { path: string; version: string | null } | null; + downloader: "curl" | "wget" | null; + registryReachable: boolean; + tools: ReviewRemoteTool[]; +} + +export const REVIEW_REMOTE_PROBE_TOOLS = ["tar", "xz", "sha256sum", "sha512sum", "openssl"] as const; +export type ReviewRemoteTool = (typeof REVIEW_REMOTE_PROBE_TOOLS)[number]; + +export type ReviewRemoteTarget = "linux-x64" | "linux-arm64"; + +export interface ReviewRemoteInstalled { + version: string; + integrity: string; +} + +export type ReviewRemoteSupport = { supported: true; target: ReviewRemoteTarget } | { supported: false; reason: string }; + +export const REVIEW_REMOTE_PROBE_TIMEOUT = 10_000; + +const TARGETS: Record = { x86_64: "linux-x64", aarch64: "linux-arm64" }; +const GLIBC_FLOOR = [2, 34] as const; +const GB = 1_000_000_000; + +export function judgeRemote(probe: ReviewRemoteProbe): ReviewRemoteSupport { + if (probe.os !== "Linux") return refuse(`This host runs ${probe.os}; Whiteboard needs Linux.`); + const target = TARGETS[probe.arch]; + if (!target) return refuse(`This host's CPU is ${probe.arch}; Whiteboard needs x86_64 or aarch64.`); + if (probe.glibc === null) { + return refuse(`This host has no glibc (Alpine and other musl systems do not); Whiteboard needs glibc ${GLIBC_FLOOR.join(".")} or newer.`); + } + const [major = 0, minor = 0] = probe.glibc.split(".").map(Number); + if (major < GLIBC_FLOOR[0] || (major === GLIBC_FLOOR[0] && minor < GLIBC_FLOOR[1])) { + return refuse(`This host runs glibc ${probe.glibc}; Whiteboard needs ${GLIBC_FLOOR.join(".")} or newer.`); + } + if (!probe.homeWritable) return refuse(`The home directory ${probe.home} cannot be written; Whiteboard needs to write under it.`); + if (probe.freeBytes < GB) { + return refuse(`The home directory has ${(Math.floor(probe.freeBytes / (GB / 10)) / 10).toFixed(1)} GB free; Whiteboard needs 1 GB.`); + } + return { supported: true, target }; +} + +const refuse = (reason: string): ReviewRemoteSupport => ({ supported: false, reason }); + +export type ReviewRemoteProbeResult = { probe: ReviewRemoteProbe } | { error: string }; + +export async function probeRemote(input: { + session: ReviewSshSession; + spawn: SpawnSsh; + env: NodeJS.ProcessEnv; + timeout?: number; +}): Promise { + const timeout = input.timeout ?? REVIEW_REMOTE_PROBE_TIMEOUT; + const alias = input.session.alias; + const result = await runSsh(input.spawn, input.env, sshExecArgs(input.session, input.env), timeout, REVIEW_REMOTE_PROBE_SCRIPT); + if (result.error) return { error: `The probe of ${alias} could not start: ${result.error.message}` }; + const parsed = parseRemoteProbe(result.stdout); + if ("probe" in parsed) return parsed; + if (result.timedOut) return { error: `The probe of ${alias} did not answer within ${timeout / 1000} seconds.` }; + const stderr = result.stderr.trim().split("\n").at(-1)?.slice(0, 300); + return { error: `The probe of ${alias} failed: ${parsed.error}${result.code ? ` It exited with ${result.code}${stderr ? `: ${stderr}` : ""}.` : ""}` }; +} + +const LINE_LIMIT = 64 * 1024; +const STRING_LIMIT = 4096; +const INSTALLED_LIMIT = 256; +const WORD = /^[\w.-]{1,64}$/; +const INTEGRITY = /^sha512-[A-Za-z0-9+/]{86}==$/; +const CONTROL = /[\x00-\x1f\x7f-\x9f]/; + +export function parseRemoteProbe(stdout: string): ReviewRemoteProbeResult { + const begin = stdout.indexOf(REVIEW_REMOTE_PROBE_BEGIN); + const end = begin < 0 ? -1 : stdout.indexOf(REVIEW_REMOTE_PROBE_END, begin + REVIEW_REMOTE_PROBE_BEGIN.length); + if (end < 0) return { error: "it printed no answer between its sentinels." }; + const line = stdout.slice(begin + REVIEW_REMOTE_PROBE_BEGIN.length, end).trim(); + if (line.length > LINE_LIMIT) return { error: "its answer is too long." }; + let value: unknown; + try { + value = JSON.parse(line); + } catch { + return { error: "its answer is not JSON." }; + } + try { + return { probe: { ...readProbe(value), pathCli: readPathCli(stdout.slice(end)) } }; + } catch (error) { + return { error: `its answer is malformed: ${(error as Error).message}` }; + } +} + +function readPathCli(after: string): ReviewRemoteProbe["pathCli"] { + const line = after.split("\n").find((candidate) => candidate.startsWith(`${REVIEW_REMOTE_PROBE_PATH_CLI} `)); + try { + const cli = object(JSON.parse(line?.slice(REVIEW_REMOTE_PROBE_PATH_CLI.length + 1) ?? "null"), "pathCli"); + const { version } = cli; + return { path: path(cli.path, "pathCli.path"), version: typeof version === "string" && version.length <= 128 && REVIEW_REMOTE_VERSION.test(version) ? version : null }; + } catch { + return null; + } +} + +function readProbe(value: unknown): Omit { + const record = object(value, "the answer"); + const node = record.node === null ? null : object(record.node, "node"); + const installed = record.installed; + if (!Array.isArray(installed) || installed.length > INSTALLED_LIMIT) throw new Error("installed is not a short list."); + const glibc = nullable(record.glibc, "glibc", (v) => string(v, "glibc", /^\d{1,4}\.\d{1,4}(\.\d{1,6})?$/)); + const downloader = nullable(record.downloader, "downloader", (v) => { + if (v !== "curl" && v !== "wget") throw new Error("downloader is neither curl nor wget."); + return v; + }); + const tools = record.tools; + if (!Array.isArray(tools) || tools.length > INSTALLED_LIMIT) throw new Error("tools is not a short list."); + if (record.root === "") throw new Error("DEV_REVIEW_HOME there is not an absolute, normalised path."); + const freeBytes = record.freeBytes; + if (typeof freeBytes !== "number" || !Number.isSafeInteger(freeBytes) || freeBytes < 0) throw new Error("freeBytes is not a byte count."); + return { + os: string(record.os, "os", WORD), + arch: string(record.arch, "arch", WORD), + glibc, + home: path(record.home, "home"), + root: path(record.root, "root"), + homeWritable: boolean(record.homeWritable, "homeWritable"), + freeBytes, + node: node && { path: path(node.path, "node.path"), version: string(node.version, "node.version", /^24\.\d{1,4}\.\d{1,4}$/) }, + npm: nullable(record.npm, "npm", (v) => path(v, "npm")), + installed: installed.flatMap((entry): ReviewRemoteInstalled[] => { + if (!entry || typeof entry !== "object") return []; + const { version, integrity } = entry as Record; + return typeof version === "string" && version.length <= 128 && REVIEW_REMOTE_VERSION.test(version) && !version.endsWith(".part") && typeof integrity === "string" && INTEGRITY.test(integrity) + ? [{ version, integrity }] + : []; + }), + managedNode: nullable(record.managedNode, "managedNode", (v) => path(v, "managedNode")), + downloader, + registryReachable: boolean(record.registryReachable, "registryReachable"), + tools: REVIEW_REMOTE_PROBE_TOOLS.filter((tool) => tools.includes(tool)), + }; +} + +function object(value: unknown, name: string): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`${name} is not an object.`); + return value as Record; +} + +function string(value: unknown, name: string, pattern: RegExp): string { + if (typeof value !== "string" || !pattern.test(value)) throw new Error(`${name} is not a valid value.`); + return value; +} + +function path(value: unknown, name: string): string { + if (typeof value !== "string" || !value.startsWith("/") || value.length > STRING_LIMIT || CONTROL.test(value)) { + throw new Error(`${name} is not an absolute path.`); + } + return value; +} + +function boolean(value: unknown, name: string): boolean { + if (typeof value !== "boolean") throw new Error(`${name} is not true or false.`); + return value; +} + +function nullable(value: unknown, name: string, read: (value: unknown) => T): T | null { + if (value === undefined) throw new Error(`${name} is missing.`); + return value === null ? null : read(value); +} diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteProbeScript.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteProbeScript.ts new file mode 100644 index 000000000..de5c1e47a --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteProbeScript.ts @@ -0,0 +1,124 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { REVIEW_REMOTE_FIND_CLI } from "./reviewRemoteAttachScript.js"; +import { REVIEW_REMOTE_COMPLETE_INTEGRITY, REVIEW_REMOTE_INSTALL_MARKER, REVIEW_REMOTE_ROOT_SCRIPT, REVIEW_REMOTE_WRAPPER_MARK, shellQuote } from "./reviewRemoteInstallScript.js"; + +export const REVIEW_REMOTE_PROBE_BEGIN = "WHITEBOARD-PROBE-BEGIN"; +export const REVIEW_REMOTE_PROBE_END = "WHITEBOARD-PROBE-END"; +export const REVIEW_REMOTE_PROBE_PATH_CLI = "WHITEBOARD-PROBE-PATH-CLI"; + +export const REVIEW_REMOTE_PROBE_SCRIPT = `LC_ALL=C +export LC_ALL +str() { printf '"%s"' "$(printf '%s' "$1" | tr -d '\\000-\\037' | sed 's/\\\\/\\\\\\\\/g; s/"/\\\\"/g')"; } +strOrNull() { if [ -n "$1" ]; then str "$1"; else printf null; fi; } + +os=$(uname -s 2>/dev/null) +arch=$(uname -m 2>/dev/null) +glibc=$(getconf GNU_LIBC_VERSION 2>/dev/null | sed -n 's/^glibc \\([0-9][0-9.]*\\)$/\\1/p') +if [ -z "$glibc" ]; then + glibc=$(ldd --version 2>&1 | head -n 1 | grep -i -e glibc -e 'gnu libc' | sed -n 's/.* \\([0-9][0-9]*\\.[0-9][0-9]*\\)$/\\1/p') +fi +home=$HOME +writable=false +[ -d "$home" ] && [ -w "$home" ] && writable=true +free=$(df -Pk "$home" 2>/dev/null | awk 'NR == 2 { printf "%.0f", $4 * 1024 }') + +# The highest 24.x among the candidates in $@: sets best and bestVersion. +best= +bestVersion= +pick() { + for candidate in "$@"; do + [ -f "$candidate" ] && [ -x "$candidate" ] || continue + version=$("$candidate" --version 2>/dev/null /dev/null)" /usr/local/bin/node /usr/bin/node \\ + "\${NVM_DIR:-$home/.nvm}"/versions/node/v24*/bin/node \\ + "\${FNM_DIR:-$home/.local/share/fnm}"/node-versions/v24*/installation/bin/node \\ + "$home"/.fnm/node-versions/v24*/installation/bin/node \\ + "\${VOLTA_HOME:-$home/.volta}"/tools/image/node/24*/bin/node \\ + "\${ASDF_DATA_DIR:-$home/.asdf}"/installs/nodejs/24*/bin/node \\ + "$home"/.local/share/mise/installs/node/24*/bin/node \\ + "$home"/.nodenv/versions/24*/bin/node \\ + "\${N_PREFIX:-/usr/local}"/n/versions/node/24*/bin/node +node=$best +nodeVersion=$bestVersion +npm= +[ -n "$node" ] && [ -x "\${node%/*}/npm" ] && npm=\${node%/*}/npm + +${REVIEW_REMOTE_ROOT_SCRIPT}remote=$root +best= +bestVersion= +pick "$remote"/node/v24*/bin/node +managed=$best + +${REVIEW_REMOTE_COMPLETE_INTEGRITY} +installed= +for dir in "$remote"/versions/*; do + [ -d "$dir" ] || continue + case "$dir" in *.part) continue ;; esac + integrity=$(completeIntegrity "$dir/${REVIEW_REMOTE_INSTALL_MARKER}") + [ -n "$integrity" ] || continue + installed="$installed\${installed:+,}{\\"version\\":$(str "\${dir##*/}"),\\"integrity\\":$(str "$integrity")}" +done + +tools= +for tool in tar xz sha256sum sha512sum openssl; do + command -v "$tool" >/dev/null 2>&1 && tools="$tools\${tools:+,}\\"$tool\\"" +done + +registry=https://registry.npmjs.org/ +downloader= +reachable=false +if command -v curl >/dev/null 2>&1; then + downloader=curl + curl -fsI --max-time 3 -o /dev/null "$registry" >/dev/null 2>&1 && reachable=true +elif command -v wget >/dev/null 2>&1; then + downloader=wget + hsts= + tries= + wget --help 2>&1 | grep -q -- --no-hsts && hsts=--no-hsts + wget --help 2>&1 | grep -q -- --tries && tries="-t 1" + limit= + command -v timeout >/dev/null 2>&1 && limit="timeout 3" + $limit wget -q $hsts $tries -T 3 --spider "$registry" >/dev/null 2>&1 && reachable=true +fi + +echo +echo ${REVIEW_REMOTE_PROBE_BEGIN} +printf '{"os":%s,"arch":%s,"glibc":%s,"home":%s,"root":%s,"homeWritable":%s,"freeBytes":%s,' \\ + "$(str "$os")" "$(str "$arch")" "$(strOrNull "$glibc")" "$(str "$home")" "$(str "$remote")" \\ + "$writable" "\${free:-0}" +if [ -n "$node" ]; then + printf '"node":{"path":%s,"version":%s},' "$(str "$node")" "$(str "$nodeVersion")" +else + printf '"node":null,' +fi +printf '"npm":%s,"installed":[%s],"managedNode":%s,"downloader":%s,"registryReachable":%s,"tools":[%s]}\\n' \\ + "$(strOrNull "$npm")" "$installed" "$(strOrNull "$managed")" "$(strOrNull "$downloader")" "$reachable" "$tools" +echo ${REVIEW_REMOTE_PROBE_END} + +${REVIEW_REMOTE_FIND_CLI}# Desktop's own installs are judged by their markers, not by a version on PATH. +grep -qxF ${shellQuote(REVIEW_REMOTE_WRAPPER_MARK)} "$wb" 2>/dev/null && wb= +case "$(readlink -f "$wb" 2>/dev/null)" in "$(readlink -f "$remote" 2>/dev/null || echo "$remote")"/*) wb= ;; esac +if [ -n "$wb" ]; then + wbVersion=$(PATH="\${wb%/*}:$PATH" DEV_FAST_REVIEW_CLI_NO_DELEGATE=1 DEV_FAST_REVIEW_TELEMETRY_DISABLED=1 bounded 2 "$wb" --version /dev/null | head -n 1) + printf '%s {"path":%s,"version":%s}\\n' ${REVIEW_REMOTE_PROBE_PATH_CLI} "$(str "$wb")" "$(strOrNull "$wbVersion")" +fi +`; diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteRegistryRelay.test.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteRegistryRelay.test.ts new file mode 100644 index 000000000..09e5f5f91 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteRegistryRelay.test.ts @@ -0,0 +1,108 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from "node:assert/strict"; +import { createServer, request, type IncomingMessage, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import test from "node:test"; + +import { startRegistryRelay, type ReviewRegistryRelay } from "./reviewRemoteRegistryRelay.js"; + +async function fakeRegistry(t: test.TestContext) { + const seen: string[] = []; + const tarball = Buffer.alloc(300_000, 7); + let origin = ""; + const server: Server = createServer((req, res) => { + seen.push(`${req.method} ${req.url}`); + if (req.url === "/left-pad" || req.url === "/@dev.fast%2fdiffr") { + res.setHeader("content-type", "application/json"); + res.end( + JSON.stringify({ + name: "left-pad", + versions: { + "1.0.0": { dist: { tarball: `${origin}/left-pad/-/left-pad-1.0.0.tgz`, integrity: "sha512-x" } }, + "0.9.0": { dist: { tarball: "https://elsewhere.example/left-pad-0.9.0.tgz" } }, + }, + }), + ); + } else if (req.url === "/left-pad/-/left-pad-1.0.0.tgz") { + res.setHeader("content-type", "application/octet-stream"); + res.end(tarball); + } else { + res.statusCode = 404; + res.end("{}"); + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + t.after(() => new Promise((resolve) => server.close(resolve))); + return { origin, seen, tarball }; +} + +async function relayFor(t: test.TestContext, upstream: string, limits?: { tarball?: number }) { + const relay = await startRegistryRelay({ upstream, limits }); + t.after(() => relay.close()); + return relay; +} + +function send(relay: ReviewRegistryRelay, method: string, path: string, host = `127.0.0.1:${relay.port}`) { + return new Promise<{ status: number; body: Buffer }>((resolve, reject) => { + const req = request({ host: "127.0.0.1", port: relay.port, method, path, headers: { host } }, (res: IncomingMessage) => { + const chunks: Buffer[] = []; + res.on("data", (chunk: Buffer) => chunks.push(chunk)); + res.on("end", () => resolve({ status: res.statusCode ?? 0, body: Buffer.concat(chunks) })); + res.on("error", reject); + }); + req.on("error", reject); + req.end(); + }); +} + +test("a packument's tarballs on the registry point at the relay, as the remote reaches it", async (t) => { + const registry = await fakeRegistry(t); + const relay = await relayFor(t, registry.origin); + + const answer = await send(relay, "GET", "/left-pad", "127.0.0.1:41234"); + + assert.equal(answer.status, 200); + const versions = JSON.parse(answer.body.toString()).versions; + assert.equal(versions["1.0.0"].dist.tarball, "http://127.0.0.1:41234/left-pad/-/left-pad-1.0.0.tgz"); + assert.equal(versions["1.0.0"].dist.integrity, "sha512-x"); + assert.equal(versions["0.9.0"].dist.tarball, "https://elsewhere.example/left-pad-0.9.0.tgz"); + assert.ok(!answer.body.toString().includes(registry.origin)); +}); + +test("scoped names and tarballs pass through; a missing package stays a 404", async (t) => { + const registry = await fakeRegistry(t); + const relay = await relayFor(t, registry.origin); + + assert.equal((await send(relay, "GET", "/@dev.fast%2fdiffr")).status, 200); + const tarball = await send(relay, "GET", "/left-pad/-/left-pad-1.0.0.tgz"); + assert.equal(tarball.status, 200); + assert.ok(tarball.body.equals(registry.tarball)); + assert.equal((await send(relay, "GET", "/no-such-package")).status, 404); + assert.deepEqual(registry.seen, ["GET /@dev.fast%2fdiffr", "GET /left-pad/-/left-pad-1.0.0.tgz", "GET /no-such-package"]); +}); + +test("only GET of a package or a tarball, from a loopback Host, reaches the registry", async (t) => { + const registry = await fakeRegistry(t); + const relay = await relayFor(t, registry.origin); + + assert.equal((await send(relay, "POST", "/left-pad")).status, 405); + assert.equal((await send(relay, "HEAD", "/left-pad")).status, 405); + assert.equal((await send(relay, "PUT", "/left-pad")).status, 405); + for (const path of ["/", "/..", "/@x/..", "/../etc/passwd", "//elsewhere.example/x", "/left-pad?write=1", "/-/npm/v1/security/advisories/bulk", "/left-pad/1.0.0", "/%2e%2e/x", "/a/b/c"]) { + assert.equal((await send(relay, "GET", path)).status, 404, path); + } + assert.equal((await send(relay, "GET", "/left-pad", "elsewhere.example")).status, 400); + assert.deepEqual(registry.seen, []); +}); + +test("a tarball over the size bound is cut off", async (t) => { + const registry = await fakeRegistry(t); + const relay = await relayFor(t, registry.origin, { tarball: 100_000 }); + + assert.equal((await send(relay, "GET", "/left-pad/-/left-pad-1.0.0.tgz")).status, 502); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteRegistryRelay.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteRegistryRelay.ts new file mode 100644 index 000000000..7a9a1faa2 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteRegistryRelay.ts @@ -0,0 +1,122 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; +import { Readable } from "node:stream"; +import { pipeline } from "node:stream/promises"; + +export interface ReviewRegistryRelay { + readonly port: number; + close(): Promise; +} + +export const REVIEW_REGISTRY = "https://registry.npmjs.org"; + +const LIMITS = { + packument: 64 << 20, + tarball: 256 << 20, +}; + +const UPSTREAM_TIMEOUT = 120_000; + +const NAME = "[A-Za-z0-9~-][A-Za-z0-9._~-]*"; +const PACKUMENT = new RegExp(`^/(?:@${NAME}(?:%2[fF]|/))?${NAME}$`); +const TARBALL = new RegExp(`^/(?:@${NAME}/)?${NAME}/-/${NAME}\\.tgz$`); +const LOOPBACK_HOST = /^(?:127\.0\.0\.1|localhost|\[::1\]):\d{1,5}$/; + +export async function startRegistryRelay( + options: { upstream?: string; limits?: Partial } = {}, +): Promise { + const upstream = options.upstream ?? REVIEW_REGISTRY; + const limits = { ...LIMITS, ...options.limits }; + const pending = new Set(); + + const server = createServer((req, res) => { + const abort = new AbortController(); + pending.add(abort); + res.once("close", () => { + abort.abort(); + pending.delete(abort); + }); + relay(req, res, abort.signal).catch(() => (res.headersSent ? res.destroy() : reply(res, 502))); + }); + + async function relay(req: IncomingMessage, res: ServerResponse, signal: AbortSignal): Promise { + if (req.method !== "GET") return reply(res, 405); + const path = req.url ?? ""; + const tarball = TARBALL.test(path); + if (!tarball && !PACKUMENT.test(path)) return reply(res, 404); + const host = req.headers.host ?? ""; + if (!LOOPBACK_HOST.test(host)) return reply(res, 400); + + const response = await fetch(`${upstream}${path}`, { + headers: { accept: req.headers.accept ?? "application/json" }, + redirect: "error", + signal: AbortSignal.any([signal, AbortSignal.timeout(UPSTREAM_TIMEOUT)]), + }); + const limit = tarball ? limits.tarball : limits.packument; + if (Number(response.headers.get("content-length") ?? 0) > limit || !response.body) { + await response.body?.cancel(); + return reply(res, 502); + } + const type = response.headers.get("content-type"); + if (type) res.setHeader("content-type", type); + res.statusCode = response.status; + + if (tarball || !response.ok) { + let size = 0; + const body = Readable.fromWeb(response.body as import("node:stream/web").ReadableStream); + body.on("data", (chunk: Buffer) => { + size += chunk.length; + if (size > limit) body.destroy(new Error("over the size bound")); + }); + return pipeline(body, res); + } + + const text = await boundedText(response.body, limit); + res.end(rewrite(text, `${upstream}/`, `http://${host}/`)); + } + + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => resolve()); + }); + + return { + port: (server.address() as AddressInfo).port, + close: () => + new Promise((resolve) => { + for (const abort of pending) abort.abort(); + server.close(() => resolve()); + server.closeAllConnections(); + }), + }; +} + +function reply(res: ServerResponse, status: number): void { + res.statusCode = status; + res.end(); +} + +async function boundedText(body: ReadableStream, limit: number): Promise { + const chunks: Buffer[] = []; + let size = 0; + for await (const chunk of body as unknown as AsyncIterable) { + size += chunk.length; + if (size > limit) throw new Error("over the size bound"); + chunks.push(Buffer.from(chunk)); + } + return Buffer.concat(chunks).toString("utf8"); +} + +function rewrite(text: string, from: string, to: string): string { + const packument = JSON.parse(text) as { versions?: Record }; + for (const version of Object.values(packument.versions ?? {})) { + const dist = version?.dist; + if (dist && typeof dist.tarball === "string" && dist.tarball.startsWith(from)) dist.tarball = to + dist.tarball.slice(from.length); + } + return JSON.stringify(packument); +} diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteUninstall.test.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteUninstall.test.ts new file mode 100644 index 000000000..422f8ba24 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteUninstall.test.ts @@ -0,0 +1,68 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import type { SpawnSsh } from "./reviewRemoteHost.js"; +import { REVIEW_REMOTE_INSTALL_MARKER } from "./reviewRemoteInstallScript.js"; +import { uninstallRemote } from "./reviewRemoteUninstall.js"; +import { reviewSshSession } from "./reviewSshCommand.js"; + +const localRemote = + (home: string): SpawnSsh => + (args, options) => + spawn("/bin/sh", ["-c", args.slice(args.indexOf("--") + 2).join(" ")], { ...options, env: { HOME: home, PATH: "/usr/bin:/bin" } }); + +async function fixture(t: test.TestContext) { + const home = await mkdtemp(join(tmpdir(), "wb uninstall-")); + t.after(() => rm(home, { recursive: true, force: true })); + const ran = join(home, "ran"); + const version = async (name: string, cli: string) => { + const dir = join(home, ".dev/whiteboard-remote/versions", name); + await mkdir(dir, { recursive: true }); + await writeFile(join(dir, "cli.js"), cli); + await writeFile(join(dir, REVIEW_REMOTE_INSTALL_MARKER), JSON.stringify({ version: name, node: process.execPath, cli: join(dir, "cli.js") })); + }; + const uninstall = () => uninstallRemote({ session: reviewSshSession("devbox", tmpdir()), spawn: localRemote(home), env: {} }); + return { home, ran, version, uninstall }; +} + +test("runs the newest version's own remote uninstall, keeping reviews", async (t) => { + const f = await fixture(t); + await f.version("0.9.0", `require("fs").writeFileSync(${JSON.stringify(f.ran)}, "old")`); + await f.version( + "0.10.0", + `require("fs").writeFileSync(${JSON.stringify(f.ran)}, process.argv.slice(2).join(" ")); +console.log("banner"); +console.log(JSON.stringify({ event: "remote.uninstall", ok: true, removed: [], keptReviews: true }));`, + ); + + await f.uninstall(); + + assert.equal(await readFile(f.ran, "utf8"), "remote uninstall --keep-reviews --json"); +}); + +test("rejects with the reason the remote gave, on one line and bounded", async (t) => { + const f = await fixture(t); + await f.version("0.10.0", `console.log(JSON.stringify({ event: "remote.uninstall", ok: false, reason: "A Whiteboard server you started (process 7)\\n\\u001b[31mruns from it." }))`); + + await assert.rejects(f.uninstall(), { message: "Could not remove Whiteboard from devbox: A Whiteboard server you started (process 7) runs from it." }); + + await f.version("0.10.0", `console.log(JSON.stringify({ event: "remote.uninstall", ok: false, reason: "x".repeat(5000) }))`); + await assert.rejects(f.uninstall(), { message: `Could not remove Whiteboard from devbox: ${"x".repeat(300)}` }); +}); + +test("rejects when nothing is installed, or the command fails", async (t) => { + const f = await fixture(t); + await assert.rejects(f.uninstall(), { message: "Could not remove Whiteboard from devbox: Whiteboard Desktop installed nothing there." }); + + await f.version("0.10.0", `console.error("error: unknown command 'uninstall'"); process.exit(1)`); + await assert.rejects(f.uninstall(), { message: "Could not remove Whiteboard from devbox: exit 1: error: unknown command 'uninstall'." }); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteUninstall.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteUninstall.ts new file mode 100644 index 000000000..41bc92451 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteUninstall.ts @@ -0,0 +1,75 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { plainText } from "./reviewRemoteAgents.js"; +import { runSsh, type SpawnSsh } from "./reviewRemoteHost.js"; +import { compareVersions } from "./reviewRemoteInstaller.js"; +import { REVIEW_REMOTE_INSTALL_MARKER, REVIEW_REMOTE_INSTALL_SAY, REVIEW_REMOTE_ROOT_SCRIPT, REVIEW_REMOTE_VERSION, shellQuote } from "./reviewRemoteInstallScript.js"; +import { sshExecArgs, type ReviewSshSession } from "./reviewSshCommand.js"; + +export const REVIEW_REMOTE_UNINSTALL_TIMEOUT = 60_000; + +const LIST = `${REVIEW_REMOTE_ROOT_SCRIPT}printf '%s ROOT %s\\n' ${REVIEW_REMOTE_INSTALL_SAY} "$root" +for d in "$root"/versions/*; do + [ -f "$d/${REVIEW_REMOTE_INSTALL_MARKER}" ] && printf '%s HAVE %s\\n' ${REVIEW_REMOTE_INSTALL_SAY} "\${d##*/}" +done +printf '%s LISTED\\n' ${REVIEW_REMOTE_INSTALL_SAY} +`; + +function uninstallScript(root: string, version: string): string { + return `m=${shellQuote(`${root}/versions/${version}/${REVIEW_REMOTE_INSTALL_MARKER}`)} +node=$(sed -n 's/.*"node":"\\([^"]*\\)".*/\\1/p' "$m") +cli=$(sed -n 's/.*"cli":"\\([^"]*\\)".*/\\1/p' "$m") +DEV_FAST_REVIEW_CLI_NO_DELEGATE=1 "$node" "$cli" remote uninstall --keep-reviews --json { + const { session, spawn, env } = input; + const timeout = input.timeout ?? REVIEW_REMOTE_UNINSTALL_TIMEOUT; + const failed = (detail: string) => new Error(`Could not remove Whiteboard from ${session.alias}: ${detail}`); + + const listed = await runSsh(spawn, env, sshExecArgs(session, env), timeout, LIST); + const said = (word: string) => + listed.stdout + .split("\n") + .filter((line) => line.startsWith(`${REVIEW_REMOTE_INSTALL_SAY} ${word}`)) + .map((line) => line.slice(REVIEW_REMOTE_INSTALL_SAY.length + word.length + 2).trim()); + if (!said("LISTED").length) throw failed(sshProblem(listed)); + const root = said("ROOT")[0]; + if (!root?.startsWith("/")) throw failed("DEV_REVIEW_HOME there is not an absolute, normalised path."); + const newest = said("HAVE") + .filter((name) => REVIEW_REMOTE_VERSION.test(name)) + .sort(compareVersions) + .at(-1); + if (!newest) throw failed("Whiteboard Desktop installed nothing there."); + + const ran = await runSsh(spawn, env, sshExecArgs(session, env), timeout, uninstallScript(root, newest)); + const result = ran.stdout + .split("\n") + .map((line) => { + try { + return JSON.parse(line) as { event?: unknown; ok?: unknown; reason?: unknown }; + } catch { + return undefined; + } + }) + .filter((value) => value?.event === "remote.uninstall") + .at(-1); + if (result?.ok === true) return; + throw failed(typeof result?.reason === "string" ? plainText(result.reason).slice(0, 300) : sshProblem(ran)); +} + +function sshProblem(result: Awaited>): string { + if (result.timedOut) return "it did not answer in time."; + if (result.error) return result.error.message; + const stderr = result.stderr.trim().split("\n").at(-1)?.slice(0, 300); + return `exit ${result.code}${stderr ? `: ${stderr}` : ""}.`; +} diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteUpload.containers.test.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteUpload.containers.test.ts new file mode 100644 index 000000000..53d37e87d --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteUpload.containers.test.ts @@ -0,0 +1,101 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from "node:assert/strict"; +import { execFile, execFileSync, spawn } from "node:child_process"; +import { createHash, randomBytes } from "node:crypto"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { after, before, test } from "node:test"; +import { promisify } from "node:util"; + +import type { SpawnSsh, SshChildProcess } from "./reviewRemoteHost.js"; +import { REVIEW_SSH_CONFIG_ENV, reviewSshSession } from "./reviewSshCommand.js"; +import { uploadFile } from "./reviewRemoteUpload.js"; + +const run = promisify(execFile); +const harness = resolve(import.meta.dirname, "../../../../../../scripts/e2e/remote/remote.mjs"); + +function skipReason(): string | undefined { + if (process.env.WB_TEST_CONTAINERS !== "1") return "set WB_TEST_CONTAINERS=1 to upload to a container"; + try { + execFileSync("docker", ["info"], { stdio: "ignore" }); + } catch { + return "Docker is not available"; + } + return undefined; +} + +const skip = skipReason(); +const runId = `s3t2${randomBytes(3).toString("hex")}`; +const container = `wb-test-${runId}-a`; +const directory = "/home/dev/wb-upload"; +const env = { ...process.env, WB_TEST_RUN: runId }; +const sshEnv = { ...process.env, VSCODE_DEV: "1", [REVIEW_SSH_CONFIG_ENV]: `/tmp/wbt.${runId}/ssh_config` }; +let root = ""; +let local = ""; +let expected = ""; +let started = false; + +const remote = (...args: string[]) => run(process.execPath, [harness, ...args], { env, maxBuffer: 16 << 20 }); +const inContainer = async (command: string) => (await run("docker", ["exec", "-u", "dev", container, "sh", "-c", command])).stdout.trim(); + +before( + async () => { + if (skip) return; + root = await mkdtemp(join(tmpdir(), "wb-upload-")); + local = join(root, "payload.bin"); + const payload = randomBytes(40 << 20); + expected = createHash("sha256").update(payload).digest("hex"); + await writeFile(local, payload); + started = true; + await remote("up", "a", "--sealed"); + await inContainer(`mkdir -p ${directory}`); + }, + { timeout: 20 * 60_000 }, +); + +after( + async () => { + if (started) await remote("down", "--all").catch((error) => console.error(error.stderr ?? error)); + if (root) await rm(root, { recursive: true, force: true }); + }, + { timeout: 5 * 60_000 }, +); + +const session = () => reviewSshSession("wb-test-a", root); + +test("a 40 MB file lands on a sealed host with the checksum it left with", { skip, timeout: 120_000 }, async () => { + const realSsh: SpawnSsh = (args, options) => spawn("ssh", args, options); + const target = `${directory}/whole.bin`; + + await uploadFile(session(), local, target, { spawn: realSsh, env: sshEnv }); + + assert.equal((await inContainer(`sha256sum ${target}`)).split(" ")[0], expected); + assert.equal(await inContainer(`ls ${directory}`), "whole.bin"); +}); + +test("ending ssh halfway leaves only the .part file", { skip, timeout: 120_000 }, async () => { + let child: SshChildProcess | undefined; + const realSsh: SpawnSsh = (args, options) => (child = spawn("ssh", args, options)); + const target = `${directory}/half.bin`; + + await assert.rejects( + uploadFile(session(), local, target, { + spawn: realSsh, + env: sshEnv, + onProgress: (sent, total) => { + if (sent > total / 2) child?.kill("SIGKILL"); + }, + }), + /did not finish/, + ); + + for (let i = 0; i < 50 && (await inContainer("pgrep -x cat || true")); i++) await new Promise((r) => setTimeout(r, 100)); + assert.deepEqual((await inContainer(`ls ${directory}`)).split("\n").sort(), ["half.bin.part", "whole.bin"]); + const partial = Number(await inContainer(`wc -c < ${target}.part`)); + assert.ok(partial > 0 && partial < 40 << 20, `${partial} bytes`); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteUpload.test.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteUpload.test.ts new file mode 100644 index 000000000..1d92ffb2c --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteUpload.test.ts @@ -0,0 +1,137 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { createHash, randomBytes } from "node:crypto"; +import { mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { after, test } from "node:test"; + +import type { SpawnSsh, SshChildProcess } from "./reviewRemoteHost.js"; +import { reviewSshSession } from "./reviewSshCommand.js"; +import { uploadFile } from "./reviewRemoteUpload.js"; + +const roots: string[] = []; + +after(async () => { + await Promise.all(roots.map((root) => rm(root, { recursive: true, force: true }))); +}); + +async function temporary(): Promise { + const root = await mkdtemp(join(tmpdir(), "wb upload ")); + roots.push(root); + return root; +} + +const localSsh = + ({ rewrite = (command: string) => command, seen = () => {} }: { rewrite?: (command: string) => string; seen?: (command: string, child: SshChildProcess) => void } = {}): SpawnSsh => + (args, options) => { + const command = args.slice(args.indexOf("--") + 2).join(" "); + const child = spawn("/bin/sh", ["-c", rewrite(command)], options); + seen(command, child); + return child; + }; + +const env = { PATH: "/usr/bin:/bin:/usr/sbin:/sbin" }; +const session = reviewSshSession("devbox", tmpdir()); + +async function fixture(bytes = 4 << 20) { + const root = await temporary(); + const local = join(root, "local.bin"); + const content = randomBytes(bytes); + await writeFile(local, content); + return { root, local, content, target: join(root, "remote file.tgz") }; +} + +test("the file lands under its name with every byte, and progress reaches the whole size", async () => { + const { root, local, content, target } = await fixture(); + const progress: number[] = []; + + const sent = await uploadFile(session, local, target, { spawn: localSsh(), env, onProgress: (bytes) => progress.push(bytes) }); + + assert.deepEqual(sent, { sha256: createHash("sha256").update(content).digest("hex"), bytes: content.length }); + assert.deepEqual(await readFile(target), content); + assert.deepEqual((await readdir(root)).sort(), ["local.bin", "remote file.tgz"]); + assert.equal(progress.at(-1), content.length); + assert.deepEqual(progress, [...progress].sort((a, b) => a - b)); +}); + +test("an upload whose ssh ends halfway leaves only the .part file", async () => { + const { root, local, target } = await fixture(32 << 20); + let upload: SshChildProcess | undefined; + const spawnSsh = localSsh({ + seen: (command, child) => { + if (command.includes("cat")) upload = child; + }, + }); + + await assert.rejects( + uploadFile(session, local, target, { + spawn: spawnSsh, + env, + onProgress: (sent, total) => { + if (sent > total / 2) upload?.kill("SIGKILL"); + }, + }), + /upload of .* to devbox/, + ); + + assert.deepEqual((await readdir(root)).sort(), ["local.bin", "remote file.tgz.part"]); +}); + +test("a slow remote that keeps reading is not a stall, and one that stops reading is", async () => { + const { local, content, target } = await fixture(1 << 20); + const slow = localSsh({ + rewrite: (command) => + command.replace( + 'cat > "$1.part"', + ': > "$1.part"; while n=$(dd bs=65536 count=1 2>/dev/null | tee -a "$1.part" | wc -c) && [ $n -gt 0 ]; do sleep 0.2; done', + ), + }); + + await uploadFile(session, local, target, { spawn: slow, env, timeouts: { stall: 1_000 } }); + assert.deepEqual(await readFile(target), content); + + const stopped = localSsh({ rewrite: (command) => command.replace('cat > "$1.part"', 'sleep 2; cat > "$1.part"') }); + await assert.rejects( + uploadFile(session, local, `${target}.2`, { spawn: stopped, env, timeouts: { stall: 300 } }), + /stalled: ssh took nothing from its stdin/, + ); +}); + +test("a remote without sha256sum checks the file with openssl", async () => { + const { local, content, target } = await fixture(); + const spawnSsh = localSsh({ rewrite: (command) => command.replace("command -v sha256sum", "command -v no-such-sha256sum") }); + + const sent = await uploadFile(session, local, target, { spawn: spawnSsh, env }); + + assert.equal(sent.sha256, createHash("sha256").update(content).digest("hex")); + assert.deepEqual(await readFile(target), content); +}); + +test("a .part that does not match what was sent is removed, not renamed", async () => { + const { root, local, target } = await fixture(); + const spawnSsh = localSsh({ rewrite: (command) => command.replace('cat > "$1.part"', 'cat > "$1.part"; printf x >> "$1.part"') }); + + await assert.rejects(uploadFile(session, local, target, { spawn: spawnSsh, env }), /does not match/); + + assert.deepEqual(await readdir(root), ["local.bin"]); +}); + +test("a remote path that is relative, or holds a quote, a backslash or a control character, is refused", async () => { + const { local } = await fixture(1); + let spawned = false; + const spawnSsh: SpawnSsh = (args, options) => { + spawned = true; + return localSsh()(args, options); + }; + + for (const path of ["relative/file", "/tmp/it's", "/tmp/a\\b", "/tmp/a\nb", "/tmp/a\u0007b"]) { + await assert.rejects(uploadFile(session, local, path, { spawn: spawnSsh, env }), /cannot upload to/, JSON.stringify(path)); + } + assert.equal(spawned, false); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteUpload.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteUpload.ts new file mode 100644 index 000000000..9e4b88ccf --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewRemoteUpload.ts @@ -0,0 +1,142 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { createHash } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { stat } from "node:fs/promises"; + +import { runSsh, type SpawnSsh } from "./reviewRemoteHost.js"; +import { sshExecArgs, type ReviewSshSession } from "./reviewSshCommand.js"; + +export const REVIEW_REMOTE_UPLOAD_TIMEOUTS = { + stall: 30_000, + flush: 120_000, + finish: 60_000, +}; + +export interface ReviewRemoteUploadOptions { + readonly spawn: SpawnSsh; + readonly env: NodeJS.ProcessEnv; + readonly onProgress?: (sent: number, total: number) => void; + readonly timeouts?: Partial; +} + +const OK = "WHITEBOARD-UPLOAD-OK"; +const MISMATCH = "WHITEBOARD-UPLOAD-MISMATCH"; + +function checkRemotePath(remotePath: string): void { + if (!remotePath.startsWith("/") || remotePath.length > 4096 || /['\\\x00-\x1f\x7f-\x9f]/.test(remotePath)) { + throw new Error(`Whiteboard cannot upload to ${JSON.stringify(remotePath)}: it needs an absolute path without quotes, backslashes or control characters.`); + } +} + +export interface ReviewRemoteUpload { + readonly sha256: string; + readonly bytes: number; +} + +export async function uploadFile( + session: ReviewSshSession, + localPath: string, + remotePath: string, + options: ReviewRemoteUploadOptions, +): Promise { + checkRemotePath(remotePath); + const timeouts = { ...REVIEW_REMOTE_UPLOAD_TIMEOUTS, ...options.timeouts }; + const total = (await stat(localPath)).size; + const where = `The upload of ${localPath} to ${session.alias}:${remotePath}`; + const sha256 = await send(session, localPath, remotePath, total, options, timeouts, where); + + const script = [ + `f='${remotePath}'`, + `size=$(wc -c < "$f.part") || exit 3`, + `if command -v sha256sum >/dev/null 2>&1; then sum=$(sha256sum "$f.part"); else sum=$(openssl dgst -sha256 -r "$f.part"); fi || exit 3`, + `sum=\${sum%% *}`, + `if [ $size -eq ${total} ] && [ "$sum" = ${sha256} ]; then mv -f "$f.part" "$f" && echo ${OK}`, + `else rm -f "$f.part"; echo ${MISMATCH} $size $sum; fi`, + "", + ].join("\n"); + const result = await runSsh(options.spawn, options.env, sshExecArgs(session, options.env), timeouts.finish, script); + if (result.stdout.includes(OK)) return { sha256, bytes: total }; + if (result.timedOut) throw new Error(`${where} was not checked within ${timeouts.finish / 1000} seconds.`); + const mismatch = result.stdout.split("\n").find((line) => line.startsWith(MISMATCH)); + if (mismatch) { + const [, size, sum] = mismatch.split(" "); + throw new Error(`${where} does not match what was sent: ${size} bytes with sha256 ${sum}, not ${total} bytes with ${sha256}. It was removed.`); + } + throw new Error(`${where} could not be checked: ${result.error?.message ?? `exit ${result.code}: ${result.stderr.trim().split("\n").at(-1) ?? ""}`}`); +} + +function send( + session: ReviewSshSession, + localPath: string, + remotePath: string, + total: number, + options: ReviewRemoteUploadOptions, + timeouts: typeof REVIEW_REMOTE_UPLOAD_TIMEOUTS, + where: string, +): Promise { + return new Promise((resolve, reject) => { + const args = sshExecArgs(session, options.env, ["sh", "-c", `'cat > "$1.part"'`, "sh", `'${remotePath}'`]); + let child: ReturnType; + try { + child = options.spawn(args, { env: options.env, detached: true, stdio: ["pipe", "ignore", "pipe"] }); + } catch (error) { + return reject(new Error(`${where} could not start: ${(error as Error).message}`)); + } + const file = createReadStream(localPath); + const hash = createHash("sha256"); + const stdin = child.stdin!; + let sent = 0; + let stderr = ""; + let failure: string | undefined; + let stall: ReturnType; + const watch = (ms: number, what: string) => { + clearTimeout(stall); + stall = setTimeout(() => { + failure = `stalled: ${what} for ${ms / 1000} seconds`; + child.kill("SIGKILL"); + }, ms); + }; + const accepted = () => watch(timeouts.stall, "ssh took nothing from its stdin"); + accepted(); + + child.stderr?.setEncoding("utf8"); + child.stderr?.on("data", (chunk: string) => (stderr = (stderr + chunk).slice(0, 4096))); + stdin.on("error", () => {}); + file.on("error", (error) => { + failure = `reading the file failed: ${error.message}`; + child.kill("SIGKILL"); + }); + file.on("data", (chunk) => { + hash.update(chunk); + sent += chunk.length; + options.onProgress?.(sent, total); + if (stdin.write(chunk)) return accepted(); + file.pause(); + stdin.once("drain", () => { + accepted(); + file.resume(); + }); + }); + file.on("end", () => { + stdin.end(); + watch(timeouts.flush, "ssh did not finish sending what it held"); + }); + child.once("exit", () => { + file.destroy(); + stdin.destroy(); + }); + child.once("error", (error: Error) => { + failure ??= `ssh could not start: ${error.message}`; + }); + child.once("close", (code: number | null, signal: NodeJS.Signals | null) => { + clearTimeout(stall); + if (!failure && code === 0 && sent === total) return resolve(hash.digest("hex")); + const reason = failure ?? (code === 0 ? `the file changed while it was sent (${sent} of ${total} bytes)` : `ssh ${signal ? `was stopped (${signal})` : `exited with ${code}`}${stderr.trim() ? `: ${stderr.trim().split("\n").at(-1)}` : ""}`); + reject(new Error(`${where} did not finish after ${sent} of ${total} bytes: ${reason}.`)); + }); + }); +} diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewSshAskpass.test.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewSshAskpass.test.ts new file mode 100644 index 000000000..41a97f249 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewSshAskpass.test.ts @@ -0,0 +1,158 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { existsSync } from "node:fs"; +import { lstat, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test, { type TestContext } from "node:test"; +import { fileURLToPath } from "node:url"; + +import { sshPromptKind, type SshPromptKind } from "../../common/reviewSshPrompt.js"; +import { createSshAskpass, type SshPromptRequest } from "./reviewSshAskpass.js"; + +const helper = [process.execPath, "--import", "tsx", fileURLToPath(new URL("../../node/reviewSshAskpassMain.ts", import.meta.url))]; + +const passphrase = "Enter passphrase for key '/home/u/.ssh/id_ed25519': "; +const password = "dev@127.0.0.1's password: "; +const hostKey = "Are you sure you want to continue connecting (yes/no/[fingerprint])? "; +const code = "Verification code: "; + +async function listener(t: TestContext, prompt: (request: SshPromptRequest) => Promise) { + const directory = await mkdtemp(join(tmpdir(), "wb-askpass-")); + const log: string[] = []; + const askpass = await createSshAskpass({ directory, prompt, helper, log: (message) => log.push(message) }); + t.after(async () => { + askpass.dispose(); + await rm(directory, { recursive: true, force: true }); + }); + return { askpass, directory, log }; +} + +function runAskpass(env: Record, text: string, command = [env.SSH_ASKPASS]): Promise<{ code: number | null; stdout: string; stderr: string }> { + return new Promise((resolve, reject) => { + const child = spawn(command[0], [...command.slice(1), text], { env: { ...process.env, ...env }, stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => (stdout += chunk)); + child.stderr.on("data", (chunk) => (stderr += chunk)); + child.on("error", reject); + child.on("close", (exitCode) => resolve({ code: exitCode, stdout, stderr })); + }); +} + +test("the helper prints the answer with no newline and exits 0, and the log never holds the answer", async (t) => { + const seen: { alias: string; text: string; kind: SshPromptKind }[] = []; + const { askpass, log } = await listener(t, async ({ alias, text, kind }) => { + seen.push({ alias, text, kind }); + return "correct horse"; + }); + + const result = await runAskpass(askpass.env("wb-test-a"), password); + + assert.deepEqual(result, { code: 0, stdout: "correct horse", stderr: "" }); + assert.deepEqual(seen, [{ alias: "wb-test-a", text: password, kind: "secret" }]); + assert.ok(log.some((line) => line.includes("secret") && line.includes("wb-test-a") && line.includes("answered")), log.join("\n")); + assert.ok(!log.join("\n").includes("correct horse")); +}); + +test("a cancelled prompt makes the helper exit non-zero and print nothing", async (t) => { + const { askpass, log } = await listener(t, async () => undefined); + + const result = await runAskpass(askpass.env("wb-test-a"), password); + + assert.notEqual(result.code, 0); + assert.equal(result.stdout, ""); + assert.ok(log.some((line) => line.includes("cancelled")), log.join("\n")); +}); + +test("after a cancel, the same ssh's retry of that prompt is not shown, but a different prompt is", async (t) => { + const shown: string[] = []; + const { askpass } = await listener(t, async ({ text }) => { + shown.push(text); + return undefined; + }); + const env = askpass.env("wb-test-a"); + + assert.notEqual((await runAskpass(env, passphrase)).code, 0); + assert.notEqual((await runAskpass(env, password)).code, 0); + assert.notEqual((await runAskpass(env, password)).code, 0); + assert.deepEqual(shown, [passphrase, password]); +}); + +test("a failing prompt is logged as an error, and the retry is still shown", async (t) => { + let calls = 0; + const { askpass, log } = await listener(t, async () => { + if (++calls === 1) throw new Error("relay broke"); + return "second"; + }); + const env = askpass.env("wb-test-a"); + + assert.notEqual((await runAskpass(env, password)).code, 0); + assert.equal((await runAskpass(env, password)).stdout, "second"); + assert.ok(log.some((line) => line.includes("failed: relay broke")), log.join("\n")); +}); + +test("the helper fails without printing when the listener is gone", async (t) => { + const { askpass, directory } = await listener(t, async () => "unused"); + const env = { ...askpass.env("wb-test-a"), DEV_FAST_REVIEW_SSH_ASKPASS_SOCKET: join(directory, "gone.sock") }; + + const result = await runAskpass(env, password, helper); + + assert.deepEqual(result, { code: 1, stdout: "", stderr: "" }); +}); + +test("OpenSSH's prompts get the right kind", () => { + assert.equal(sshPromptKind(passphrase), "secret"); + assert.equal(sshPromptKind(password), "secret"); + assert.equal(sshPromptKind(`The authenticity of host '[127.0.0.1]:2222 ([127.0.0.1]:2222)' can't be established.\nED25519 key fingerprint is SHA256:abc.\n${hostKey}`), "confirm"); + assert.equal(sshPromptKind("Please type 'yes', 'no' or the fingerprint: "), "text"); + assert.equal(sshPromptKind(code), "text"); +}); + +test("two hosts that prompt at the same time are asked one after the other", async (t) => { + let active = 0; + let most = 0; + const order: string[] = []; + const { askpass } = await listener(t, async ({ alias }) => { + most = Math.max(most, ++active); + order.push(alias); + await new Promise((resolve) => setTimeout(resolve, 100)); + active--; + return `answer for ${alias}`; + }); + + const [a, b] = await Promise.all([runAskpass(askpass.env("wb-test-a"), password), runAskpass(askpass.env("wb-test-b"), passphrase)]); + + assert.equal(most, 1); + assert.deepEqual(order.toSorted(), ["wb-test-a", "wb-test-b"]); + assert.equal(a.stdout, "answer for wb-test-a"); + assert.equal(b.stdout, "answer for wb-test-b"); +}); + +test("the script and the socket are the owner's alone, and dispose removes both", async (t) => { + const { askpass } = await listener(t, async () => undefined); + const { SSH_ASKPASS: script, DEV_FAST_REVIEW_SSH_ASKPASS_SOCKET: socket } = askpass.env("wb-test-a"); + + assert.equal((await lstat(script)).mode & 0o777, 0o700); + assert.equal((await lstat(socket)).mode & 0o777, 0o600); + assert.ok((await lstat(socket)).isSocket()); + + askpass.dispose(); + assert.equal(existsSync(script), false); + assert.equal(existsSync(socket), false); +}); + +test("env forces the helper, carries no answer, and refuses an invalid alias", async (t) => { + const { askpass } = await listener(t, async () => undefined); + const env = askpass.env("wb-test-a"); + + assert.equal(env.SSH_ASKPASS_REQUIRE, "force"); + assert.ok(env.DISPLAY); + assert.equal(env.DEV_FAST_REVIEW_SSH_ASKPASS_ALIAS, "wb-test-a"); + assert.throws(() => askpass.env("-oProxyCommand=x"), /starts with -/); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewSshAskpass.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewSshAskpass.ts new file mode 100644 index 000000000..310a81063 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewSshAskpass.ts @@ -0,0 +1,161 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { rmSync } from "node:fs"; +import { chmod, rm, writeFile } from "node:fs/promises"; +import { createServer, type Socket } from "node:net"; +import { join } from "node:path"; +import { FileAccess } from "../../../base/common/network.js"; +import { + REVIEW_SSH_ASKPASS_ALIAS_ENV, + REVIEW_SSH_ASKPASS_SOCKET_ENV, + sshPromptKind, + type SshPromptKind, +} from "../../common/reviewSshPrompt.js"; +import { validateSshAlias } from "./reviewSshCommand.js"; + +export interface SshPromptRequest { + readonly alias: string; + readonly text: string; + readonly kind: SshPromptKind; + readonly signal?: AbortSignal; +} + +export interface ReviewSshAskpass { + env(alias: string): Record; + dispose(): void; +} + +const MAX_REQUEST_BYTES = 64 * 1024; + +function defaultHelper(): string[] { + return [process.execPath, FileAccess.asFileUri("vs/review/node/reviewSshAskpassMain.js").fsPath]; +} + +const shellQuote = (value: string) => `'${value.replaceAll("'", `'\\''`)}'`; + +function isAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} + +export async function createSshAskpass(input: { + directory: string; + prompt(request: SshPromptRequest): Promise; + log?: (message: string) => void; + helper?: readonly string[]; +}): Promise { + const log = input.log ?? (() => {}); + const name = `askpass-${process.pid}`; + const scriptPath = join(input.directory, name); + const socketPath = join(input.directory, `${name}.sock`); + + await rm(scriptPath, { force: true }); + await writeFile( + scriptPath, + `#!/bin/sh\nELECTRON_RUN_AS_NODE=1 exec ${(input.helper ?? defaultHelper()).map(shellQuote).join(" ")} "$@"\n`, + { mode: 0o700, flag: "wx" }, + ); + + // ssh sends a cancelled password as empty and asks again with the same text; those retries are not shown. + const cancelled = new Map>(); + let queue = Promise.resolve(); + + async function ask(alias: string, text: string, pid: number, signal: AbortSignal): Promise { + if (signal.aborted) return undefined; + const kind = sshPromptKind(text); + for (const ssh of cancelled.keys()) if (!isAlive(ssh)) cancelled.delete(ssh); + if (cancelled.get(pid)?.has(text)) { + log(`ssh prompt (${kind}) for ${alias}: cancelled, the same prompt of this ssh was cancelled`); + return undefined; + } + log(`ssh prompt (${kind}) for ${alias}: shown`); + let answer: string | undefined; + try { + answer = await input.prompt({ alias, text, kind, signal }); + } catch (error) { + log(`ssh prompt (${kind}) for ${alias}: failed: ${(error as Error).message}`); + return undefined; + } + if (answer === undefined) cancelled.set(pid, (cancelled.get(pid) ?? new Set()).add(text)); + log(`ssh prompt (${kind}) for ${alias}: ${answer === undefined ? "cancelled" : "answered"}`); + return answer; + } + + const sockets = new Set(); + + function serve(socket: Socket) { + const closed = new AbortController(); + sockets.add(socket); + socket.on("close", () => { + sockets.delete(socket); + closed.abort(); + }); + socket.on("error", () => socket.destroy()); + socket.setEncoding("utf8"); + let buffer = ""; + const onData = (chunk: string) => { + buffer += chunk; + const end = buffer.indexOf("\n"); + if (end < 0) { + if (buffer.length > MAX_REQUEST_BYTES) socket.destroy(); + return; + } + socket.off("data", onData); + const request = parseRequest(buffer.slice(0, end)); + if (!request) return void socket.destroy(); + const turn = queue.then(() => ask(request.alias, request.text, request.pid, closed.signal)); + queue = turn.then(() => undefined); + void turn.then((answer) => { + if (!socket.destroyed) socket.end(`${JSON.stringify(answer === undefined ? { cancelled: true } : { answer })}\n`); + }); + }; + socket.on("data", onData); + } + + const server = createServer(serve); + await rm(socketPath, { force: true }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(socketPath, () => resolve()); + }); + await chmod(socketPath, 0o600); + + return { + env(alias) { + const valid = validateSshAlias(alias); + if (!valid.ok) throw new Error(`SSH alias ${JSON.stringify(alias)} ${valid.reason}.`); + return { + SSH_ASKPASS: scriptPath, + SSH_ASKPASS_REQUIRE: "force", + // OpenSSH before 8.4 ignores SSH_ASKPASS_REQUIRE and uses askpass only with DISPLAY set and no tty. + DISPLAY: process.env.DISPLAY || "whiteboard-askpass:0", + [REVIEW_SSH_ASKPASS_SOCKET_ENV]: socketPath, + [REVIEW_SSH_ASKPASS_ALIAS_ENV]: alias, + }; + }, + dispose() { + server.close(); + for (const socket of sockets) socket.destroy(); + rmSync(socketPath, { force: true }); + rmSync(scriptPath, { force: true }); + }, + }; +} + +function parseRequest(line: string): { alias: string; text: string; pid: number } | undefined { + try { + const { alias, text, pid } = JSON.parse(line) as Record; + if (typeof alias !== "string" || !validateSshAlias(alias).ok) return undefined; + if (typeof text !== "string" || !Number.isInteger(pid) || (pid as number) < 2) return undefined; + return { alias, text, pid: pid as number }; + } catch { + return undefined; + } +} diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewSshCommand.test.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewSshCommand.test.ts new file mode 100644 index 000000000..2093e283f --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewSshCommand.test.ts @@ -0,0 +1,108 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from "node:assert/strict"; +import { chmod, lstat, mkdtemp, rm, symlink } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; +import test from "node:test"; + +import { + prepareSshControlDirectory, + reviewSshConfigPath, + reviewSshControlDirectory, + reviewSshInstancePrefix, + reviewSshSession, + sshCancelForwardArgs, + sshCheckArgs, + sshCloseArgs, + sshExecArgs, + sshForwardArgs, + sshMasterArgs, + validateSshAlias, +} from "./reviewSshCommand.js"; + +const session = { alias: "wb-test-a", controlPath: "/tmp/wb-ssh-501/0123456789ab" }; + +const allArgs = (env: NodeJS.ProcessEnv = {}) => [ + sshMasterArgs(session, env), + sshExecArgs(session, env), + sshForwardArgs(session, 41000, 42000, env), + sshCancelForwardArgs(session, 41000, 42000, env), + sshCloseArgs(session, env), + sshCheckArgs(session, env), +]; + +test("refuses aliases that could become options or reach a shell", () => { + for (const alias of ["-oProxyCommand=x", "a b", "a`id`", "", "a\u0001b", "a$HOME", "a;b", "a|b", "a&b", "ab", "a(b", "a)b", "a'b", 'a"b', "a\\b"]) + assert.equal(validateSshAlias(alias).ok, false, JSON.stringify(alias)); +}); + +test("accepts ordinary aliases", () => { + for (const alias of ["wb-test-a", "dev@build.example.com", "box_2", "10.0.0.7", "a-"]) + assert.deepEqual(validateSshAlias(alias), { ok: true }, alias); +}); + +test("a session and every builder refuse an invalid alias", () => { + assert.throws(() => reviewSshSession("-oProxyCommand=x", "/tmp/d"), /starts with -/); + const handBuilt = { alias: "-oProxyCommand=x", controlPath: "/tmp/d/x" }; + assert.throws(() => sshMasterArgs(handBuilt, {}), /starts with -/); + assert.throws(() => sshForwardArgs(handBuilt, 1, 2, {}), /starts with -/); +}); + +test("the forward refuses a port outside 1-65535", () => { + assert.throws(() => sshForwardArgs(session, 0, 42000, {}), /Invalid port 0/); + assert.throws(() => sshCancelForwardArgs(session, 41000, 1.5, {}), /Invalid port 1.5/); +}); + +test("no argument list overrides host-key checking or prompting", () => { + for (const args of allArgs({ DEV_FAST_REVIEW_SSH_CONFIG: "/tmp/c", VSCODE_DEV: "1" })) + for (const arg of args) assert.doesNotMatch(arg, /StrictHostKeyChecking|UserKnownHostsFile|BatchMode/i); +}); + +test("the forward and its cancel bind loopback on both ends", () => { + for (const args of [sshForwardArgs(session, 41000, 42000, {}), sshCancelForwardArgs(session, 41000, 42000, {})]) + assert.equal(args[args.indexOf("-L") + 1], "127.0.0.1:41000:127.0.0.1:42000"); +}); + +test("a development config file is passed to every call, and ignored in a packaged build", () => { + for (const args of allArgs({ DEV_FAST_REVIEW_SSH_CONFIG: "/tmp/c", VSCODE_DEV: "1" })) + assert.deepEqual(args.slice(0, 2), ["-F", "/tmp/c"]); + for (const args of allArgs({ DEV_FAST_REVIEW_SSH_CONFIG: "/tmp/c" })) + assert.equal(args.includes("-F"), false); + assert.equal(reviewSshConfigPath({ DEV_FAST_REVIEW_SSH_CONFIG: "/tmp/c", VSCODE_DEV: "1" }, "/home/u"), "/tmp/c"); + assert.equal(reviewSshConfigPath({ DEV_FAST_REVIEW_SSH_CONFIG: "/tmp/c" }, "/home/u"), "/home/u/.ssh/config"); +}); + +test("the control path is under 100 bytes for a 40-character alias", () => { + const alias = "a".repeat(40); + const { controlPath } = reviewSshSession(alias); + assert.ok(Buffer.byteLength(controlPath) < 100, controlPath); + assert.ok(controlPath.startsWith(reviewSshControlDirectory())); + assert.notEqual(reviewSshSession("b".repeat(40)).controlPath, controlPath); + const instance = "/Users/u/Library/Application Support/Whiteboard"; + const ofInstance = reviewSshSession(alias, undefined, instance).controlPath; + assert.notEqual(ofInstance, controlPath); + assert.ok(Buffer.byteLength(ofInstance) < 100, ofInstance); + assert.ok(basename(ofInstance).startsWith(reviewSshInstancePrefix(instance))); + assert.ok(!basename(reviewSshSession(alias, undefined, `${instance} Preview`).controlPath).startsWith(reviewSshInstancePrefix(instance))); +}); + +test("the control directory is created 0700, repaired, and a symlink is refused", async (t) => { + const parent = await mkdtemp(join(tmpdir(), "wb-ssh-test-")); + t.after(() => rm(parent, { recursive: true, force: true })); + const dir = join(parent, "control"); + + await prepareSshControlDirectory(dir); + assert.equal((await lstat(dir)).mode & 0o777, 0o700); + + await chmod(dir, 0o755); + await prepareSshControlDirectory(dir); + assert.equal((await lstat(dir)).mode & 0o777, 0o700); + + const link = join(parent, "link"); + await symlink(dir, link); + await assert.rejects(prepareSshControlDirectory(link), /not a directory owned by this user/); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewSshCommand.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewSshCommand.ts new file mode 100644 index 000000000..75d66ccec --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewSshCommand.ts @@ -0,0 +1,137 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { createHash } from "node:crypto"; +import { chmod, lstat, mkdir } from "node:fs/promises"; +import { homedir, tmpdir } from "node:os"; +import { join } from "node:path"; + +import { validateSshAlias } from "../../common/reviewSshAlias.js"; + +export { validateSshAlias }; + +/** + * Development only: an ssh_config file every `ssh` call and the alias list + * use, because OpenSSH finds `~/.ssh` from the account, not from `HOME`. + */ +export const REVIEW_SSH_CONFIG_ENV = "DEV_FAST_REVIEW_SSH_CONFIG"; + +export interface ReviewSshSession { + readonly alias: string; + readonly controlPath: string; +} + +/** Connection sharing needs Unix sockets and a uid; Windows has neither. */ +function currentUid(): number { + if (!process.getuid) throw new Error("SSH connection sharing is not supported on this platform."); + return process.getuid(); +} + +/** Per user and short: macOS limits a socket path to about 104 bytes, and ssh appends 17 to it while binding. */ +export function reviewSshControlDirectory(): string { + return join(tmpdir(), `wb-ssh-${currentUid()}`); +} + +const sha = (value: string) => createHash("sha256").update(value).digest("hex"); + +export function reviewSshInstancePrefix(instance: string): string { + return `${sha(instance).slice(0, 6)}-`; +} + +export function reviewSshSession(alias: string, controlDirectory = reviewSshControlDirectory(), instance?: string): ReviewSshSession { + checkAlias(alias); + const name = instance === undefined ? sha(alias).slice(0, 12) : `${reviewSshInstancePrefix(instance)}${sha(`${instance}\n${alias}`).slice(0, 12)}`; + return { alias, controlPath: join(controlDirectory, name) }; +} + +function checkAlias(alias: string): void { + const valid = validateSshAlias(alias); + if (!valid.ok) throw new Error(`SSH alias ${JSON.stringify(alias)} ${valid.reason}.`); +} + +export async function prepareSshControlDirectory(dir: string): Promise { + await mkdir(dir, { mode: 0o700 }).catch((error: NodeJS.ErrnoException) => { + if (error.code !== "EEXIST") throw error; + }); + const stat = await lstat(dir); + if (!stat.isDirectory() || stat.uid !== currentUid()) { + throw new Error(`${dir} is not a directory owned by this user.`); + } + if ((stat.mode & 0o777) !== 0o700) await chmod(dir, 0o700); +} + +function developmentConfig(env: NodeJS.ProcessEnv): string | undefined { + return env.VSCODE_DEV ? env[REVIEW_SSH_CONFIG_ENV] || undefined : undefined; +} + +export function reviewSshConfigPath(env: NodeJS.ProcessEnv = process.env, home = homedir()): string { + return developmentConfig(env) ?? join(home, ".ssh", "config"); +} + +function base(session: ReviewSshSession, env: NodeJS.ProcessEnv): string[] { + checkAlias(session.alias); + const config = developmentConfig(env); + return [...(config ? ["-F", config] : []), "-S", session.controlPath]; +} + +export function sshMasterArgs(session: ReviewSshSession, env: NodeJS.ProcessEnv = process.env): string[] { + return [ + ...base(session, env), + "-M", + "-N", + "-oServerAliveInterval=15", + "-oServerAliveCountMax=3", + "-oControlPersist=no", + "--", + session.alias, + ]; +} + +export function sshExecArgs(session: ReviewSshSession, env: NodeJS.ProcessEnv = process.env, command: readonly string[] = ["sh", "-s"]): string[] { + return [...base(session, env), "-oControlMaster=no", "-T", "--", session.alias, ...command]; +} + +function localForward(localPort: number, remotePort: number): string { + for (const port of [localPort, remotePort]) { + if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error(`Invalid port ${port}.`); + } + return `127.0.0.1:${localPort}:127.0.0.1:${remotePort}`; +} + +export function sshForwardArgs( + session: ReviewSshSession, + localPort: number, + remotePort: number, + env: NodeJS.ProcessEnv = process.env, +): string[] { + return [...base(session, env), "-O", "forward", "-L", localForward(localPort, remotePort), "--", session.alias]; +} + +export function sshCancelForwardArgs( + session: ReviewSshSession, + localPort: number, + remotePort: number, + env: NodeJS.ProcessEnv = process.env, +): string[] { + return [...base(session, env), "-O", "cancel", "-L", localForward(localPort, remotePort), "--", session.alias]; +} + +export function sshRemoteForwardArgs( + session: ReviewSshSession, + localPort: number, + operation: "forward" | "cancel", + env: NodeJS.ProcessEnv = process.env, +): string[] { + if (!Number.isInteger(localPort) || localPort < 1 || localPort > 65535) throw new Error(`Invalid port ${localPort}.`); + return [...base(session, env), "-O", operation, "-R", `127.0.0.1:0:127.0.0.1:${localPort}`, "--", session.alias]; +} + +export function sshCheckArgs(session: ReviewSshSession, env: NodeJS.ProcessEnv = process.env): string[] { + return [...base(session, env), "-O", "check", "--", session.alias]; +} + +export function sshCloseArgs(session: ReviewSshSession, env: NodeJS.ProcessEnv = process.env): string[] { + return [...base(session, env), "-O", "exit", "--", session.alias]; +} diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewSshConfigAliases.test.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewSshConfigAliases.test.ts new file mode 100644 index 000000000..e2cd88404 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewSshConfigAliases.test.ts @@ -0,0 +1,77 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { listSshAliases } from "./reviewSshConfigAliases.js"; + +async function fixture(t: test.TestContext, files: Record): Promise { + const dir = await mkdtemp(join(tmpdir(), "wb-ssh-aliases-")); + t.after(() => rm(dir, { recursive: true, force: true })); + for (const [name, text] of Object.entries(files)) { + await mkdir(join(dir, name, ".."), { recursive: true }); + await writeFile(join(dir, name), text); + } + return dir; +} + +test("lists Host aliases in file order, follows Include globs, and skips patterns", async (t) => { + const dir = await fixture(t, { + config: [ + "# comment", + "Host first second", + " HostName 10.0.0.1", + "include conf.d/*.conf", + "Host *", + " ServerAliveInterval 30", + "Host *.example.com web? !blocked kept", + "HOST=equals", + "Match host first exec true", + " User dev", + 'Host "quoted"', + "Host first", + "Include missing/*.conf", + ].join("\n"), + "conf.d/b.conf": "Host from-b\n", + "conf.d/a.conf": "Host from-a first\n", + "conf.d/skip.txt": "Host not-included\n", + }); + + assert.deepEqual(await listSshAliases(join(dir, "config")), [ + "first", + "second", + "from-a", + "from-b", + "kept", + "equals", + "quoted", + ]); +}); + +test("follows absolute Include paths and stops a cyclic Include", async (t) => { + const other = await fixture(t, { extra: "Host absolute\n" }); + const dir = await fixture(t, { + config: `Host top\nInclude loop ${join(other, "extra")}\n`, + loop: "Host looped\nInclude config\nInclude loop\n", + }); + + assert.deepEqual(await listSshAliases(join(dir, "config")), ["top", "looped", "absolute"]); +}); + +test("leaves out aliases Whiteboard would refuse", async (t) => { + const dir = await fixture(t, { config: 'Host ok db(prod) -flag a;b "two words"\n' }); + + assert.deepEqual(await listSshAliases(join(dir, "config")), ["ok"]); +}); + +test("a missing or unreadable file gives an empty list", async (t) => { + const dir = await fixture(t, { "config/inner": "" }); + assert.deepEqual(await listSshAliases(join(dir, "nothing")), []); + assert.deepEqual(await listSshAliases(join(dir, "config")), []); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewSshConfigAliases.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewSshConfigAliases.ts new file mode 100644 index 000000000..f5222c746 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewSshConfigAliases.ts @@ -0,0 +1,62 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { glob, readFile } from "node:fs/promises"; +import { homedir } from "node:os"; +import { dirname, isAbsolute, join } from "node:path"; + +import { validateSshAlias } from "../../common/reviewSshAlias.js"; + +export async function listSshAliases(configPath: string): Promise { + const aliases = new Set(); + await readConfig(configPath, dirname(configPath), new Set(), aliases); + return [...aliases].filter((alias) => validateSshAlias(alias).ok); +} + +async function readConfig(file: string, includeDir: string, read: Set, aliases: Set): Promise { + if (read.has(file)) return; + read.add(file); + let text: string; + try { + text = await readFile(file, "utf8"); + } catch { + return; + } + for (const line of text.split(/\r?\n/)) { + const match = /^\s*(\w+)(?:\s*=\s*|\s+)(.*)$/.exec(line); + if (!match) continue; + const keyword = match[1].toLowerCase(); + const words = argumentsOf(match[2]); + if (keyword === "host") { + for (const word of words) if (!/[*?!]/.test(word)) aliases.add(word); + } else if (keyword === "include") { + for (const word of words) { + for (const included of await expand(word, includeDir)) { + await readConfig(included, includeDir, read, aliases); + } + } + } + } +} + +function argumentsOf(rest: string): string[] { + const words: string[] = []; + for (const [, quoted, bare] of rest.matchAll(/"([^"]*)"|(\S+)/g)) { + if (bare?.startsWith("#")) break; + words.push(quoted ?? bare); + } + return words; +} + +async function expand(pattern: string, includeDir: string): Promise { + const path = pattern.startsWith("~/") ? join(homedir(), pattern.slice(2)) : isAbsolute(pattern) ? pattern : join(includeDir, pattern); + const files: string[] = []; + try { + for await (const file of glob(path)) files.push(file); + } catch { + return []; + } + return files.sort(); +} diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewSshPromptRelay.test.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewSshPromptRelay.test.ts new file mode 100644 index 000000000..ca9ea1c76 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewSshPromptRelay.test.ts @@ -0,0 +1,84 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from "node:assert/strict"; +import test from "node:test"; + +import type { ReviewSshPromptEvent } from "../../common/reviewSshPrompt.js"; +import { ReviewSshPromptRelay } from "./reviewSshPromptRelay.js"; + +const request = { alias: "wb-test-a", text: "dev@127.0.0.1's password: ", kind: "secret" as const }; + +function relayWithWindow(windowWaitMs = 1000) { + const relay = new ReviewSshPromptRelay(windowWaitMs); + const events: ReviewSshPromptEvent[] = []; + const window = relay.onPrompt((event) => events.push(event)); + return { relay, events, window }; +} + +test("a window gets the prompt and its answer resolves it", async () => { + const { relay, events } = relayWithWindow(); + + const answer = relay.prompt(request); + const [shown] = events; + assert.ok(shown && !("closed" in shown)); + assert.equal(shown.alias, "wb-test-a"); + relay.answer(shown.id, "hunter2"); + + assert.equal(await answer, "hunter2"); + assert.deepEqual(events.at(-1), { id: shown.id, closed: true }); + relay.dispose(); +}); + +test("with no window, the prompt waits for one", async () => { + const relay = new ReviewSshPromptRelay(1000); + const answer = relay.prompt(request); + + const events: ReviewSshPromptEvent[] = []; + relay.onPrompt((event) => { + events.push(event); + if (!("closed" in event)) relay.answer(event.id, "late"); + }); + + assert.equal(await answer, "late"); + relay.dispose(); +}); + +test("with no window for the whole wait, the prompt is cancelled", async () => { + const relay = new ReviewSshPromptRelay(20); + assert.equal(await relay.prompt(request), undefined); + relay.dispose(); +}); + +test("an abandoned prompt is closed in the window", async () => { + const { relay, events } = relayWithWindow(); + const abort = new AbortController(); + + const answer = relay.prompt({ ...request, signal: abort.signal }); + abort.abort(); + + assert.equal(await answer, undefined); + assert.ok(events.some((event) => "closed" in event)); + relay.dispose(); +}); + +test("a prompt open in a window that reloads is shown to the reloaded window", async () => { + const relay = new ReviewSshPromptRelay(1000); + const first: ReviewSshPromptEvent[] = []; + const window = relay.onPrompt((event) => first.push(event)); + const answer = relay.prompt(request); + assert.equal(first.length, 1); + + window.dispose(); + const second: ReviewSshPromptEvent[] = []; + relay.onPrompt((event) => second.push(event)); + const [shown] = second; + assert.ok(shown && !("closed" in shown)); + assert.equal(shown.id, first[0].id); + relay.answer(shown.id, "late"); + + assert.equal(await answer, "late"); + relay.dispose(); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewSshPromptRelay.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewSshPromptRelay.ts new file mode 100644 index 000000000..e24fc549c --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/reviewSshPromptRelay.ts @@ -0,0 +1,77 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Emitter } from "../../../base/common/event.js"; +import type { ReviewSshPromptEvent } from "../../common/reviewSshPrompt.js"; +import type { SshPromptRequest } from "./reviewSshAskpass.js"; + +export const REVIEW_SSH_WINDOW_WAIT_MS = 60_000; + +export class ReviewSshPromptRelay { + private nextId = 1; + private readonly pending = new Map(); + private windowTimer: ReturnType | undefined; + + private readonly emitter = new Emitter({ + onDidAddListener: () => { + this.stopWaiting(); + for (const id of this.pending.keys()) this.show(id); + }, + onDidRemoveLastListener: () => this.startWaiting(), + }); + + readonly onPrompt = this.emitter.event; + + constructor(private readonly windowWaitMs = REVIEW_SSH_WINDOW_WAIT_MS) {} + + prompt(request: SshPromptRequest): Promise { + return new Promise((resolve) => { + const id = this.nextId++; + this.pending.set(id, { request, resolve }); + request.signal?.addEventListener("abort", () => this.answer(id, undefined), { once: true }); + if (this.emitter.hasListeners()) this.show(id); + else this.startWaiting(); + }); + } + + answer(id: number, answer: string | undefined): void { + const pending = this.pending.get(id); + if (!pending) return; + this.pending.delete(id); + if (!this.pending.size) this.stopWaiting(); + pending.resolve(answer); + this.emitter.fire({ id, closed: true }); + } + + dispose(): void { + for (const id of [...this.pending.keys()]) this.answer(id, undefined); + this.stopWaiting(); + this.emitter.dispose(); + } + + private show(id: number): void { + const pending = this.pending.get(id); + if (!pending) return; + const { alias, text, kind } = pending.request; + this.emitter.fire({ id, alias, text, kind }); + } + + private startWaiting(): void { + if (this.windowTimer || !this.pending.size) return; + this.windowTimer = setTimeout(() => { + this.windowTimer = undefined; + for (const id of [...this.pending.keys()]) this.answer(id, undefined); + }, this.windowWaitMs); + } + + private stopWaiting(): void { + clearTimeout(this.windowTimer); + this.windowTimer = undefined; + } +} + +export const reviewSshPromptRelay = new ReviewSshPromptRelay(); + +export const reviewRemoteInstallPromptRelay = new ReviewSshPromptRelay(); diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/test/fakeSsh.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/test/fakeSsh.ts new file mode 100644 index 000000000..b50da3314 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/remote/test/fakeSsh.ts @@ -0,0 +1,227 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { EventEmitter } from "node:events"; +import { PassThrough } from "node:stream"; +import type { ReviewRemoteClock, SpawnSsh, SshChildProcess } from "../reviewRemoteHost.js"; +import { REVIEW_REMOTE_INSTALL_SAY } from "../reviewRemoteInstallScript.js"; +import { REVIEW_REMOTE_PROBE_BEGIN, REVIEW_REMOTE_PROBE_END, REVIEW_REMOTE_PROBE_PATH_CLI } from "../reviewRemoteProbeScript.js"; + +class FakeChild extends EventEmitter { + static nextPid = 1000; + readonly pid = FakeChild.nextPid++; + exitCode: number | null = null; + signalCode: NodeJS.Signals | null = null; + exitedAt: number | undefined; + readonly stdin = new PassThrough(); + readonly stdout = new PassThrough(); + readonly stderr = new PassThrough(); + input = ""; + + constructor() { + super(); + this.stdin.setEncoding("utf8").on("data", (chunk: string) => (this.input += chunk)); + } + + get alive() { + return this.exitCode === null && this.signalCode === null; + } + + finish(code: number | null, output: { stdout?: string; stderr?: string } = {}, signal: NodeJS.Signals | null = null) { + if (!this.alive) return; + if (output.stdout) this.stdout.write(output.stdout); + if (output.stderr) this.stderr.write(output.stderr); + this.stdout.end(); + this.stderr.end(); + setImmediate(() => { + this.exitCode = code; + this.signalCode = signal; + this.exitedAt = Date.now(); + this.emit("exit", code, signal); + this.emit("close", code, signal); + }); + } + + kill(signal: NodeJS.Signals = "SIGTERM") { + this.finish(null, {}, signal); + return true; + } +} + +type MasterOutcome = "up" | "hang" | "missing" | { code: number; stderr: string }; + +export interface FakeRemote { + master?: MasterOutcome | ((alias: string) => MasterOutcome); + attach?: Attach | ((call: number) => Attach); + probe?: Partial>; + remotePort?: number; + masterStderr?: string; + exitDelayMs?: number; + checkAnswered?: (call: number) => Promise | undefined; + detect?: Attach; + connect?: Attach; + uninstall?: { ok: boolean; after?: Promise }; +} + +type Attach = { code: number; stdout?: string; stderr?: string }; + +export interface FakeCall { + readonly alias: string; + kind: "master" | "check" | "exec" | "probe" | "detect" | "connect" | "uninstall" | "forward" | "cancel" | "exit"; + readonly args: readonly string[]; + input?: string; + readonly at: number; + readonly wall: number; +} + +export const FAKE_SERVER_ID = "0199a3f2-7c1e-7d4a-9b2f-3e5d6c7b8a90"; + +export const detectOutput = (agents: readonly Record[]) => `${JSON.stringify({ event: "connect.detect", agents })}\n`; + +const connectedOutput = (input: string) => + `${JSON.stringify({ event: "connect.run", agents: [...input.matchAll(/'(claude|codex|opencode|pi)'/g)].map(([, id]) => ({ id, name: id, connected: true, output: "" })) })}\n`; + +export const attachOutput = (port: number, token = "remote-token", extra: Record = {}) => + `WHITEBOARD-REMOTE-BEGIN\n${JSON.stringify({ event: "remote.attach", version: "0.1.6", commit: "abc", serverId: FAKE_SERVER_ID, url: `http://127.0.0.1:${port}`, token, startedServer: true, ...extra })}\nWHITEBOARD-REMOTE-END\n`; + +export const probeOutput = (probe: Partial> = {}) => + `${REVIEW_REMOTE_PROBE_BEGIN}\n${JSON.stringify({ + os: "Linux", + arch: "aarch64", + glibc: "2.35", + home: "/home/dev", + root: "/home/dev/.dev/whiteboard-remote", + homeWritable: true, + freeBytes: 50e9, + node: null, + npm: null, + installed: [], + managedNode: null, + downloader: "curl", + registryReachable: true, + tools: ["tar", "xz", "sha256sum", "sha512sum"], + ...probe, + })}\n${REVIEW_REMOTE_PROBE_END}\n${probe.pathCli ? `${REVIEW_REMOTE_PROBE_PATH_CLI} ${JSON.stringify(probe.pathCli)}\n` : ""}`; + +export function fakeClock() { + let time = 0; + let pending: { at: number; run(): void }[] = []; + const clock: ReviewRemoteClock & { next(): boolean; advance(ms: number): void; readonly pending: number; delays: number[] } = { + delays: [], + now: () => time, + schedule(ms, run) { + clock.delays.push(ms); + const entry = { at: time + ms, run }; + pending.push(entry); + return () => void (pending = pending.filter((e) => e !== entry)); + }, + next() { + pending.sort((a, b) => a.at - b.at); + const entry = pending.shift(); + if (!entry) return false; + time = Math.max(time, entry.at); + entry.run(); + return true; + }, + advance(ms) { + time += ms; + }, + get pending() { + return pending.length; + }, + }; + return clock; +} + +export function fakeSsh(remotes: Record, clock?: { now(): number }) { + const calls: FakeCall[] = []; + const masters = new Map(); + const up = new Set(); + + const spawn: SpawnSsh = (args) => { + const child = new FakeChild(); + const alias = args[args.indexOf("--") + 1]; + const remote = remotes[alias] ?? {}; + const operation = args.includes("-O") ? args[args.indexOf("-O") + 1] : undefined; + const kind: FakeCall["kind"] = args.includes("-M") ? "master" : args.at(-1) === "-s" ? "exec" : (operation as FakeCall["kind"]); + const entry: FakeCall = { alias, kind, args, at: clock?.now() ?? Date.now(), wall: Date.now() }; + calls.push(entry); + const master = masters.get(alias); + setImmediate(() => { + if (kind === "master") { + masters.set(alias, child); + const how = typeof remote.master === "function" ? remote.master(alias) : (remote.master ?? "up"); + if (how === "up") { + up.add(child); + if (remote.masterStderr) child.stderr.write(remote.masterStderr); + } + else if (how === "missing") child.emit("error", Object.assign(new Error("spawn ssh ENOENT"), { code: "ENOENT" })); + else if (how !== "hang") child.finish(how.code, { stderr: how.stderr }); + } else if (kind === "check") { + const code = master && master.alive && up.has(master) ? 0 : 255; + const output = master?.alive ? {} : { stderr: "Control socket connect: No such file or directory\n" }; + const call = calls.filter((c) => c.alias === alias && c.kind === "check").length; + void Promise.resolve(remote.checkAnswered?.(call)).then(() => child.finish(code, output)); + } else if (kind === "exec") { + const answer = () => { + entry.input = child.input; + if (child.input.includes(REVIEW_REMOTE_PROBE_BEGIN)) { + entry.kind = "probe"; + return child.finish(0, { stdout: probeOutput(remote.probe) }); + } + if (child.input.includes("'connect' '--detect'")) { + entry.kind = "detect"; + const answer = remote.detect ?? { code: 0, stdout: detectOutput([]) }; + return child.finish(answer.code, answer); + } + if (child.input.includes("'connect' '--yes'")) { + entry.kind = "connect"; + const answer = remote.connect ?? { code: 0, stdout: connectedOutput(child.input) }; + return child.finish(answer.code, answer); + } + if (child.input.includes(" LISTED\\n")) { + entry.kind = "uninstall"; + return child.finish(0, { stdout: `${REVIEW_REMOTE_INSTALL_SAY} ROOT /home/dev/.dev/whiteboard-remote\n${REVIEW_REMOTE_INSTALL_SAY} HAVE 0.1.6\n${REVIEW_REMOTE_INSTALL_SAY} LISTED\n` }); + } + if (child.input.includes("remote uninstall")) { + entry.kind = "uninstall"; + const done = () => child.finish(0, { stdout: `${JSON.stringify({ event: "remote.uninstall", ok: remote.uninstall?.ok ?? true, reason: "refused" })}\n` }); + return void (remote.uninstall?.after ?? Promise.resolve()).then(done); + } + const call = calls.filter((c) => c.alias === alias && c.kind === "exec").length; + const attach = + typeof remote.attach === "function" + ? remote.attach(call) + : (remote.attach ?? { code: 0, stdout: attachOutput(remote.remotePort ?? 41234) }); + child.finish(attach.code, attach); + }; + if (child.stdin.writableFinished) answer(); + else child.stdin.once("finish", answer); + } else if (kind === "exit") { + const listening = master !== undefined && master.alive && up.has(master); + child.finish(listening ? 0 : 255, { stderr: listening ? "Exit request sent.\n" : "Control socket connect: No such file or directory\n" }); + if (listening) setTimeout(() => master.finish(255), remote.exitDelayMs ?? 0); + } else child.finish(master?.alive ? 0 : 255); + }); + return child as unknown as SshChildProcess; + }; + + return { + spawn, + calls, + of: (alias: string, kind?: FakeCall["kind"]) => calls.filter((c) => c.alias === alias && (!kind || c.kind === kind)), + master: (alias: string) => masters.get(alias), + wedge: (alias: string) => void up.delete(masters.get(alias)!), + alive: () => [...masters.values()].filter((m) => m.alive).length, + }; +} + +export async function until(condition: () => boolean, ms = 3000) { + const end = Date.now() + ms; + while (!condition()) { + if (Date.now() > end) throw new Error("timed out waiting"); + await new Promise((resolve) => setTimeout(resolve, 2)); + } +} diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/reviewDesktopChannel.test.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/reviewDesktopChannel.test.ts index 3970fcf34..5bf1c2c2e 100644 --- a/apps/review-desktop/code-oss/src/vs/review/electron-main/reviewDesktopChannel.test.ts +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/reviewDesktopChannel.test.ts @@ -6,6 +6,9 @@ import assert from "node:assert/strict"; import test from "node:test"; +import { REVIEW_REMOTE_INSTALL_ANSWER_CALL, REVIEW_REMOTE_INSTALL_PROMPT_EVENT } from "../common/reviewRemoteInstallPrompt.js"; +import { REVIEW_SSH_ANSWER_CALL, REVIEW_SSH_PROMPT_EVENT, type ReviewSshPromptEvent } from "../common/reviewSshPrompt.js"; +import { ReviewSshPromptRelay } from "./remote/reviewSshPromptRelay.js"; import { ReviewDesktopChannel } from "./reviewDesktopChannel.js"; const review = "0199a3f2-7c1e-7d4a-9b2f-3e5d6c7b8a90"; @@ -45,3 +48,65 @@ test("matches a review by the storage segment the host names its directory with" assert.deepEqual(closed, ["shared"]); }); + +test("relays ssh prompts to the window and takes its answer", async () => { + const relay = new ReviewSshPromptRelay(); + const channel = new ReviewDesktopChannel({} as never, { getWindows: () => [] } as never, () => false, relay); + const events: ReviewSshPromptEvent[] = []; + channel.listen("", REVIEW_SSH_PROMPT_EVENT)((event) => events.push(event)); + + const answer = relay.prompt({ alias: "wb-test-a", text: "dev@127.0.0.1's password: ", kind: "secret" }); + await channel.call("", REVIEW_SSH_ANSWER_CALL, { id: events[0].id, answer: "hunter2" }); + + assert.equal(await answer, "hunter2"); + relay.dispose(); +}); + +test("reads and connects a remote host's agents through the host, refusing malformed calls", async () => { + const calls: unknown[] = []; + const host = { + detectRemoteAgents: async (alias: string) => (calls.push(["detect", alias]), alias === "gpu" ? [{ id: "pi", connected: false }] : undefined), + connectRemoteAgents: async (alias: string, ids: unknown[]) => (calls.push(["connect", alias, ids]), [{ id: "pi", connected: true, output: "" }]), + }; + const channel = new ReviewDesktopChannel(host as never, { getWindows: () => [] } as never, () => false); + + assert.deepEqual(await channel.call("", "detectRemoteAgents", "gpu"), [{ id: "pi", connected: false }]); + assert.equal(await channel.call("", "detectRemoteAgents", "devbox"), null); + assert.deepEqual(await channel.call("", "connectRemoteAgents", { alias: "gpu", agents: ["pi"] }), [{ id: "pi", connected: true, output: "" }]); + for (const bad of [undefined, "gpu", { alias: 7, agents: ["pi"] }, { alias: "gpu", agents: "pi" }, { alias: "gpu", agents: Array(17).fill("pi") }]) { + await assert.rejects(channel.call("", "connectRemoteAgents", bad), /Unknown agents/); + } + await assert.rejects(channel.call("", "detectRemoteAgents", 7), /Unknown agents/); + + assert.deepEqual(calls, [["detect", "gpu"], ["detect", "devbox"], ["connect", "gpu", ["pi"]]]); +}); + +test("relays install prompts apart from ssh prompts", async () => { + const ssh = new ReviewSshPromptRelay(); + const install = new ReviewSshPromptRelay(); + const channel = new ReviewDesktopChannel({} as never, { getWindows: () => [] } as never, () => false, ssh, install); + const events: ReviewSshPromptEvent[] = []; + const sshEvents: ReviewSshPromptEvent[] = []; + channel.listen("", REVIEW_REMOTE_INSTALL_PROMPT_EVENT)((event) => events.push(event)); + channel.listen("", REVIEW_SSH_PROMPT_EVENT)((event) => sshEvents.push(event)); + + const answer = install.prompt({ alias: "box", text: "Whiteboard 0.1.6 is not installed on box.", kind: "confirm" }); + await channel.call("", REVIEW_REMOTE_INSTALL_ANSWER_CALL, { id: events[0].id, answer: "install" }); + + assert.equal(await answer, "install"); + assert.deepEqual(sshEvents, []); + ssh.dispose(); + install.dispose(); +}); + +test("hands a window a remote machine's VS Code server by its server id, and nothing for anything else", async () => { + const asked: string[] = []; + const endpoint = { host: "127.0.0.1", port: 50123, connectionToken: "vscode-token" }; + const host = { getRemoteLanguageEndpoint: async (serverId: string) => (asked.push(serverId), serverId === "s1" ? endpoint : undefined) }; + const channel = new ReviewDesktopChannel(host as never, { getWindows: () => [] } as never, () => false); + + assert.deepEqual(await channel.call("window", "getRemoteLanguageEndpoint", "s1"), endpoint); + assert.equal(await channel.call("window", "getRemoteLanguageEndpoint", "s2"), undefined); + assert.equal(await channel.call("window", "getRemoteLanguageEndpoint", { serverId: "s1" }), undefined); + assert.deepEqual(asked, ["s1", "s2"]); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/reviewDesktopChannel.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/reviewDesktopChannel.ts index 2158e8f00..2f2a59eda 100644 --- a/apps/review-desktop/code-oss/src/vs/review/electron-main/reviewDesktopChannel.ts +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/reviewDesktopChannel.ts @@ -7,6 +7,13 @@ import { Event } from "../../base/common/event.js"; import { IServerChannel } from "../../base/parts/ipc/common/ipc.js"; import type { IWindowsMainService } from "../../platform/windows/electron-main/windows.js"; import type { ReviewDesktopConnection } from "../common/reviewDesktopBootstrap.js"; +import { REVIEW_REMOTE_INSTALL_ANSWER_CALL, REVIEW_REMOTE_INSTALL_PROMPT_EVENT } from "../common/reviewRemoteInstallPrompt.js"; +import { REVIEW_SSH_ANSWER_CALL, REVIEW_SSH_PROMPT_EVENT } from "../common/reviewSshPrompt.js"; +import { + reviewRemoteInstallPromptRelay, + reviewSshPromptRelay, + type ReviewSshPromptRelay, +} from "./remote/reviewSshPromptRelay.js"; import type { ReviewDesktopHost } from "./reviewDesktopHost.js"; export { REVIEW_DESKTOP_CHANNEL } from "../common/reviewDesktopBootstrap.js"; @@ -21,9 +28,13 @@ export class ReviewDesktopChannel implements IServerChannel { private readonly host: ReviewDesktopHost, private readonly windows: IWindowsMainService, private readonly moveToApplications: () => boolean, + private readonly sshPrompts: ReviewSshPromptRelay = reviewSshPromptRelay, + private readonly installPrompts: ReviewSshPromptRelay = reviewRemoteInstallPromptRelay, ) {} - listen(): Event { + listen(_context: string, event: string): Event { + if (event === REVIEW_SSH_PROMPT_EVENT) return this.sshPrompts.onPrompt as Event; + if (event === REVIEW_REMOTE_INSTALL_PROMPT_EVENT) return this.installPrompts.onPrompt as Event; return Event.None as Event; } @@ -43,6 +54,45 @@ export class ReviewDesktopChannel implements IServerChannel { this.closeSourceWindows(Array.isArray(arg) ? arg.map(String) : []); return undefined as T; } + if (command === "listSshAliases") { + return (await this.host.listSshAliases()) as T; + } + if (command === "getRemoteLanguageEndpoint") { + if (typeof arg !== "string") return undefined as T; + return (await this.host.getRemoteLanguageEndpoint(arg)) as T; + } + if (command === "getRemoteHostState") { + if (typeof arg !== "string") return undefined as T; + return (await this.host.getRemoteHostState(arg)) as T; + } + if (command === "retryRemoteHost") { + if (typeof arg === "string") this.host.retryRemoteHost(arg); + return undefined as T; + } + if (command === "detectRemoteAgents") { + if (typeof arg !== "string") throw new Error("Unknown agents request."); + return ((await this.host.detectRemoteAgents(arg)) ?? null) as T; + } + if (command === "connectRemoteAgents") { + const { alias, agents } = (arg ?? {}) as { alias?: unknown; agents?: unknown }; + if (typeof alias !== "string" || !Array.isArray(agents) || agents.length > 16) throw new Error("Unknown agents request."); + return (await this.host.connectRemoteAgents(alias, agents)) as T; + } + if (command === "installRemoteHost") { + if (typeof arg === "string") await this.host.installRemoteHost(arg); + return undefined as T; + } + if (command === "uninstallRemoteHost") { + if (typeof arg !== "string") throw new Error("uninstallRemoteHost needs an alias."); + return (await this.host.uninstallRemoteHost(arg)) as T; + } + if (command === REVIEW_SSH_ANSWER_CALL || command === REVIEW_REMOTE_INSTALL_ANSWER_CALL) { + const { id, answer } = (arg ?? {}) as { id?: unknown; answer?: unknown }; + const relay = command === REVIEW_SSH_ANSWER_CALL ? this.sshPrompts : this.installPrompts; + if (typeof id === "number") + relay.answer(id, typeof answer === "string" ? answer : undefined); + return undefined as T; + } throw new Error(`Unknown Review Desktop channel call: ${command}`); } diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/reviewDesktopHost.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/reviewDesktopHost.ts index e63b15f9b..5a0e58726 100644 --- a/apps/review-desktop/code-oss/src/vs/review/electron-main/reviewDesktopHost.ts +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/reviewDesktopHost.ts @@ -3,12 +3,17 @@ * Licensed under the MIT License. See LICENSE in the repository root for license information. *--------------------------------------------------------------------------------------------*/ -import { app, BrowserWindow } from "electron"; +import { spawn } from "node:child_process"; +import { app, BrowserWindow, powerMonitor } from "electron"; import { Disposable, toDisposable } from "../../base/common/lifecycle.js"; import { join } from "../../base/common/path.js"; import { IConfigurationService } from "../../platform/configuration/common/configuration.js"; import { IEnvironmentMainService } from "../../platform/environment/electron-main/environmentMainService.js"; -import { ILifecycleMainService } from "../../platform/lifecycle/electron-main/lifecycleMainService.js"; +import { IInstantiationService } from "../../platform/instantiation/common/instantiation.js"; +import { + ILifecycleMainService, + LifecycleMainPhase, +} from "../../platform/lifecycle/electron-main/lifecycleMainService.js"; import { ILogService } from "../../platform/log/common/log.js"; import { IProductService } from "../../platform/product/common/productService.js"; import { getResolvedShellEnv } from "../../platform/shell/node/shellEnv.js"; @@ -16,12 +21,55 @@ import { IApplicationStorageMainService } from "../../platform/storage/electron- import { NullTelemetryService } from "../../platform/telemetry/common/telemetryUtils.js"; import { IUpdateService } from "../../platform/update/common/update.js"; import { UtilityProcess } from "../../platform/utilityProcess/electron-main/utilityProcess.js"; +import { IWindowsMainService } from "../../platform/windows/electron-main/windows.js"; import type { ReviewDesktopConnection } from "../common/reviewDesktopBootstrap.js"; -import { REVIEW_TELEMETRY_SETTING } from "../common/reviewConfigurationDefaults.js"; +import type { + ReviewGatewayHostState, + ReviewRemoteAgent, + ReviewRemoteAgentResult, +} from "../common/reviewProtocol.js"; +import { + REVIEW_REMOTE_HOSTS_ENABLED_SETTING, + REVIEW_REMOTE_HOSTS_SETTING, + REVIEW_REMOTE_INSTALL_MODES, + REVIEW_REMOTE_INSTALL_SETTING, + REVIEW_TELEMETRY_SETTING, +} from "../common/reviewConfigurationDefaults.js"; +import { + REVIEW_REMOTE_INSTALL_NO, + REVIEW_REMOTE_INSTALL_YES, +} from "../common/reviewRemoteInstallPrompt.js"; +import { remoteHostAliases } from "../common/reviewSshAlias.js"; import { REVIEW_CRASH_DUMPS_DIRNAME } from "../node/reviewCrashReporter.js"; import { ReviewCrashDumps } from "./reviewCrashDumps.js"; import { ReviewCrashTelemetry } from "./reviewCrashTelemetry.js"; import { ReviewMainErrorTelemetry } from "./reviewMainErrorTelemetry.js"; +import { reviewEnabledExtensionGroups } from "./remote/reviewEnabledExtensionGroups.js"; +import { + remoteArtifacts, + remotePackageIntegrity, + reviewRemoteCacheDirectory, +} from "./remote/reviewRemoteArtifacts.js"; +import type { + ReviewRemoteInstallFlow, + ReviewRemoteInstallMode, +} from "./remote/reviewRemoteHost.js"; +import { closeRemoteHostWindows, ReviewRemoteHosts } from "./remote/reviewRemoteHosts.js"; +import { + openRemoteInstallConsent, + reviewRemoteInstallConsentPath, +} from "./remote/reviewRemoteInstallConsent.js"; +import { installRemote } from "./remote/reviewRemoteInstaller.js"; +import { createSshAskpass } from "./remote/reviewSshAskpass.js"; +import { + reviewSshConfigPath, + reviewSshControlDirectory, +} from "./remote/reviewSshCommand.js"; +import { listSshAliases } from "./remote/reviewSshConfigAliases.js"; +import { + reviewRemoteInstallPromptRelay, + reviewSshPromptRelay, +} from "./remote/reviewSshPromptRelay.js"; import { ReviewServerSupervisor } from "./reviewServerSupervisor.js"; import { darwinShipItLogPath, @@ -35,7 +83,9 @@ import { */ export class ReviewDesktopHost extends Disposable { private readonly supervisor: ReviewServerSupervisor; + private remoteHosts: ReviewRemoteHosts | undefined; private terminating = false; + private disposed = false; private readonly onTerminationSignal = () => { if (this.terminating) return; @@ -55,9 +105,18 @@ export class ReviewDesktopHost extends Disposable { @IUpdateService private readonly updateService: IUpdateService, @IApplicationStorageMainService private readonly applicationStorageMainService: IApplicationStorageMainService, + @IInstantiationService + private readonly instantiationService: IInstantiationService, ) { super(); let resolvedEnvironment: Promise | undefined; + const shellEnvironment = () => + (resolvedEnvironment ??= getResolvedShellEnv( + this.configurationService, + this.logService, + this.environmentMainService.args, + process.env, + )); let crashTelemetry: ReviewCrashTelemetry | undefined; let errorTelemetry: ReviewMainErrorTelemetry | undefined; const crashDumpsDir = join( @@ -79,13 +138,7 @@ export class ReviewDesktopHost extends Disposable { appUrlProtocol: this.productService.urlProtocol, releaseChannel: this.productService.quality, serverEntryOverride: process.env["DEV_FAST_REVIEW_SERVER_ENTRY"], - resolveEnvironment: () => - (resolvedEnvironment ??= getResolvedShellEnv( - this.configurationService, - this.logService, - this.environmentMainService.args, - process.env, - )), + resolveEnvironment: shellEnvironment, logInfo: (message) => this.logService.info(message), logError: (message) => this.logService.error(message), createProcess: () => @@ -103,6 +156,7 @@ export class ReviewDesktopHost extends Disposable { crashTelemetry?.reportServerExit(detail); }, onServerReady: () => errorTelemetry?.serverReady(), + onRemoteHostRestarted: (alias) => this.remoteHosts?.reattach(alias), }), ); this._register( @@ -118,8 +172,13 @@ export class ReviewDesktopHost extends Disposable { this._register( this.lifecycleMainService.onWillShutdown((event) => { event.join("reviewDesktopHost", this.supervisor.stop()); + if (this.remoteHosts) + event.join("reviewRemoteHosts", this.remoteHosts.dispose()); }), ); + void this.lifecycleMainService + .when(LifecycleMainPhase.AfterWindowOpen) + .then(() => this.startRemoteHosts(shellEnvironment)); // Main-process errors report through the embedded server, so they pass the // same opt-out checks and the same redaction step as every other event. errorTelemetry = new ReviewMainErrorTelemetry({ @@ -193,7 +252,185 @@ export class ReviewDesktopHost extends Disposable { this.supervisor.stageRustAnalyzer(); } + listSshAliases(): Promise { + return listSshAliases(reviewSshConfigPath()); + } + + retryRemoteHost(alias: string): void { + this.remoteHosts?.retry(alias); + } + + async getRemoteLanguageEndpoint(serverId: string) { + const manager = this.remoteHosts; + const states = manager && (await this.remoteHostStates()); + return states && manager.languageEndpoint(serverId, states).catch(() => undefined); + } + + async getRemoteHostState(serverId: string) { + const manager = this.remoteHosts; + const states = manager && (await this.remoteHostStates()); + return states && manager.hostState(serverId, states); + } + + private async remoteHostStates(): Promise { + try { + const { url, token } = await this.whenConnected(); + const response = await fetch(new URL("/remote-hosts", url), { + headers: { "x-review-token": token }, + }); + return response.ok ? ((await response.json()) as ReviewGatewayHostState[]) : undefined; + } catch { + return undefined; + } + } + + installRemoteHost(alias: string): Promise { + return this.remoteHosts?.install(alias) ?? Promise.resolve(); + } + + detectRemoteAgents(alias: string): Promise { + return this.remoteHosts?.detectAgents(alias) ?? Promise.resolve(undefined); + } + + connectRemoteAgents(alias: string, ids: readonly unknown[]): Promise { + if (!this.remoteHosts) return Promise.reject(new Error("Remote hosts are off.")); + return this.remoteHosts.connectAgents(alias, ids); + } + + async uninstallRemoteHost(alias: string): Promise { + if (!this.remoteHosts) return "Remote hosts are off."; + try { + await this.remoteHosts.uninstall(alias); + return null; + } catch (error) { + return error instanceof Error ? error.message : String(error); + } + } + + private remoteInstallFlow(): ReviewRemoteInstallFlow { + const { userDataPath, isBuilt, appRoot } = this.environmentMainService; + const cacheDirectory = reviewRemoteCacheDirectory(userDataPath); + const pin = this.productService.whiteboardRemote; + const artifactsOptions = { + pin, + checkout: isBuilt ? undefined : join(appRoot, "..", "..", ".."), + cacheDirectory, + }; + return { + mode: () => { + const mode = this.configurationService.getValue( + REVIEW_REMOTE_INSTALL_SETTING, + ); + return REVIEW_REMOTE_INSTALL_MODES.includes( + mode as ReviewRemoteInstallMode, + ) + ? (mode as ReviewRemoteInstallMode) + : "ask"; + }, + consent: openRemoteInstallConsent( + reviewRemoteInstallConsentPath(userDataPath), + ), + confirm: async (request) => { + const answer = await reviewRemoteInstallPromptRelay.prompt({ + ...request, + kind: "confirm", + }); + return answer === REVIEW_REMOTE_INSTALL_YES + ? true + : answer === REVIEW_REMOTE_INSTALL_NO + ? false + : undefined; + }, + run: async (input) => { + const artifacts = await remoteArtifacts(input.target, artifactsOptions); + return installRemote({ + ...input, + artifacts, + published: pin !== undefined, + cacheDirectory, + }); + }, + integrity: () => remotePackageIntegrity(artifactsOptions), + }; + } + + private startRemoteHosts( + shellEnvironment: () => Promise, + ): void { + if (this.disposed) return; + let version: Promise | undefined; + const manager = new ReviewRemoteHosts({ + spawn: (args, options) => spawn("ssh", args, options), + controlDirectory: reviewSshControlDirectory(), + instance: this.environmentMainService.userDataPath, + environment: async () => ({ + ...process.env, + ...(await shellEnvironment().catch(() => ({}))), + }), + createAskpass: (input) => createSshAskpass(input), + prompt: (request) => reviewSshPromptRelay.prompt(request), + desktopVersion: () => (version ??= this.desktopVersion()), + desktopCommit: this.productService.commit, + groups: () => + reviewEnabledExtensionGroups(this.environmentMainService.extensionsPath), + send: (hosts) => this.supervisor.setRemoteHosts(hosts), + log: (message) => this.logService.info(`[Remote hosts] ${message}`), + install: this.remoteInstallFlow(), + removed: (serverId) => + this.instantiationService.invokeFunction((accessor) => + closeRemoteHostWindows(accessor.get(IWindowsMainService), serverId), + ), + }); + this.remoteHosts = manager; + const update = () => { + manager.update( + this.configurationService.getValue( + REVIEW_REMOTE_HOSTS_ENABLED_SETTING, + ) === true, + remoteHostAliases( + this.configurationService.getValue(REVIEW_REMOTE_HOSTS_SETTING), + ), + ); + }; + const onResume = () => manager.resume(); + const onExit = () => manager.killNow(); + powerMonitor.on("resume", onResume); + process.once("exit", onExit); + this._register( + toDisposable(() => { + powerMonitor.off("resume", onResume); + process.off("exit", onExit); + void manager.dispose(); + }), + ); + this._register( + this.configurationService.onDidChangeConfiguration((event) => { + if ( + event.affectsConfiguration(REVIEW_REMOTE_HOSTS_SETTING) || + event.affectsConfiguration(REVIEW_REMOTE_HOSTS_ENABLED_SETTING) || + event.affectsConfiguration(REVIEW_REMOTE_INSTALL_SETTING) + ) + update(); + }), + ); + update(); + } + + private async desktopVersion(): Promise { + const fallback = + this.productService.reviewVersion ?? this.productService.version; + try { + const { url } = await this.whenConnected(); + const response = await fetch(new URL("/health", url)); + const { version } = (await response.json()) as { version?: unknown }; + return typeof version === "string" ? version : fallback; + } catch { + return fallback; + } + } + override dispose(): void { + this.disposed = true; process.off("SIGINT", this.onTerminationSignal); process.off("SIGTERM", this.onTerminationSignal); super.dispose(); diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/reviewServerSupervisor.test.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/reviewServerSupervisor.test.ts index 1a9c90c62..a3ef4d4b6 100644 --- a/apps/review-desktop/code-oss/src/vs/review/electron-main/reviewServerSupervisor.test.ts +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/reviewServerSupervisor.test.ts @@ -93,6 +93,8 @@ class FakeServerProcess implements IReviewServerProcess { readonly onExit = this.exit.event; private readonly crashed = new Emitter<{ readonly code: number; readonly reason: string }>(); readonly onCrash = this.crashed.event; + readonly message = new Emitter(); + readonly onMessage = this.message.event; env: Record = {}; start(configuration: { readonly env?: Record }): boolean { @@ -117,7 +119,7 @@ class FakeServerProcess implements IReviewServerProcess { exitWith(code: number, signal: string): void { this.exit.fire({ code, signal }); } - postMessage(): void { } + postMessage(_message?: unknown): void { } kill(): void { } dispose(): void { } } @@ -225,6 +227,42 @@ test('calls onServerReady for the first server and for each restarted one', asyn assert.equal(ready, 2); }); +test('sends the remote hosts to a server once it is ready, and again after a restart', async (t) => { + class RecordingServerProcess extends FakeServerProcess { + readonly messages: unknown[] = []; + override postMessage(message?: unknown): void { this.messages.push(message); } + } + const processes: RecordingServerProcess[] = []; + let restarted!: () => void; + const whenRestarted = new Promise((resolve) => restarted = resolve); + const supervisor = new ReviewServerSupervisor({ + appRoot: '/app', + appVersion: '0.0.34', + isBuilt: true, + channel: 'stable', + logInfo: () => { }, + logError: () => { }, + createProcess: () => { + const serverProcess = new RecordingServerProcess(); + processes.push(serverProcess); + if (processes.length === 2) queueMicrotask(restarted); + return serverProcess; + }, + }); + t.after(() => supervisor.dispose()); + const hosts = [{ alias: 'devbox', endpoint: { url: 'http://127.0.0.1:41000', token: 'remote' } }]; + + supervisor.start(); + supervisor.setRemoteHosts(hosts); + assert.deepEqual(processes[0].messages, []); + processes[0].announceReady(); + assert.deepEqual(processes[0].messages, [{ type: 'remote-hosts', hosts }]); + processes[0].crash(); + await whenRestarted; + processes[1].announceReady(); + assert.deepEqual(processes[1].messages, [{ type: 'remote-hosts', hosts }]); +}); + test('the app path names the macOS bundle, else the executable', () => { assert.equal(applicationPath('/Applications/Review.app/Contents/MacOS/Review'), '/Applications/Review.app'); assert.equal(applicationPath('/usr/share/review-desktop/review-desktop'), '/usr/share/review-desktop/review-desktop'); diff --git a/apps/review-desktop/code-oss/src/vs/review/electron-main/reviewServerSupervisor.ts b/apps/review-desktop/code-oss/src/vs/review/electron-main/reviewServerSupervisor.ts index 7de1a0167..dd282d85f 100644 --- a/apps/review-desktop/code-oss/src/vs/review/electron-main/reviewServerSupervisor.ts +++ b/apps/review-desktop/code-oss/src/vs/review/electron-main/reviewServerSupervisor.ts @@ -21,6 +21,7 @@ import { type ReviewServerAnnouncement, resolveReviewServerEntry, } from "../common/reviewDesktopBootstrap.js"; +import type { ReviewGatewayHost } from "../common/reviewProtocol.js"; import { REVIEW_SERVER_RESTART_DELAYS, REVIEW_SERVER_STARTUP_TIMEOUT_MS, @@ -37,6 +38,7 @@ export interface IReviewServerProcess extends IDisposable { readonly onStderr: Event; readonly onExit: Event<{ readonly code: number; readonly signal: string }>; readonly onCrash: Event<{ readonly code: number; readonly reason: string }>; + readonly onMessage: Event; start(configuration: { readonly type: string; readonly name: string; @@ -86,6 +88,7 @@ export interface ReviewServerSupervisorOptions { readonly onServerTerminated?: (detail: ReviewServerTermination) => void; /** Called every time a server, first or restarted, announces its endpoint. */ readonly onServerReady?: () => void; + readonly onRemoteHostRestarted?: (alias: string) => void; } export function createReviewServerEnvironment(options: { @@ -213,6 +216,8 @@ export class ReviewServerSupervisor extends Disposable { private restartCount = 0; private stopping = false; private telemetryEnabled: boolean; + private readyProcess: IReviewServerProcess | undefined; + private remoteHosts: ReviewGatewayHost[] | undefined; /** * Credentials are minted once and reused for the life of the application, so @@ -246,6 +251,11 @@ export class ReviewServerSupervisor extends Disposable { this.serverProcess?.postMessage({ type: "telemetry-setting", enabled }); } + setRemoteHosts(hosts: ReviewGatewayHost[]): void { + this.remoteHosts = hosts; + this.readyProcess?.postMessage({ type: "remote-hosts", hosts }); + } + stageRustAnalyzer(): void { const sourcePath = resolveRustAnalyzerSource(this.options); if (!sourcePath) { @@ -330,6 +340,9 @@ export class ReviewServerSupervisor extends Disposable { appSessionId: this.appSessionId, }; ready = true; + this.readyProcess = serverProcess; + if (this.remoteHosts) + serverProcess.postMessage({ type: "remote-hosts", hosts: this.remoteHosts }); this.readyTimer.cancel(); this.port = Number(new URL(connection.url).port); this.restartCount = 0; @@ -342,6 +355,13 @@ export class ReviewServerSupervisor extends Disposable { this.options.onServerReady?.(); }), ); + this.processListeners.add( + serverProcess.onMessage((message) => { + const { type, alias } = (message ?? {}) as { type?: unknown; alias?: unknown }; + if (type === "remote-host-restarted" && typeof alias === "string") + this.options.onRemoteHostRestarted?.(alias); + }), + ); this.processListeners.add( serverProcess.onStderr((value) => this.options.logError(`[Review server] ${value.trimEnd()}`), @@ -356,6 +376,7 @@ export class ReviewServerSupervisor extends Disposable { `[Review Desktop] server host terminated: ${detail.reason}`, ); this.serverProcess = undefined; + if (this.readyProcess === serverProcess) this.readyProcess = undefined; this.processListeners.dispose(); if (this.stopping) return; if (died) this.options.onServerTerminated?.(detail); diff --git a/apps/review-desktop/code-oss/src/vs/review/navigator.desktop.main.ts b/apps/review-desktop/code-oss/src/vs/review/navigator.desktop.main.ts index d4bbff646..e92118933 100644 --- a/apps/review-desktop/code-oss/src/vs/review/navigator.desktop.main.ts +++ b/apps/review-desktop/code-oss/src/vs/review/navigator.desktop.main.ts @@ -8,7 +8,7 @@ // own entry point; the editor/extension-host services are shared. import './editor.common.main.js'; import './editor.desktop.main.js'; -import { reviewConfigurationDefaults } from './common/reviewConfigurationDefaults.js'; +import { reviewConfigurationDefaults, reviewSourceWindowDefaults } from './common/reviewConfigurationDefaults.js'; import '../workbench/browser/workbench.zenMode.contribution.js'; import '../workbench/browser/actions/layoutActions.js'; import '../workbench/browser/parts/editor/editorParts.js'; @@ -47,6 +47,36 @@ import { ILanguageFeaturesService } from '../editor/common/services/languageFeat import { SymbolNavigationAnchor } from '../editor/contrib/gotoSymbol/browser/goToCommands.js'; import { CommandsRegistry, ICommandService } from '../platform/commands/common/commands.js'; import { IEditorService } from '../workbench/services/editor/common/editorService.js'; +import { NativeExtensionService } from '../workbench/services/extensions/electron-browser/nativeExtensionService.js'; +import { IExtensionService } from '../workbench/services/extensions/common/extensions.js'; +import { ReviewExtensionGalleryManifestService, ReviewExtensionManagementServerService } from './services/remote/reviewRemoteWindowExtensionManagement.js'; +import { IExtensionManagementServerService } from '../workbench/services/extensionManagement/common/extensionManagement.js'; +import { IExtensionGalleryManifestService } from '../platform/extensionManagement/common/extensionGalleryManifest.js'; +import { IRemoteAuthorityResolverService, type ResolverResult } from '../platform/remote/common/remoteAuthorityResolver.js'; +import { IMainProcessService } from '../platform/ipc/common/mainProcessService.js'; +import { REVIEW_DESKTOP_CHANNEL } from './common/reviewDesktopBootstrap.js'; +import { isReviewRemoteAuthority } from './services/remote/reviewRemoteAuthority.js'; +import { reloadWhenOnline, reviewWindowAuthorityResolver, reviewWindowHosts } from './services/remote/reviewWindowAuthorityResolver.js'; +import { ReviewSourceWindowHostState } from './services/remote/reviewSourceWindowHostState.js'; +import { IHostService } from '../workbench/services/host/browser/host.js'; +import { DesktopMain } from '../workbench/electron-browser/desktop.main.js'; +import type { INativeWindowConfiguration } from '../platform/window/common/window.js'; +import type { WorkspaceService } from '../workbench/services/configuration/browser/configurationService.js'; +import { isReviewSourceTitle, REVIEW_SOURCE_TITLE_KEY, reviewSourceWindowConfiguration, type ReviewSourceWindowConfiguration } from './services/configuration/reviewSourceWindowConfiguration.js'; + +let sourceWindow: ReviewSourceWindowConfiguration | undefined; + +class NavigatorDesktopMain extends DesktopMain { + protected override async createWorkspaceService(...args: Parameters): Promise { + const [, , , , , , , logService] = args; + sourceWindow = reviewSourceWindowConfiguration(await super.createWorkspaceService(...args), logService); + return sourceWindow.service; + } +} + +export function main(configuration: INativeWindowConfiguration): Promise { + return new NavigatorDesktopMain(configuration).open(); +} class NavigatorDefaults { constructor(@IStorageService storage: IStorageService) { @@ -55,10 +85,45 @@ class NavigatorDefaults { if (storage.get(key, StorageScope.PROFILE) === undefined) { storage.store(key, false, StorageScope.PROFILE, StorageTarget.USER); } + const title = storage.getObject(REVIEW_SOURCE_TITLE_KEY, StorageScope.WORKSPACE); + if (isReviewSourceTitle(title)) sourceWindow?.setTitle(title); } } registerWorkbenchContribution2('review.navigator.defaults', NavigatorDefaults, WorkbenchPhase.BlockStartup); +registerWorkbenchContribution2(ReviewSourceWindowHostState.ID, ReviewSourceWindowHostState, WorkbenchPhase.BlockRestore); + +class NavigatorExtensionService extends NativeExtensionService { + private readonly windowHosts = this._instantiationService.invokeFunction((accessor) => reviewWindowHosts(accessor.get(IMainProcessService).getChannel(REVIEW_DESKTOP_CHANNEL))); + protected override readonly _remoteAuthorityResolverService = this._instantiationService.invokeFunction((accessor) => + reviewWindowAuthorityResolver(accessor.get(IRemoteAuthorityResolverService), this.windowHosts)); + private retrying = false; + + protected override async _resolveAuthority(remoteAuthority: string): Promise { + if (!isReviewRemoteAuthority(remoteAuthority)) return super._resolveAuthority(remoteAuthority); + try { + return await this._remoteAuthorityResolverService.resolveAuthority(remoteAuthority); + } catch (error) { + if (!this.retrying) { + this.retrying = true; + const host = this._instantiationService.invokeFunction((accessor) => accessor.get(IHostService)); + this._register(reloadWhenOnline(this.windowHosts, remoteAuthority.slice('whiteboard+'.length), () => void host.reload())); + } + throw error; + } + } +} + +registerSingleton(IExtensionService, NavigatorExtensionService, InstantiationType.Eager); +registerSingleton(IExtensionManagementServerService, ReviewExtensionManagementServerService, InstantiationType.Delayed); +registerSingleton(IExtensionGalleryManifestService, ReviewExtensionGalleryManifestService, InstantiationType.Eager); + +CommandsRegistry.registerCommand('review.action.setSourceTitle', (accessor, title: unknown) => { + if (!isReviewSourceTitle(title)) return; + const value = { side: title.side, title: title.title, ...(title.alias && { alias: title.alias }) }; + accessor.get(IStorageService).store(REVIEW_SOURCE_TITLE_KEY, value, StorageScope.WORKSPACE, StorageTarget.MACHINE); + sourceWindow?.setTitle(value); +}); CommandsRegistry.registerCommand('review.action.showReferencesInSource', async (accessor, resource: string, lineNumber: number, column: number) => { const editorService = accessor.get(IEditorService); @@ -111,7 +176,6 @@ Registry.as(Extensions.Configuration).registerDefaultCon 'workbench.colorTheme': reviewConfigurationDefaults['workbench.colorTheme'], 'workbench.preferredDarkColorTheme': reviewConfigurationDefaults['workbench.preferredDarkColorTheme'], 'workbench.preferredLightColorTheme': reviewConfigurationDefaults['workbench.preferredLightColorTheme'], + ...reviewSourceWindowDefaults, }, }]); - -export { main } from '../workbench/electron-browser/desktop.main.js'; diff --git a/apps/review-desktop/code-oss/src/vs/review/node/reviewOptionalExtensionCatalog.ts b/apps/review-desktop/code-oss/src/vs/review/node/reviewOptionalExtensionCatalog.ts index 05173fe10..3018267e2 100644 --- a/apps/review-desktop/code-oss/src/vs/review/node/reviewOptionalExtensionCatalog.ts +++ b/apps/review-desktop/code-oss/src/vs/review/node/reviewOptionalExtensionCatalog.ts @@ -32,6 +32,11 @@ export const reviewOptionalExtensionCatalog = [ url: 'https://open-vsx.org/api/rust-lang/rust-analyzer/linux-x64/0.4.2990/file/rust-lang.rust-analyzer-0.4.2990@linux-x64.vsix', sha256: '317cb128e8caf2495b955ef6612d828fef809187ac445242116ad8e2e32382ff', size: 16313907 + }, + 'linux-arm64': { + url: 'https://open-vsx.org/api/rust-lang/rust-analyzer/linux-arm64/0.4.2990/file/rust-lang.rust-analyzer-0.4.2990@linux-arm64.vsix', + sha256: 'f8b326bc52f5bd9af94979fea09224697c431ef0e76f0239cb599a86df1e32b7', + size: 15931320 } } }, @@ -86,6 +91,11 @@ export const reviewOptionalExtensionCatalog = [ url: 'https://open-vsx.org/api/muhammad-sammy/csharp/linux-x64/2.145.21-g154a82fd27/file/muhammad-sammy.csharp-2.145.21-g154a82fd27@linux-x64.vsix', sha256: '78bc006683cc998e9fd1a6f2760d8cb3da63096464a217bbd192ecfb490a5516', size: 78144854 + }, + 'linux-arm64': { + url: 'https://open-vsx.org/api/muhammad-sammy/csharp/linux-arm64/2.145.21-g154a82fd27/file/muhammad-sammy.csharp-2.145.21-g154a82fd27@linux-arm64.vsix', + sha256: '74a50cd78ca1eb45d87eb922b76e6ef3b4d843b4fc8b7b8187a7d2a9583ab532', + size: 75204300 } } }, diff --git a/apps/review-desktop/code-oss/src/vs/review/node/reviewRemoteFailureIsolation.test.ts b/apps/review-desktop/code-oss/src/vs/review/node/reviewRemoteFailureIsolation.test.ts new file mode 100644 index 000000000..4a6b9bc31 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/node/reviewRemoteFailureIsolation.test.ts @@ -0,0 +1,216 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'node:assert/strict'; +import { once } from 'node:events'; +import { AddressInfo, createConnection, createServer, Server, Socket } from 'node:net'; +import { afterEach, beforeEach, test } from 'node:test'; +import { VSBuffer } from '../../base/common/buffer.js'; +import { Emitter, Event } from '../../base/common/event.js'; +import { ClientConnectionEvent, IPCServer } from '../../base/parts/ipc/common/ipc.js'; +import { PersistentProtocol } from '../../base/parts/ipc/common/ipc.net.js'; +import { NodeSocket } from '../../base/parts/ipc/node/ipc.net.js'; +import { NullLogService } from '../../platform/log/common/log.js'; +import { connectRemoteAgentManagement, IConnectionOptions, ManagementPersistentConnection, PersistentConnection, PersistentConnectionEventType } from '../../platform/remote/common/remoteAgentConnection.js'; +import { RemoteConnectionType, WebSocketRemoteConnection } from '../../platform/remote/common/remoteAuthorityResolver.js'; +import { RemoteSocketFactoryService } from '../../platform/remote/common/remoteSocketFactoryService.js'; +import { ISignService } from '../../platform/sign/common/sign.js'; + +// The client's handshake and reconnect timeouts (10 s, 30 s) are never cleared, +// and its LoadEstimator ticks for the life of the process. Unreference them so +// this file exits when its tests end; open sockets keep it alive until then. +const setTimeoutReferenced = globalThis.setTimeout; +const setIntervalReferenced = globalThis.setInterval; + +const signService: ISignService = { + _serviceBrand: undefined, + createNewMessage: async data => ({ id: data, data }), + validate: async () => true, + sign: async data => data, +}; + +const json = (message: object) => VSBuffer.fromString(JSON.stringify(message)); + +class FakeRemote { + private readonly server: Server = createServer(socket => this.accept(socket)); + private readonly sockets = new Set(); + private readonly clients = new Emitter(); + private readonly ipc = new IPCServer(this.clients.event); + port = 0; + + constructor() { + this.ipc.registerChannel('echo', { call: async (_ctx, _command, arg) => arg, listen: () => Event.None }); + } + + async start(port = 0): Promise { + this.server.listen(port, '127.0.0.1'); + await once(this.server, 'listening'); + this.port = (this.server.address() as AddressInfo).port; + } + + restart(): void { + for (const socket of this.sockets) { + socket.end(); + } + } + + async stop(): Promise { + this.restart(); + this.ipc.dispose(); + await new Promise(resolve => this.server.close(resolve)); + } + + private accept(socket: Socket): void { + this.sockets.add(socket); + socket.once('close', () => this.sockets.delete(socket)); + socket.once('data', (chunk: Buffer) => { + const newline = chunk.indexOf(10); + const query = new URLSearchParams(chunk.subarray(0, newline).toString()); + const rest = chunk.subarray(newline + 1); + const protocol = new PersistentProtocol({ socket: new NodeSocket(socket), initialChunk: rest.length ? VSBuffer.wrap(rest) : null }); + Event.once(protocol.onControlMessage)(() => { + protocol.sendControl(json({ type: 'sign', data: 'data', signedData: 'data' })); + Event.once(protocol.onControlMessage)(() => { + if (query.get('reconnection') === 'true') { + protocol.sendControl(json({ type: 'error', reason: 'Unknown reconnection token (never seen)' })); + return; + } + protocol.sendControl(json({ type: 'ok' })); + this.clients.fire({ protocol, onDidClientDisconnect: Event.None }); + }); + }); + }); + } +} + +function options(remote: FakeRemote): IConnectionOptions { + const remoteSocketFactoryService = new RemoteSocketFactoryService(); + remoteSocketFactoryService.register(RemoteConnectionType.WebSocket, { + supports: () => true, + connect: async (connectTo, _path, query) => { + const socket = createConnection(connectTo.port, connectTo.host); + await once(socket, 'connect'); + socket.write(`${query}\n`); + return new NodeSocket(socket); + }, + }); + return { + commit: undefined, + quality: undefined, + addressProvider: { getAddress: async () => ({ connectTo: new WebSocketRemoteConnection('127.0.0.1', remote.port), connectionToken: undefined }) }, + remoteSocketFactoryService, + signService, + logService: new NullLogService(), + ipcLogger: null, + }; +} + +let remotes: FakeRemote[]; +let connections: ManagementPersistentConnection[]; + +beforeEach(() => { + remotes = []; + connections = []; + globalThis.setTimeout = ((...args: Parameters) => (setTimeoutReferenced(...args) as unknown as NodeJS.Timeout).unref()) as unknown as typeof setTimeout; + globalThis.setInterval = ((...args: Parameters) => (setIntervalReferenced(...args) as unknown as NodeJS.Timeout).unref()) as unknown as typeof setInterval; +}); + +afterEach(async () => { + for (const connection of connections) { + connection.dispose(); + } + await Promise.all(remotes.map(remote => remote.stop())); + globalThis.setTimeout = setTimeoutReferenced; + globalThis.setInterval = setIntervalReferenced; +}); + +async function startRemote(port = 0): Promise { + const remote = new FakeRemote(); + remotes.push(remote); + await remote.start(port); + return remote; +} + +async function connect(remote: FakeRemote, authority: string): Promise { + const connection = await connectRemoteAgentManagement(options(remote), authority, 'renderer'); + connections.push(connection); + return connection; +} + +async function echo(connection: ManagementPersistentConnection, text: string): Promise { + let timer: ReturnType | undefined; + const noReply = new Promise((_, reject) => timer = setTimeoutReferenced(() => reject(new Error(`no reply to ${text}`)), 2_000)); + try { + return await Promise.race([connection.client.getChannel('echo').call('echo', text), noReply]); + } finally { + clearTimeout(timer); + } +} + +async function failPermanently(remote: FakeRemote, connection: ManagementPersistentConnection): Promise { + const failed = Event.toPromise(Event.filter(connection.onDidStateChange, e => e.type === PersistentConnectionEventType.ReconnectionPermanentFailure)); + remote.restart(); + await failed; +} + +test('a permanent reconnection failure of one authority leaves another authority sending and receiving', async () => { + const [a, b] = [await startRemote(), await startRemote()]; + const [toA, toB] = [await connect(a, 'wb-test+a'), await connect(b, 'wb-test+b')]; + assert.equal(await echo(toB, 'b'), 'b'); + + await failPermanently(b, toB); + + assert.equal(await echo(toA, 'a'), 'a'); +}); + +test('a connection that failed permanently does not start reconnecting again', async () => { + const b = await startRemote(); + const toB = await connect(b, 'wb-test+b'); + await failPermanently(b, toB); + const events: PersistentConnectionEventType[] = []; + toB.onDidStateChange(e => events.push(e.type)); + + PersistentConnection.debugTriggerReconnection(); + await new Promise(resolve => setTimeoutReferenced(resolve, 100)); + + assert.deepEqual(events, []); +}); + +test('a failed initial connection to one authority leaves another authority sending and receiving', async () => { + const a = await startRemote(); + const toA = await connect(a, 'wb-test+a'); + const down = await startRemote(); + await down.stop(); + + await assert.rejects(connect(down, 'wb-test+b'), { code: 'ECONNREFUSED' }); + + assert.equal(await echo(toA, 'a'), 'a'); +}); + +test('a new connection to an authority that failed succeeds once its server is back', async () => { + const [a, b] = [await startRemote(), await startRemote()]; + const [toA, toB] = [await connect(a, 'wb-test+a'), await connect(b, 'wb-test+b')]; + await failPermanently(b, toB); + await b.stop(); + await assert.rejects(connect(b, 'wb-test+b'), { code: 'ECONNREFUSED' }); + + const back = await startRemote(b.port); + const again = await connect(back, 'wb-test+b'); + + assert.equal(await echo(again, 'b again'), 'b again'); + assert.equal(await echo(toA, 'a'), 'a'); +}); + +test('a new connection to a third authority succeeds after another authority failed', async () => { + const [a, b, c] = [await startRemote(), await startRemote(), await startRemote()]; + const toA = await connect(a, 'wb-test+a'); + const toB = await connect(b, 'wb-test+b'); + await failPermanently(b, toB); + + const toC = await connect(c, 'wb-test+c'); + + assert.equal(await echo(toC, 'c'), 'c'); + assert.equal(await echo(toA, 'a'), 'a'); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/node/reviewSshAskpassMain.ts b/apps/review-desktop/code-oss/src/vs/review/node/reviewSshAskpassMain.ts new file mode 100644 index 000000000..72c4fbafb --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/node/reviewSshAskpassMain.ts @@ -0,0 +1,34 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { connect } from "node:net"; +import { REVIEW_SSH_ASKPASS_ALIAS_ENV, REVIEW_SSH_ASKPASS_SOCKET_ENV } from "../common/reviewSshPrompt.js"; + +const TIMEOUT_MS = 10 * 60_000; + +const socketPath = process.env[REVIEW_SSH_ASKPASS_SOCKET_ENV]; +const alias = process.env[REVIEW_SSH_ASKPASS_ALIAS_ENV]; +if (!socketPath || !alias) process.exit(1); + +process.exitCode = 1; +const fail = () => process.exit(1); + +const socket = connect(socketPath); +socket.setEncoding("utf8"); +socket.setTimeout(TIMEOUT_MS, fail); +socket.on("error", fail); +socket.on("connect", () => socket.write(`${JSON.stringify({ alias, text: process.argv[2] ?? "", pid: process.ppid })}\n`)); +let reply = ""; +socket.on("data", (chunk: string) => (reply += chunk)); +socket.on("end", () => { + let answer: unknown; + try { + answer = (JSON.parse(reply) as { answer?: unknown }).answer; + } catch { + fail(); + } + if (typeof answer !== "string") fail(); + else process.stdout.write(answer, () => (process.exitCode = 0)); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/review.desktop.main.ts b/apps/review-desktop/code-oss/src/vs/review/review.desktop.main.ts index ac16ba077..6219842f1 100644 --- a/apps/review-desktop/code-oss/src/vs/review/review.desktop.main.ts +++ b/apps/review-desktop/code-oss/src/vs/review/review.desktop.main.ts @@ -6,5 +6,8 @@ import './review.common.main.js'; import './editor.desktop.main.js'; import './contrib/update/reviewUpdate.contribution.js'; +import './contrib/remote/reviewSshPrompt.contribution.js'; +import './services/remote/reviewRemoteHosts.js'; +import './contrib/remote/reviewRemoteInstallPrompt.contribution.js'; export { main } from './electron-browser/review.main.js'; diff --git a/apps/review-desktop/code-oss/src/vs/review/reviewExtensionHostUnsupportedApiPeers.ts b/apps/review-desktop/code-oss/src/vs/review/reviewExtensionHostUnsupportedApiPeers.ts index a733405de..3ba3076dd 100644 --- a/apps/review-desktop/code-oss/src/vs/review/reviewExtensionHostUnsupportedApiPeers.ts +++ b/apps/review-desktop/code-oss/src/vs/review/reviewExtensionHostUnsupportedApiPeers.ts @@ -98,3 +98,48 @@ export class ReviewMainThreadUrls { $unregisterUriHandler(): Promise { return Promise.resolve(); } $createAppUri(uri: UriComponents): Promise { return Promise.resolve(uri); } } + +// A Linux remote extension host calls $setRemoteTunnelService at start-up. +@extHostNamedCustomer(MainContext.MainThreadTunnelService) +export class ReviewMainThreadTunnelService { + constructor(_context: IExtHostContext) { } + dispose(): void { } + $openTunnel(): Promise { return Promise.resolve(undefined); } + $closeTunnel(): Promise { return Promise.resolve(); } + $getTunnels(): Promise { return Promise.resolve([]); } + $setTunnelProvider(): Promise { return Promise.resolve(); } + $hasTunnelProvider(): Promise { return Promise.resolve(false); } + $setRemoteTunnelService(): Promise { return Promise.resolve(); } + $setCandidateFilter(): Promise { return Promise.resolve(); } + $onFoundNewCandidates(): Promise { return Promise.resolve(); } + $setCandidatePortSource(): Promise { return Promise.resolve(); } + $registerPortsAttributesProvider(): Promise { return Promise.resolve(); } + $unregisterPortsAttributesProvider(): Promise { return Promise.resolve(); } +} + +@extHostNamedCustomer(MainContext.MainThreadTesting) +export class ReviewMainThreadTesting { + constructor(_context: IExtHostContext) { } + dispose(): void { } + $registerTestController(): void { } + $updateController(): void { } + $unregisterTestController(): void { } + $subscribeToDiffs(): void { } + $unsubscribeFromDiffs(): void { } + $publishDiff(): void { } + $getCoverageDetails(): Promise { return unavailable('testing'); } + $publishTestRunProfile(): void { } + $updateTestRunConfig(): void { } + $removeTestProfile(): void { } + $runTests(): Promise { return unavailable('testing'); } + $addTestsToRun(): void { } + $updateTestStateInRun(): void { } + $appendTestMessagesInRun(): void { } + $appendOutputToRun(): void { } + $appendCoverage(): void { } + $startedTestRunTask(): void { } + $finishedTestRunTask(): void { } + $startedExtensionTestRun(): void { } + $finishedExtensionTestRun(): void { } + $markTestRetired(): void { } +} diff --git a/apps/review-desktop/code-oss/src/vs/review/services/configuration/reviewSourceWindowConfiguration.test.ts b/apps/review-desktop/code-oss/src/vs/review/services/configuration/reviewSourceWindowConfiguration.test.ts new file mode 100644 index 000000000..fcc9a9270 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/services/configuration/reviewSourceWindowConfiguration.test.ts @@ -0,0 +1,74 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { Emitter } from "../../../base/common/event.js"; +import { URI } from "../../../base/common/uri.js"; +import type { IConfigurationChangeEvent, IConfigurationModel, IConfigurationService } from "../../../platform/configuration/common/configuration.js"; +import { Configuration } from "../../../platform/configuration/common/configurationModels.js"; +import { NullLogService } from "../../../platform/log/common/log.js"; +import { Workspace, WorkspaceFolder, type IWorkspaceContextService } from "../../../platform/workspace/common/workspace.js"; +import { reviewSourceWindowDefaults } from "../../common/reviewConfigurationDefaults.js"; +import { reviewSourceWindowConfiguration } from "./reviewSourceWindowConfiguration.js"; + +const log = new NullLogService(); +const root = URI.file("/checkout"); +const file = URI.file("/checkout/src/a.ts"); + +function model(contents: Record): IConfigurationModel { + const keys: string[] = []; + const walk = (value: Record, prefix: string) => { + for (const [key, child] of Object.entries(value)) { + const path = prefix ? `${prefix}.${key}` : key; + if (["files", "window"].includes(path) && child && typeof child === "object") walk(child as Record, path); + else keys.push(path); + } + }; + walk(contents, ""); + return { contents, keys, overrides: [] }; +} + +function window() { + const [include, value] = Object.entries(reviewSourceWindowDefaults["files.readonlyInclude"])[0]; + const empty = model({}); + const data = { + // A host extension's configurationDefaults merge into the fork's, key by key. + defaults: model({ files: { readonlyInclude: { [include]: !value, "**/*.md": true }, readonlyExclude: { "**/*": true } }, window: { title: "${rootName}" } }), + policy: empty, + application: empty, + userLocal: empty, + userRemote: model({ files: { readonlyExclude: { "**/*": true } } }), + workspace: model({ files: { readonlyInclude: { [include]: false }, readonlyExclude: { "**/*": true } }, window: { title: "x" } }), + // The checkout's .vscode/settings.json. + folders: [[root, { contents: { files: { readonlyInclude: { [include]: false }, readonlyExclude: { "src/**": true } } }, keys: ["files.readonlyInclude", "files.readonlyExclude"], overrides: [] }]] as [URI, IConfigurationModel][], + }; + const workspace = new Workspace("w", [new WorkspaceFolder({ uri: root, name: "repo", index: 0 })], false, URI.file("/reviews/navigator/workspaces/worktree/repo.code-workspace"), () => false); + const configuration = Configuration.parse(data, log); + const changed = new Emitter(); + const base = { + getConfigurationData: () => configuration.toData(), + getValue: (section?: unknown, overrides?: unknown) => configuration.getValue(typeof section === "string" ? section : undefined, (typeof section === "string" ? overrides : section) as object ?? {}, workspace), + inspect: (key: string, overrides = {}) => configuration.inspect(key, overrides, workspace), + getWorkspace: () => workspace, + onDidChangeConfiguration: changed.event, + } as unknown as IConfigurationService & IWorkspaceContextService; + return reviewSourceWindowConfiguration(base, log); +} + +test("a Source window's extension defaults, workspace file, its checkout's .vscode/settings.json and the host's settings cannot lower read-only, and its title is the fork's", () => { + const { service, setTitle } = window(); + assert.deepEqual(service.getValue("files.readonlyInclude", { resource: file }), { "**/*": true }); + assert.deepEqual(service.getValue("files.readonlyExclude", { resource: file }), {}); + assert.deepEqual(service.getValue<{ files: object }>({ resource: file }).files, { readonlyInclude: { "**/*": true }, readonlyExclude: {} }); + assert.equal(service.inspect("files.readonlyInclude", { resource: file }).workspaceFolderValue, undefined); + assert.equal(service.inspect("files.readonlyExclude", { resource: file }).workspaceFolderValue, undefined); + assert.equal(service.getValue("window.title"), "x"); + + const events: IConfigurationChangeEvent[] = []; + service.onDidChangeConfiguration((e) => events.push(e)); + setTitle({ side: "live", title: "Fix the parser" }); + assert.equal(service.getValue("window.title"), "Fix the parser — Live source — Whiteboard"); + assert.equal(events.length, 1); + assert.ok(events[0].affectsConfiguration("window.title")); + setTitle({ side: "base", title: "Fix the parser" }); + assert.equal(service.getValue<{ window: { title: string } }>().window.title, "Fix the parser — Base source — Whiteboard"); + assert.equal(service.inspect("window.title").value, "Fix the parser — Base source — Whiteboard"); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/services/configuration/reviewSourceWindowConfiguration.ts b/apps/review-desktop/code-oss/src/vs/review/services/configuration/reviewSourceWindowConfiguration.ts new file mode 100644 index 000000000..b526ea0e0 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/services/configuration/reviewSourceWindowConfiguration.ts @@ -0,0 +1,107 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Emitter, Event } from '../../../base/common/event.js'; +import { deepClone } from '../../../base/common/objects.js'; +import { addToValueTree, ConfigurationTarget, isConfigurationOverrides, removeFromValueTree, type IConfigurationChangeEvent, type IConfigurationData, type IConfigurationModel, type IConfigurationOverrides, type IConfigurationService } from '../../../platform/configuration/common/configuration.js'; +import { Configuration, ConfigurationModel } from '../../../platform/configuration/common/configurationModels.js'; +import type { ILogService } from '../../../platform/log/common/log.js'; +import type { IWorkspaceContextService, Workspace } from '../../../platform/workspace/common/workspace.js'; +import { reviewSourceWindowDefaults } from '../../common/reviewConfigurationDefaults.js'; +import { override } from '../remote/reviewRemoteAuthority.js'; + +export type ReviewSourceSide = 'live' | 'base' | 'head'; + +const SIDES: Record = { live: 'Live source', base: 'Base source', head: 'Source' }; +const PINNED: Record = { 'files.readonlyInclude': reviewSourceWindowDefaults['files.readonlyInclude'], 'files.readonlyExclude': {} }; +const PINNED_KEYS = Object.keys(PINNED); +const TITLE = 'window.title'; + +export const REVIEW_SOURCE_TITLE_KEY = 'review.source.title'; + +export interface ReviewSourceTitle { + readonly side: ReviewSourceSide; + readonly title: string; + readonly alias?: string; +} + +export function isReviewSourceTitle(value: unknown): value is ReviewSourceTitle { + const { side, title, alias } = (value ?? {}) as Partial>; + return typeof side === 'string' && Object.hasOwn(SIDES, side) && typeof title === 'string' && (alias === undefined || typeof alias === 'string'); +} + +const within = (key: string, section: string) => key === section || key.startsWith(`${section}.`) || section.startsWith(`${key}.`); + +function without(model: IConfigurationModel, keys: readonly string[]): IConfigurationModel { + const contents = deepClone(model.contents); + const overrides = model.overrides.map(entry => ({ ...entry, contents: deepClone(entry.contents) })); + for (const key of keys) { + for (const tree of [contents, ...overrides.map(entry => entry.contents)]) removeFromValueTree(tree, key); + } + return { contents, overrides, keys: model.keys.filter(key => !keys.some(pinned => within(pinned, key))) }; +} + +function pinned(model: IConfigurationModel): IConfigurationModel { + const result = without(model, PINNED_KEYS); + for (const [key, value] of Object.entries(PINNED)) addToValueTree(result.contents, key, deepClone(value), () => { }); + return { ...result, keys: [...result.keys, ...PINNED_KEYS] }; +} + +export interface ReviewSourceWindowConfiguration { + readonly service: T; + setTitle(title: ReviewSourceTitle): void; +} + +/** + * A Source window's settings: no default an extension contributes, and no workspace, + * folder or remote machine setting, can lower its read-only rule, and `setTitle` + * sets the title above every layer. + */ +export function reviewSourceWindowConfiguration(base: T, logService: ILogService): ReviewSourceWindowConfiguration { + let source: ReviewSourceTitle | undefined; + let cached: Configuration | undefined; + const changed = new Emitter(); + const data = (): IConfigurationData => { + const empty = ConfigurationModel.createEmptyModel(logService).toJSON(); + const window = base.getConfigurationData() ?? { defaults: empty, policy: empty, application: empty, userLocal: empty, userRemote: empty, workspace: empty, folders: [] }; + return { + ...window, + defaults: pinned(window.defaults), + userRemote: without(window.userRemote, PINNED_KEYS), + workspace: without(window.workspace, PINNED_KEYS), + folders: window.folders.map(([folder, model]) => [folder, without(model, PINNED_KEYS)]), + }; + }; + const title = () => source && `${source.title} — ${SIDES[source.side]} — Whiteboard`; + const configuration = () => { + if (!cached) { + cached = Configuration.parse(data(), logService); + const value = title(); + if (value) cached.updateValue(TITLE, value); + } + return cached; + }; + const ours = (section: string | undefined) => section === undefined || [...PINNED_KEYS, ...(title() ? [TITLE] : [])].some(key => within(key, section)); + const workspace = () => base.getWorkspace() as Workspace; + base.onDidChangeConfiguration(() => cached = undefined); + return { + service: override(base, { + getConfigurationData: data, + getValue: ((arg1?: unknown, arg2?: unknown) => { + const section = typeof arg1 === 'string' ? arg1 : undefined; + const overrides = isConfigurationOverrides(arg1) ? arg1 : isConfigurationOverrides(arg2) ? arg2 : {}; + return ours(section) ? configuration().getValue(section, overrides, workspace()) : (base.getValue as (arg1?: unknown, arg2?: unknown) => unknown)(arg1, arg2); + }) as T['getValue'], + inspect: ((key: string, overrides: IConfigurationOverrides = {}) => ours(key) ? configuration().inspect(key, overrides, workspace()) : base.inspect(key, overrides)) as T['inspect'], + onDidChangeConfiguration: Event.any(base.onDidChangeConfiguration, changed.event), + } as Partial), + setTitle(next) { + if (source?.side === next.side && source.title === next.title) return; + source = { side: next.side, title: next.title }; + cached = undefined; + changed.fire({ source: ConfigurationTarget.MEMORY, change: { keys: [TITLE], overrides: [] }, affectedKeys: new Set([TITLE]), affectsConfiguration: section => within(TITLE, section) }); + }, + }; +} diff --git a/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteAuthority.ts b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteAuthority.ts new file mode 100644 index 000000000..9b860ef31 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteAuthority.ts @@ -0,0 +1,30 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Schemas } from "../../../base/common/network.js"; +import type { URI } from "../../../base/common/uri.js"; + +export function reviewRemoteAuthority(serverId: string): string | undefined { + return /^[0-9a-z-]+$/i.test(serverId) ? `whiteboard+${serverId.toLowerCase()}` : undefined; +} + +export function ownsRemoteResource(authority: string, resource: URI): boolean { + return resource.scheme === Schemas.vscodeRemote && resource.authority.toLowerCase() === authority; +} + +/** Exactly the lower-case `whiteboard+` that `reviewRemoteAuthority` makes. */ +export function isReviewRemoteAuthority(authority: string | null | undefined): authority is string { + return !!authority && /^whiteboard\+[0-9a-z-]+$/.test(authority); +} + +export function override(base: T, members: Partial): T { + return new Proxy(base, { + get(target, key) { + if (key in members) return (members as Record)[key]; + const value = Reflect.get(target, key); + return typeof value === "function" ? value.bind(target) : value; + }, + }); +} diff --git a/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteExtensionService.ts b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteExtensionService.ts new file mode 100644 index 000000000..7cfab23d6 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteExtensionService.ts @@ -0,0 +1,23 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { ExtensionIdentifier, type IExtensionDescription } from "../../../platform/extensions/common/extensions.js"; +import type { IExtensionService } from "../../../workbench/services/extensions/common/extensions.js"; +import { override } from "./reviewRemoteAuthority.js"; + +export function reviewRemoteExtensionService( + base: IExtensionService, + extensions: readonly IExtensionDescription[], + activate: (event: string) => Promise, +): IExtensionService { + const byId = new Map(extensions.map((extension) => [ExtensionIdentifier.toKey(extension.identifier), extension])); + return override(base, { + extensions, + activateByEvent: (event) => activate(event), + activationEventIsDone: () => false, + whenInstalledExtensionsRegistered: async () => true, + getExtension: async (id) => byId.get(ExtensionIdentifier.toKey(id)), + }); +} diff --git a/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteFileEvents.test.ts b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteFileEvents.test.ts new file mode 100644 index 000000000..491cb5050 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteFileEvents.test.ts @@ -0,0 +1,35 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { Emitter } from "../../../base/common/event.js"; +import { URI } from "../../../base/common/uri.js"; +import { FileChangesEvent, FileChangeType, FileOperation, FileOperationEvent, type IFileService } from "../../../platform/files/common/files.js"; +import { reviewRemoteFileEvents } from "./reviewRemoteFileEvents.js"; + +const A = "whiteboard+aaaa-1111"; +const B = "whiteboard+bbbb-2222"; +const onA = URI.parse(`vscode-remote://${A}/home/dev/repo/f.ts`); +const onB = URI.parse(`vscode-remote://${B}/home/dev/repo/f.ts`); +const laptop = URI.file("/home/dev/repo/f.ts"); + +test("a host hears only its own file changes and operations, though another host and the laptop have the same path", () => { + const changes = new Emitter(); + const operations = new Emitter(); + const window = { onDidFilesChange: changes.event, onDidRunOperation: operations.event, readFile: async () => "the window's" } as unknown as IFileService; + const listen = (authority: string) => { + const files = reviewRemoteFileEvents(window, authority); + const seen: string[] = []; + files.onDidFilesChange((e) => seen.push(...e.rawUpdated.map((uri) => `changed ${uri}`), ...e.rawDeleted.map((uri) => `deleted ${uri}`))); + files.onDidRunOperation((e) => seen.push(`wrote ${e.resource}`)); + return { files, seen }; + }; + const [a, b] = [listen(A), listen(B)]; + + changes.fire(new FileChangesEvent([{ resource: onA, type: FileChangeType.UPDATED }, { resource: laptop, type: FileChangeType.UPDATED }], false)); + changes.fire(new FileChangesEvent([{ resource: laptop, type: FileChangeType.DELETED }], false)); + operations.fire(new FileOperationEvent(onA, FileOperation.WRITE)); + operations.fire(new FileOperationEvent(laptop, FileOperation.WRITE)); + + assert.deepEqual(a.seen, [`changed ${onA}`, `wrote ${onA}`]); + assert.deepEqual(b.seen, []); + return b.files.readFile(onB).then((value) => assert.equal(value, "the window's", "reads still go to the window's service")); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteFileEvents.ts b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteFileEvents.ts new file mode 100644 index 000000000..930955b8b --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteFileEvents.ts @@ -0,0 +1,22 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Event } from "../../../base/common/event.js"; +import type { URI } from "../../../base/common/uri.js"; +import { FileChangesEvent, FileChangeType, type IFileService } from "../../../platform/files/common/files.js"; +import { override, ownsRemoteResource } from "./reviewRemoteAuthority.js"; + +/** Only this host's file events: upstream's URI transformer drops the authority, so another host's change to the same path would look like its own. */ +export function reviewRemoteFileEvents(base: IFileService, authority: string): IFileService { + const mine = (resource: URI) => ownsRemoteResource(authority, resource); + return override(base, { + onDidFilesChange: Event.filter(Event.map(base.onDidFilesChange, (e) => new FileChangesEvent([ + ...e.rawAdded.filter(mine).map((resource) => ({ resource, type: FileChangeType.ADDED })), + ...e.rawUpdated.filter(mine).map((resource) => ({ resource, type: FileChangeType.UPDATED })), + ...e.rawDeleted.filter(mine).map((resource) => ({ resource, type: FileChangeType.DELETED })), + ], false)), (e) => e.rawAdded.length + e.rawUpdated.length + e.rawDeleted.length > 0), + onDidRunOperation: Event.filter(base.onDidRunOperation, (e) => mine(e.resource) && (!e.target || mine(e.target.resource))), + }); +} diff --git a/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteFileSystemRouter.test.ts b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteFileSystemRouter.test.ts new file mode 100644 index 000000000..634a56be8 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteFileSystemRouter.test.ts @@ -0,0 +1,90 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { Emitter } from "../../../base/common/event.js"; +import { URI } from "../../../base/common/uri.js"; +import { + FileSystemProviderCapabilities, + FileSystemProviderErrorCode, + type IFileChange, + type IFileSystemProviderWithFileReadWriteCapability, + toFileSystemProviderErrorCode, +} from "../../../platform/files/common/files.js"; +import { ReviewRemoteFileSystemRouter } from "./reviewRemoteFileSystemRouter.js"; + +const A = "whiteboard+aaaa-1111"; +const B = "whiteboard+bbbb-2222"; + +function remote(name: string) { + const changed = new Emitter(); + const provider = { + onDidChangeFile: changed.event, + readFile: async (resource: URI) => new TextEncoder().encode(`${name}:${resource.path}`), + stat: async () => ({ type: 1, ctime: 0, mtime: 0, size: name.length }), + readdir: async () => [[name, 1]], + watch: () => ({ dispose() { } }), + writeFile: async () => { throw new Error(`${name} was written`); }, + } as unknown as IFileSystemProviderWithFileReadWriteCapability; + return { provider, changed }; +} + +const text = (bytes: Uint8Array) => new TextDecoder().decode(bytes); +async function code(promise: Promise) { + try { + await promise; + } catch (error) { + return toFileSystemProviderErrorCode(error as Error); + } + return "resolved"; +} + +test("each call goes to the host the authority names, whatever its case", async () => { + const router = new ReviewRemoteFileSystemRouter(); + router.add(A, remote("a").provider); + router.add(B, remote("b").provider); + assert.equal(text(await router.readFile(URI.parse(`vscode-remote://${A}/p/a.ts`))), "a:/p/a.ts"); + assert.equal(text(await router.readFile(URI.parse(`vscode-remote://${B}/p/a.ts`))), "b:/p/a.ts"); + assert.equal(text(await router.readFile(URI.parse(`vscode-remote://${A.toUpperCase()}/p/a.ts`))), "a:/p/a.ts"); + assert.deepEqual(await router.readdir(URI.parse(`vscode-remote://${B}/p`)), [["b", 1]]); + router.dispose(); +}); + +test("an authority that names no connected host is a missing file", async () => { + const router = new ReviewRemoteFileSystemRouter(); + const added = router.add(A, remote("a").provider); + assert.equal(await code(router.stat(URI.parse(`vscode-remote://${B}/p/a.ts`))), FileSystemProviderErrorCode.FileNotFound); + added.dispose(); + assert.equal(await code(router.readFile(URI.parse(`vscode-remote://${A}/p/a.ts`))), FileSystemProviderErrorCode.FileNotFound); + router.dispose(); +}); + +test("it is read-only and refuses every change without reaching a host", async () => { + const router = new ReviewRemoteFileSystemRouter(); + router.add(A, remote("a").provider); + const file = URI.parse(`vscode-remote://${A}/p/a.ts`); + assert.ok(router.capabilities & FileSystemProviderCapabilities.Readonly); + for (const change of [ + router.writeFile(), + router.mkdir(), + router.delete(), + router.rename(), + ]) assert.equal(await code(change), FileSystemProviderErrorCode.NoPermissions); + assert.equal(text(await router.readFile(file)), "a:/p/a.ts"); + router.dispose(); +}); + +test("changes from a host are passed on until that host is removed, and a newer host keeps its route", async () => { + const router = new ReviewRemoteFileSystemRouter(); + const first = remote("first"); + const second = remote("second"); + const seen: string[] = []; + router.onDidChangeFile((changes) => seen.push(...changes.map((change) => change.resource.path))); + const removeFirst = router.add(A, first.provider); + first.changed.fire([{ type: 0, resource: URI.parse(`vscode-remote://${A}/one`) }]); + router.add(A, second.provider); + removeFirst.dispose(); + first.changed.fire([{ type: 0, resource: URI.parse(`vscode-remote://${A}/two`) }]); + second.changed.fire([{ type: 0, resource: URI.parse(`vscode-remote://${A}/three`) }]); + assert.deepEqual(seen, ["/one", "/three"]); + assert.equal(text(await router.readFile(URI.parse(`vscode-remote://${A}/p`))), "second:/p"); + router.dispose(); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteFileSystemRouter.ts b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteFileSystemRouter.ts new file mode 100644 index 000000000..4e9f41ea3 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteFileSystemRouter.ts @@ -0,0 +1,84 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Emitter, Event } from "../../../base/common/event.js"; +import { Disposable, type IDisposable, toDisposable } from "../../../base/common/lifecycle.js"; +import type { URI } from "../../../base/common/uri.js"; +import { + createFileSystemProviderError, + type FileType, + FileSystemProviderCapabilities, + FileSystemProviderErrorCode, + type IFileChange, + type IFileSystemProviderWithFileReadWriteCapability, + type IStat, + type IWatchOptions, +} from "../../../platform/files/common/files.js"; + +export class ReviewRemoteFileSystemRouter extends Disposable implements IFileSystemProviderWithFileReadWriteCapability { + readonly capabilities = + FileSystemProviderCapabilities.FileReadWrite | + FileSystemProviderCapabilities.PathCaseSensitive | + FileSystemProviderCapabilities.Readonly; + readonly onDidChangeCapabilities = Event.None; + private readonly changed = this._register(new Emitter()); + readonly onDidChangeFile = this.changed.event; + private readonly watchErrors = this._register(new Emitter()); + readonly onDidWatchError = this.watchErrors.event; + private readonly hosts = new Map(); + + add(authority: string, provider: IFileSystemProviderWithFileReadWriteCapability): IDisposable { + this.hosts.set(authority, provider); + const changes = provider.onDidChangeFile((e) => this.changed.fire(e)); + const errors = provider.onDidWatchError?.((e) => this.watchErrors.fire(e)); + return toDisposable(() => { + changes.dispose(); + errors?.dispose(); + if (this.hosts.get(authority) === provider) this.hosts.delete(authority); + }); + } + + private host(resource: URI): IFileSystemProviderWithFileReadWriteCapability { + const host = this.hosts.get(resource.authority.toLowerCase()); + if (!host) throw createFileSystemProviderError("No connected remote host for this file.", FileSystemProviderErrorCode.FileNotFound); + return host; + } + + async stat(resource: URI): Promise { + return this.host(resource).stat(resource); + } + + async readdir(resource: URI): Promise<[string, FileType][]> { + return this.host(resource).readdir(resource); + } + + async readFile(resource: URI): Promise { + return this.host(resource).readFile(resource); + } + + watch(resource: URI, opts: IWatchOptions): IDisposable { + return this.hosts.get(resource.authority.toLowerCase())?.watch(resource, opts) ?? Disposable.None; + } + + writeFile(): Promise { + return refuse(); + } + + mkdir(): Promise { + return refuse(); + } + + delete(): Promise { + return refuse(); + } + + rename(): Promise { + return refuse(); + } +} + +function refuse(): Promise { + return Promise.reject(createFileSystemProviderError("Remote review files are read-only.", FileSystemProviderErrorCode.NoPermissions)); +} diff --git a/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteHost.test.ts b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteHost.test.ts new file mode 100644 index 000000000..289bf3297 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteHost.test.ts @@ -0,0 +1,162 @@ +import assert from "node:assert/strict"; +import { mock, test } from "node:test"; +import { URI } from "../../../base/common/uri.js"; +import { NullLogService } from "../../../platform/log/common/log.js"; +import { type IReviewRemoteSession, ReviewRemoteHost } from "./reviewRemoteHost.js"; + +const AUTHORITY = "whiteboard+aaaa-1111"; + +function session() { + let fail!: (reason: string) => void; + const failed = new Promise((resolve) => (fail = resolve)); + const value = { + failed, + activated: [] as string[], + disposed: false, + closed: false, + async activateByEvent(event: string) { value.activated.push(event); }, + async close() { value.closed = true; }, + dispose() { value.disposed = true; }, + }; + return { value: value as IReviewRemoteSession & typeof value, fail }; +} + +function host(script: (() => IReviewRemoteSession | undefined | Error)[], endpoint: () => string | undefined = () => undefined) { + const opens: number[] = []; + const probes: number[] = []; + const target = new ReviewRemoteHost("aaaa-1111", AUTHORITY, async () => { + opens.push(Date.now()); + const next = script.shift()?.(); + if (next instanceof Error) throw next; + return next; + }, async () => (probes.push(Date.now()), endpoint()), new NullLogService()); + return { target, opens, probes }; +} + +async function settle() { + for (let i = 0; i < 10; i++) await Promise.resolve(); +} + +test("while the endpoint is missing or the connect fails, the host tries again after each delay, then connects", async (t) => { + mock.timers.enable({ apis: ["setTimeout", "Date"], now: 0 }); + t.after(() => mock.timers.reset()); + const connected = session(); + const { target, opens } = host([() => undefined, () => new Error("WebSocket close with status code 1006"), () => undefined, () => connected.value]); + + assert.equal(await target.connect(), false, "the first ask gets undefined, not a hang"); + for (const step of [1_000, 2_000, 4_000]) { + mock.timers.tick(step); + await settle(); + } + assert.deepEqual(opens, [0, 1_000, 3_000, 7_000]); + assert.equal(await target.connect(), true); + assert.equal(opens.length, 4, "a connected host does not open again"); + target.dispose(); + assert.equal(connected.value.disposed, true); +}); + +test("a failed session is replaced after a delay, with the roots and activation events carried over", async (t) => { + mock.timers.enable({ apis: ["setTimeout", "Date"], now: 0 }); + t.after(() => mock.timers.reset()); + const first = session(); + const second = session(); + const { target, opens } = host([() => first.value, () => second.value]); + + assert.equal(await target.connect(), true); + const root = await target.addRoot(URI.parse(`vscode-remote://${AUTHORITY}/home/dev/proj`)); + await target.activateByEvent("onLanguage:typescript"); + first.fail("the Management connection"); + await settle(); + assert.equal(first.value.disposed, true, "the whole session ends, both connections"); + assert.equal(await target.connect(), false, "asking again during the delay neither connects nor waits"); + mock.timers.tick(999); + await settle(); + assert.equal(opens.length, 1); + mock.timers.tick(1); + await settle(); + assert.equal(opens.length, 2); + assert.deepEqual(second.value.activated, ["onLanguage:typescript"]); + assert.deepEqual(target.workspace.getWorkspace().folders.map((folder) => folder.uri.path), ["/home/dev/proj"]); + root.dispose(); + target.dispose(); +}); + +test("a root on another machine is refused, and a closed host stops trying and closes its session", async (t) => { + mock.timers.enable({ apis: ["setTimeout", "Date"], now: 0 }); + t.after(() => mock.timers.reset()); + const only = session(); + const { target, opens } = host([() => undefined, () => only.value]); + + await assert.rejects(target.addRoot(URI.parse("vscode-remote://whiteboard+bbbb-2222/home/dev/proj"))); + await assert.rejects(target.addRoot(URI.file("/home/dev/proj"))); + assert.equal(await target.connect(), false); + await target.close(); + mock.timers.tick(60_000); + await settle(); + assert.equal(opens.length, 1); + assert.equal(await target.connect(), false); + + const connected = host([() => only.value]); + await connected.target.connect(); + await connected.target.close(); + assert.deepEqual([only.value.closed, only.value.disposed], [true, true]); +}); + +test("a reload during a first connect waits for it, then closes that session: no extension host is left behind", async () => { + const late = session(); + let resolveOpen!: (value: IReviewRemoteSession) => void; + let opens = 0; + const target = new ReviewRemoteHost("aaaa-1111", AUTHORITY, () => { + opens++; + return new Promise((resolve) => (resolveOpen = resolve)); + }, async () => undefined, new NullLogService()); + const connecting = target.connect(); + const closing = target.close(); + resolveOpen(late.value); + await closing; + assert.equal(await connecting, true); + assert.deepEqual([late.value.closed, late.value.disposed, opens], [true, true, 1]); + assert.equal(await target.connect(), false); + assert.equal(opens, 1); +}); + +test("after a server restart, a request connects as soon as main has a new endpoint; none, or one already tried, keeps the delays", async (t) => { + mock.timers.enable({ apis: ["setTimeout", "Date"], now: 0 }); + t.after(() => mock.timers.reset()); + const [first, second, third] = [session(), session(), session()]; + const refused = () => new Error("WebSocket close with status code 1006"); + let endpoint: string | undefined; + const { target, opens, probes } = host([() => first.value, () => undefined, () => undefined, () => undefined, () => second.value, refused, refused, () => third.value], () => endpoint); + const tick = async (ms: number) => { + mock.timers.tick(ms); + await settle(); + }; + + assert.equal(await target.connect(), true); + first.fail("the extension host connection"); + await settle(); + for (const ms of [1_000, 2_000, 4_000]) await tick(ms); + assert.deepEqual(opens, [0, 1_000, 3_000, 7_000], "main is attaching again; the next try would be at 15 s"); + + assert.equal(await target.connect(), false, "main has no endpoint yet"); + await tick(1_000); + assert.equal(await target.connect(), false); + assert.deepEqual(probes, [7_000], "at most one ask per 5 s"); + + await tick(4_000); + endpoint = "new"; + assert.equal(await target.connect(), true, "main has attached again: connected at once"); + assert.deepEqual(opens.at(-1), 12_000); + + second.fail("the extension host connection"); + await settle(); + await tick(1_000); + await tick(2_000); + await tick(2_000); + assert.equal(await target.connect(), false, "the endpoint already tried keeps the delay"); + assert.deepEqual(opens, [0, 1_000, 3_000, 7_000, 12_000, 13_000, 15_000]); + await tick(2_000); + assert.deepEqual(opens.at(-1), 19_000); + assert.equal(await target.connect(), true); + target.dispose(); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteHost.ts b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteHost.ts new file mode 100644 index 000000000..374836ff2 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteHost.ts @@ -0,0 +1,323 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { DeferredPromise, raceTimeout, TimeoutTimer } from "../../../base/common/async.js"; +import { toErrorMessage } from "../../../base/common/errorMessage.js"; +import { CancellationError } from "../../../base/common/errors.js"; +import { Disposable, type IDisposable } from "../../../base/common/lifecycle.js"; +import * as platform from "../../../base/common/platform.js"; +import type { Mutable } from "../../../base/common/types.js"; +import { URI } from "../../../base/common/uri.js"; +import type { ILanguageFeaturesService } from "../../../editor/common/services/languageFeatures.js"; +import { LanguageFeaturesService } from "../../../editor/common/services/languageFeaturesService.js"; +import type { IExtensionDescription } from "../../../platform/extensions/common/extensions.js"; +import { DiskFileSystemProviderClient } from "../../../platform/files/common/diskFileSystemProviderClient.js"; +import { IInstantiationService } from "../../../platform/instantiation/common/instantiation.js"; +import { ILogService } from "../../../platform/log/common/log.js"; +import { IProductService } from "../../../platform/product/common/productService.js"; +import { + connectRemoteAgentManagement, + type IConnectionOptions, + PersistentConnectionEventType, +} from "../../../platform/remote/common/remoteAgentConnection.js"; +import { + IRemoteAuthorityResolverService, + type IRemoteConnectionData, + RemoteAuthorityResolverError, + RemoteAuthorityResolverErrorCode, + WebSocketRemoteConnection, +} from "../../../platform/remote/common/remoteAuthorityResolver.js"; +import { RemoteExtensionsScannerChannelName } from "../../../platform/remote/common/remoteExtensionsScanner.js"; +import { IRemoteSocketFactoryService } from "../../../platform/remote/common/remoteSocketFactoryService.js"; +import { ISignService } from "../../../platform/sign/common/sign.js"; +import { ExtensionHostManager } from "../../../workbench/services/extensions/common/extensionHostManager.js"; +import { RemoteRunningLocation } from "../../../workbench/services/extensions/common/extensionRunningLocation.js"; +import { + ActivationKind, + ExtensionHostExtensions, + type IInternalExtensionService, +} from "../../../workbench/services/extensions/common/extensions.js"; +import { RemoteExtensionHost } from "../../../workbench/services/extensions/common/remoteExtensionHost.js"; +import { RemoteExtensionEnvironmentChannelClient } from "../../../workbench/services/remote/common/remoteAgentEnvironmentChannel.js"; +import { REMOTE_FILE_SYSTEM_CHANNEL_NAME } from "../../../workbench/services/remote/common/remoteFileSystemProviderClient.js"; +import { IReviewDesktopConnectionService, type ReviewRemoteLanguageEndpoint } from "../reviewDesktopConnectionService.js"; +import type { ReviewRemoteFileSystemRouter } from "./reviewRemoteFileSystemRouter.js"; +import { ownsRemoteResource } from "./reviewRemoteAuthority.js"; +import { ReviewRemoteWorkspace, reviewRemoteResolver, reviewRemoteScope } from "./reviewRemoteScope.js"; + +export interface IReviewRemoteHost { + readonly authority: string; + readonly languageFeatures: ILanguageFeaturesService; + addRoot(root: URI): Promise; + activateByEvent(event: string): Promise; +} + +export interface IReviewRemoteSession extends IDisposable { + readonly failed: Promise; + activateByEvent(event: string): Promise; + close(): Promise; +} + +function reviewRemoteRetryDelay(failures: number): number { + return Math.min(60_000, 1_000 * 2 ** failures); +} + +const STABLE_MS = 60_000; + +const PROBE_MS = 5_000; + +export class ReviewRemoteHost extends Disposable implements IReviewRemoteHost { + readonly languageFeatures: ILanguageFeaturesService = new LanguageFeaturesService(); + readonly workspace: ReviewRemoteWorkspace; + private session: IReviewRemoteSession | undefined; + private connecting: Promise | undefined; + private failures = 0; + private waiting = false; + private probing: Promise | undefined; + private probed = -Infinity; + private probedEndpoint: string | undefined; + private readonly retry = this._register(new TimeoutTimer()); + private readonly activations = new Set(); + + constructor( + readonly serverId: string, + readonly authority: string, + private readonly open: (host: ReviewRemoteHost) => Promise, + private readonly endpoint: () => Promise, + private readonly logService: ILogService, + ) { + super(); + this.workspace = this._register(new ReviewRemoteWorkspace(`whiteboard-remote-${serverId}`)); + } + + connect(): Promise { + if (this.session) return Promise.resolve(true); + if (this._store.isDisposed) return Promise.resolve(false); + if (this.waiting) return this.probe(); + this.connecting ??= this.attempt().finally(() => (this.connecting = undefined)); + return this.connecting; + } + + private probe(): Promise { + if (this.probing || Date.now() - this.probed < PROBE_MS) return this.probing ?? Promise.resolve(false); + this.probed = Date.now(); + this.probing = (async () => { + const endpoint = await this.endpoint().catch(() => undefined); + if (this.waiting && endpoint && endpoint !== this.probedEndpoint) { + this.probedEndpoint = endpoint; + this.retry.cancel(); + this.waiting = false; + this.failures = 0; + } + return this.waiting ? false : this.connect(); + })().finally(() => (this.probing = undefined)); + return this.probing; + } + + private async attempt(): Promise { + let session: IReviewRemoteSession | undefined; + try { + session = await this.open(this); + } catch (error) { + this.logService.warn(`[Remote language] ${this.authority}: could not connect: ${toErrorMessage(error)}`); + } + if (this._store.isDisposed) { + session?.dispose(); + return false; + } + if (!session) { + this.retryLater(); + return false; + } + const opened = Date.now(); + this.session = session; + void session.failed.then((reason) => { + if (this.session !== session) return; + this.logService.warn(`[Remote language] ${this.authority}: ${reason} failed; connecting again`); + this.session = undefined; + session.dispose(); + if (Date.now() - opened >= STABLE_MS) this.failures = 0; + this.retryLater(); + }); + for (const event of this.activations) { + session.activateByEvent(event).catch((error) => this.logService.warn(`[Remote language] ${this.authority}: ${event} failed: ${toErrorMessage(error)}`)); + } + return true; + } + + private retryLater(): void { + this.waiting = true; + this.retry.cancelAndSet(() => { + this.waiting = false; + void this.connect(); + }, reviewRemoteRetryDelay(this.failures++)); + } + + async addRoot(root: URI): Promise { + if (!ownsRemoteResource(this.authority, root)) throw new Error("That root is not on this remote host."); + return this.workspace.add(root); + } + + async activateByEvent(event: string): Promise { + this.activations.add(event); + await this.session?.activateByEvent(event); + } + + async close(): Promise { + if (this.connecting) await raceTimeout(this.connecting, 2_000); + const session = this.session; + this.session = undefined; + this.dispose(); + if (!session) return; + try { + await session.close(); + } finally { + session.dispose(); + } + } + + override dispose(): void { + this.session?.dispose(); + this.session = undefined; + super.dispose(); + } +} + +class ReviewRemoteExtensionHost extends RemoteExtensionHost { + private readonly startResult = new DeferredPromise(); + readonly started = this.startResult.p; + + override start() { + const started = super.start(); + started.then(() => this.startResult.complete(), (error) => this.startResult.error(error)); + return started; + } +} + +export class ReviewRemoteSession extends Disposable implements IReviewRemoteSession { + private readonly failure = new DeferredPromise(); + readonly failed = this.failure.p; + private manager: ExtensionHostManager | undefined; + private started = false; + + constructor( + private readonly host: ReviewRemoteHost, + private endpoint: ReviewRemoteLanguageEndpoint | undefined, + private readonly router: ReviewRemoteFileSystemRouter, + @IInstantiationService private readonly instantiationService: IInstantiationService, + @IProductService private readonly productService: IProductService, + @IRemoteSocketFactoryService private readonly remoteSocketFactoryService: IRemoteSocketFactoryService, + @ISignService private readonly signService: ISignService, + @ILogService private readonly logService: ILogService, + @IRemoteAuthorityResolverService private readonly resolverService: IRemoteAuthorityResolverService, + @IReviewDesktopConnectionService private readonly connection: IReviewDesktopConnectionService, + ) { + super(); + } + + private async address(): Promise { + const endpoint = this.started ? await this.connection.getRemoteLanguageEndpoint(this.host.serverId) : this.endpoint; + if (!endpoint) throw new RemoteAuthorityResolverError("The remote host is not available.", RemoteAuthorityResolverErrorCode.TemporarilyNotAvailable); + return { connectTo: new WebSocketRemoteConnection(endpoint.host, endpoint.port), connectionToken: endpoint.connectionToken }; + } + + private options(): IConnectionOptions { + return { + commit: this.productService.commit, + quality: this.productService.quality, + addressProvider: { getAddress: () => this.address() }, + remoteSocketFactoryService: this.remoteSocketFactoryService, + signService: this.signService, + logService: this.logService, + ipcLogger: null, + }; + } + + private fail(reason: string): void { + if (!this.failure.isSettled) this.failure.complete(reason); + } + + private check(): void { + if (this._store.isDisposed) throw new CancellationError(); + if (this.failure.isSettled) throw new Error("A connection failed while it started."); + } + + async start(): Promise { + const { authority } = this.host; + const management = await connectRemoteAgentManagement(this.options(), authority, "renderer"); + if (this._store.isDisposed) { + management.dispose(); + throw new CancellationError(); + } + this._register(management); + this._register(management.onDidStateChange((event) => { + if (event.type === PersistentConnectionEventType.ReconnectionPermanentFailure) this.fail("the Management connection"); + })); + const environment = await RemoteExtensionEnvironmentChannelClient.getEnvironmentData( + management.client.getChannel("remoteextensionsenvironment"), + authority, + undefined, + ); + if (environment.reconnectionGraceTime !== undefined) management.updateGraceTime(environment.reconnectionGraceTime); + const extensions = await management.client + .getChannel(RemoteExtensionsScannerChannelName) + .call[]>("scanExtensions", [platform.language, undefined, [], undefined, undefined]); + for (const extension of extensions) extension.extensionLocation = URI.revive(extension.extensionLocation); + this.check(); + + const files = this._register(new DiskFileSystemProviderClient(management.client.getChannel(REMOTE_FILE_SYSTEM_CHANNEL_NAME), { pathCaseSensitive: true })); + this._register(this.router.add(authority, files)); + + const scope = this._register(this.instantiationService.createChild(this.instantiationService.invokeFunction((window) => reviewRemoteScope({ + authority, + extensions, + activate: (event) => this.activateByEvent(event), + languageFeatures: this.host.languageFeatures, + workspace: this.host.workspace, + resolver: reviewRemoteResolver(this.resolverService, authority, () => this.address()), + }, window)))); + const extensionHost = scope.createInstance(ReviewRemoteExtensionHost, new RemoteRunningLocation(), { + remoteAuthority: authority, + getInitData: async () => ({ + connectionData: await this.address(), + pid: environment.pid, + appRoot: environment.appRoot, + extensionHostLogsPath: environment.extensionHostLogsPath, + globalStorageHome: environment.globalStorageHome, + workspaceStorageHome: environment.workspaceStorageHome, + extensions: new ExtensionHostExtensions(1, extensions, extensions.filter((extension) => extension.main).map((extension) => extension.identifier)), + }), + }); + const manager = (this.manager = this._register(scope.createInstance(ExtensionHostManager, extensionHost, [], this.internalExtensionService()))); + this._register(manager.onDidExit(() => this.fail("the extension host connection"))); + await extensionHost.started; + await manager.ready(); + this.check(); + this.started = true; + this.endpoint = undefined; + } + + private internalExtensionService(): IInternalExtensionService { + const log = (message: string) => this.logService.info(`[Remote language] ${this.host.authority}: ${message}`); + return { + _activateById: async (id, reason) => { + await this.manager?.activate(id, reason); + }, + _onWillActivateExtension: () => { }, + _onDidActivateExtension: (id) => log(`activated ${id.value}`), + _onDidActivateExtensionError: (id, error) => log(`${id.value} failed to activate: ${toErrorMessage(error)}`), + _onExtensionRuntimeError: (id, error) => log(`${id.value}: ${toErrorMessage(error)}`), + }; + } + + async activateByEvent(event: string): Promise { + await this.manager?.activateByEvent(event, ActivationKind.Normal); + } + + async close(): Promise { + await raceTimeout(this.manager?.disconnect() ?? Promise.resolve(), 2_000); + this.dispose(); + } +} diff --git a/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteHosts.test.ts b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteHosts.test.ts new file mode 100644 index 000000000..c108a11e7 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteHosts.test.ts @@ -0,0 +1,45 @@ +import assert from "node:assert/strict"; +import { mock, test } from "node:test"; +import { Emitter } from "../../../base/common/event.js"; +import { NullLogService } from "../../../platform/log/common/log.js"; +import type { ReviewRemoteLanguageEndpoint } from "../reviewDesktopConnectionService.js"; +import { ReviewRemoteHostsService } from "./reviewRemoteHosts.js"; + +const SERVER_ID = "3480c31a-77f0-4d6e-9a53-1b2c3d4e5f60"; + +function service() { + const asked: string[] = []; + const shutdown = new Emitter<{ join(promise: Promise, joiner: unknown): void }>(); + const joined: Promise[] = []; + const target = new ReviewRemoteHostsService( + {} as never, + { + getRemoteLanguageEndpoint: async (serverId: string): Promise => { + asked.push(serverId); + return undefined; + }, + readRemoteHosts: async () => [], + } as never, + {} as never, + new NullLogService(), + { registerProvider: () => ({ dispose() { } }) } as never, + { onWillShutdown: shutdown.event } as never, + ); + const shut = () => shutdown.fire({ join: (promise) => joined.push(promise) }); + return { target, asked, shut, joined }; +} + +test("shutdown and reload close every host, and none is created afterwards", async (t) => { + mock.timers.enable({ apis: ["setTimeout", "Date"], now: 0 }); + t.after(() => mock.timers.reset()); + const { target, asked, shut, joined } = service(); + await target.host(SERVER_ID); + shut(); + await Promise.all(joined); + assert.equal(joined.length, 1); + mock.timers.tick(60_000); + for (let i = 0; i < 10; i++) await Promise.resolve(); + assert.equal(await target.host(SERVER_ID), undefined); + assert.deepEqual(asked, [SERVER_ID], "no retry after the host closed"); + target.dispose(); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteHosts.ts b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteHosts.ts new file mode 100644 index 000000000..e64d3a01c --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteHosts.ts @@ -0,0 +1,115 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { hash } from "../../../base/common/hash.js"; +import { Disposable, DisposableMap } from "../../../base/common/lifecycle.js"; +import { Schemas } from "../../../base/common/network.js"; +import { IFileService } from "../../../platform/files/common/files.js"; +import { InstantiationType, registerSingleton } from "../../../platform/instantiation/common/extensions.js"; +import { createDecorator, IInstantiationService } from "../../../platform/instantiation/common/instantiation.js"; +import { ILabelService } from "../../../platform/label/common/label.js"; +import { ILogService } from "../../../platform/log/common/log.js"; +import { registerWorkbenchContribution2, WorkbenchPhase } from "../../../workbench/common/contributions.js"; +import { ILifecycleService } from "../../../workbench/services/lifecycle/common/lifecycle.js"; +import { IReviewDesktopConnectionService } from "../reviewDesktopConnectionService.js"; +import { ReviewRemoteFileSystemRouter } from "./reviewRemoteFileSystemRouter.js"; +import { type IReviewRemoteHost, ReviewRemoteHost, ReviewRemoteSession } from "./reviewRemoteHost.js"; +import { reviewRemoteAuthority } from "./reviewRemoteAuthority.js"; + +export const IReviewRemoteHostsService = createDecorator("reviewRemoteHostsService"); + +export interface IReviewRemoteHostsService { + readonly _serviceBrand: undefined; + host(serverId: string): Promise; +} + +export function reviewRemoteLabel(serverId: string, alias: string | undefined): string { + return `${alias ?? serverId.slice(0, 8)}: \${path}`; +} + +export class ReviewRemoteHostsService extends Disposable implements IReviewRemoteHostsService { + declare readonly _serviceBrand: undefined; + private readonly hosts = new Map(); + private readonly router = this._register(new ReviewRemoteFileSystemRouter()); + private readonly labels = this._register(new DisposableMap()); + private closing = false; + + constructor( + @IInstantiationService private readonly instantiationService: IInstantiationService, + @IReviewDesktopConnectionService private readonly connection: IReviewDesktopConnectionService, + @ILabelService private readonly labelService: ILabelService, + @ILogService private readonly logService: ILogService, + @IFileService fileService: IFileService, + @ILifecycleService lifecycleService: ILifecycleService, + ) { + super(); + this._register(fileService.registerProvider(Schemas.vscodeRemote, this.router)); + this._register(lifecycleService.onWillShutdown((event) => + event.join(this.closeAll(), { id: "join.reviewRemoteHosts", label: "Disconnecting remote hosts" }))); + } + + async host(serverId: string): Promise { + const authority = reviewRemoteAuthority(serverId); + if (!authority || this.closing) return undefined; + let host = this.hosts.get(authority); + if (!host) { + host = new ReviewRemoteHost(serverId, authority, (target) => this.open(target), () => this.endpoint(serverId), this.logService); + this.hosts.set(authority, host); + } + return (await host.connect()) ? host : undefined; + } + + private async open(host: ReviewRemoteHost): Promise { + const endpoint = await this.connection.getRemoteLanguageEndpoint(host.serverId); + if (!endpoint || this.closing) return undefined; + void this.label(host); + const session = this.instantiationService.createInstance(ReviewRemoteSession, host, endpoint, this.router); + try { + await session.start(); + return session; + } catch (error) { + session.dispose(); + throw error; + } + } + + private async endpoint(serverId: string): Promise { + const endpoint = this.closing ? undefined : await this.connection.getRemoteLanguageEndpoint(serverId); + return endpoint && `${endpoint.host}:${endpoint.port}:${hash(endpoint.connectionToken)}`; + } + + private async label(host: ReviewRemoteHost): Promise { + const states = await this.connection.readRemoteHosts().catch(() => []); + const alias = states.find((state) => state.serverId === host.serverId)?.alias; + if (this.closing) return; + this.labels.set(host.authority, this.labelService.registerFormatter({ + scheme: Schemas.vscodeRemote, + authority: host.authority, + formatting: { label: reviewRemoteLabel(host.serverId, alias), separator: "/" }, + })); + } + + private async closeAll(): Promise { + this.closing = true; + const hosts = [...this.hosts.values()]; + this.hosts.clear(); + await Promise.all(hosts.map((host) => host.close())); + } + + override dispose(): void { + this.closing = true; + for (const host of this.hosts.values()) host.dispose(); + this.hosts.clear(); + super.dispose(); + } +} + +registerSingleton(IReviewRemoteHostsService, ReviewRemoteHostsService, InstantiationType.Eager); + +class ReviewRemoteHostsStartup { + static readonly ID = "workbench.contrib.reviewRemoteHosts"; + constructor(@IReviewRemoteHostsService _hosts: IReviewRemoteHostsService) { } +} +registerWorkbenchContribution2(ReviewRemoteHostsStartup.ID, ReviewRemoteHostsStartup, WorkbenchPhase.BlockStartup); diff --git a/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteScope.test.ts b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteScope.test.ts new file mode 100644 index 000000000..1fc9263a1 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteScope.test.ts @@ -0,0 +1,214 @@ +import assert from "node:assert/strict"; +import { createRequire, registerHooks } from "node:module"; +import test from "node:test"; +import { Emitter, Event } from "../../../base/common/event.js"; +import { URI } from "../../../base/common/uri.js"; +import { Range } from "../../../editor/common/core/range.js"; +import { SymbolKind, type DocumentSymbol, type Hover } from "../../../editor/common/languages.js"; +import { ILanguageFeatureDebounceService } from "../../../editor/common/services/languageFeatureDebounce.js"; +import { ILanguageFeaturesService } from "../../../editor/common/services/languageFeatures.js"; +import { LanguageFeaturesService } from "../../../editor/common/services/languageFeaturesService.js"; +import { IModelService } from "../../../editor/common/services/model.js"; +import { ITextModelService } from "../../../editor/common/services/resolverService.js"; +import { IOutlineModelService, OutlineModelService } from "../../../editor/contrib/documentSymbols/browser/outlineModel.js"; +import { CommandsRegistry, ICommandService } from "../../../platform/commands/common/commands.js"; +import { IExtensionStorageService } from "../../../platform/extensionManagement/common/extensionStorage.js"; +import { IFileService } from "../../../platform/files/common/files.js"; +import { SyncDescriptor } from "../../../platform/instantiation/common/descriptors.js"; +import { InstantiationService } from "../../../platform/instantiation/common/instantiationService.js"; +import { ServiceCollection } from "../../../platform/instantiation/common/serviceCollection.js"; +import { ILogService, NullLogService } from "../../../platform/log/common/log.js"; +import { MarkerService } from "../../../platform/markers/common/markerService.js"; +import { IMarkerService, MarkerSeverity } from "../../../platform/markers/common/markers.js"; +import type { IWorkspaceSymbol } from "../../../workbench/contrib/search/common/search.js"; +import { IExtensionService } from "../../../workbench/services/extensions/common/extensions.js"; +import type { IRemoteAuthorityResolverService } from "../../../platform/remote/common/remoteAuthorityResolver.js"; +import { reviewRemoteAuthority } from "./reviewRemoteAuthority.js"; +import { + ReviewRemoteWorkspace, + reviewRemoteExtensionStorage, + reviewRemoteMarkerService, + reviewRemoteModelService, + reviewRemoteResolver, + reviewRemoteScope, +} from "./reviewRemoteScope.js"; + +const A = "whiteboard+aaaa-1111"; +const B = "whiteboard+bbbb-2222"; +const onA = URI.parse(`vscode-remote://${A}/home/dev/proj/b.ts`); +const onB = URI.parse(`vscode-remote://${B}/home/dev/proj/b.ts`); +const laptop = URI.file("/home/dev/proj/b.ts"); + +test("the authority is whiteboard+ in lower case, and an id that is not one is refused", () => { + assert.equal(reviewRemoteAuthority("3480C31A-77f0-4d6e-9a53-1b2c3d4e5f60"), "whiteboard+3480c31a-77f0-4d6e-9a53-1b2c3d4e5f60"); + assert.equal(URI.parse(`vscode-remote://${reviewRemoteAuthority("ABC-1")}/x`).authority, "whiteboard+abc-1"); + for (const id of ["", "a/b", "a@b", "a:1", "a b", "wb+x"]) assert.equal(reviewRemoteAuthority(id), undefined, id); +}); + +test("a host's model service shows it only its own remote models", () => { + const models = [onA, onB, laptop].map((uri) => ({ uri })); + const added = new Emitter<{ uri: URI }>(); + const base = { + getModels: () => models, + getModel: (uri: URI) => models.find((model) => model.uri.toString() === uri.toString()) ?? null, + onModelAdded: added.event, + onModelRemoved: new Emitter().event, + onModelLanguageChanged: new Emitter().event, + createModel: () => "the window's", + } as unknown as IModelService; + const scoped = reviewRemoteModelService(base, A); + + assert.deepEqual(scoped.getModels().map((model) => model.uri.toString()), [onA.toString()]); + assert.equal(scoped.getModel(onA)?.uri.toString(), onA.toString()); + assert.equal(scoped.getModel(onB), null); + assert.equal(scoped.getModel(laptop), null); + assert.equal((scoped as unknown as { createModel(): string }).createModel(), "the window's"); + + const seen: string[] = []; + const listener = scoped.onModelAdded((model) => seen.push(model.uri.toString())); + for (const uri of [onB, laptop, onA]) added.fire({ uri }); + listener.dispose(); + assert.deepEqual(seen, [onA.toString()]); +}); + +test("a host's markers use its own owner names, and it sees and changes markers of its own files only", () => { + const markers = new MarkerService(); + const a = reviewRemoteMarkerService(markers, A); + const b = reviewRemoteMarkerService(markers, B); + const marker = (message: string) => ({ message, severity: MarkerSeverity.Error, startLineNumber: 1, startColumn: 1, endLineNumber: 1, endColumn: 2 }); + markers.changeOne("typescript", laptop, [marker("laptop")]); + a.changeOne("typescript", onA, [marker("a")]); + b.changeOne("typescript", onB, [marker("b")]); + a.changeOne("typescript", laptop, [marker("a on the laptop")]); + a.changeOne("typescript", onB, [marker("a on b")]); + assert.deepEqual(markers.read({ resource: laptop }).map((m) => m.message), ["laptop"]); + assert.deepEqual(markers.read({ resource: onB }).map((m) => m.message), ["b"]); + + const changes: string[][] = []; + const listener = b.onMarkerChanged((resources) => changes.push(resources.map(String))); + a.changeAll("typescript", []); + listener.dispose(); + + assert.deepEqual(markers.read().map((m) => [m.owner, m.message]).sort(), [ + ["typescript", "laptop"], + [`${B}/typescript`, "b"], + ]); + assert.deepEqual(b.read({ owner: "typescript" }).map((m) => m.message), ["b"]); + assert.deepEqual(b.read({ resource: laptop }), []); + assert.deepEqual(changes, [], "b is not told about a's files"); + markers.dispose(); +}); + +test("each host keeps its own workspace state for an extension, and shares global state with the window", () => { + const storage = new Map(); + const base = { + getExtensionStateRaw: (id: string, global: boolean) => storage.get(`${global}:${id}`), + getExtensionState: (id: string, global: boolean) => JSON.parse(storage.get(`${global}:${id}`) ?? "null") ?? undefined, + setExtensionState: (id: string, state: object, global: boolean) => storage.set(`${global}:${id}`, JSON.stringify(state)), + } as unknown as IExtensionStorageService; + const a = reviewRemoteExtensionStorage(base, A); + const b = reviewRemoteExtensionStorage(base, B); + base.setExtensionState("pub.ext", { project: "laptop" }, false); + a.setExtensionState("pub.ext", { project: "a" }, false); + b.setExtensionState("pub.ext", { project: "b" }, false); + a.setExtensionState("pub.ext", { theme: "a" }, true); + + assert.deepEqual([base, a, b].map((service) => service.getExtensionState("pub.ext", false)), [{ project: "laptop" }, { project: "a" }, { project: "b" }]); + assert.equal(b.getExtensionStateRaw("pub.ext", false), JSON.stringify({ project: "b" })); + assert.deepEqual([base, b].map((service) => service.getExtensionState("pub.ext", true)), [{ theme: "a" }, { theme: "a" }]); +}); + +test("a host's resolver answers for its own authority with a fresh address, and leaves others to the window", async () => { + let port = 4000; + const base = { + resolveAuthority: async (name: string) => ({ authority: { authority: `window:${name}` } }), + getConnectionData: () => "window", + } as unknown as IRemoteAuthorityResolverService; + const resolver = reviewRemoteResolver(base, A, async () => ({ + connectTo: { type: 0, host: "127.0.0.1", port: port++ } as never, + connectionToken: "secret", + })); + + assert.equal(resolver.getConnectionData(A), null); + assert.equal(((await resolver.resolveAuthority(A)).authority.connectTo as { port: number }).port, 4000); + const second = await resolver.resolveAuthority(A); + assert.deepEqual([second.authority.authority, (second.authority.connectTo as { port: number }).port, second.authority.connectionToken], [A, 4001, "secret"]); + assert.equal((resolver.getConnectionData(A)?.connectTo as { port: number }).port, 4001); + assert.equal((await resolver.resolveAuthority(B)).authority.authority, `window:${B}`); + assert.equal(resolver.getConnectionData(B), "window"); +}); + +test("a host's workspace holds each root while any caller holds it", () => { + const workspace = new ReviewRemoteWorkspace("whiteboard-remote-a"); + const events: string[] = []; + workspace.onDidChangeWorkspaceFolders((e) => events.push(`+${e.added.map((f) => f.uri.path)} -${e.removed.map((f) => f.uri.path)}`)); + const root = URI.parse(`vscode-remote://${A}/home/dev/proj`); + const other = URI.parse(`vscode-remote://${A}/home/dev/other`); + const first = workspace.add(root); + const second = workspace.add(root); + const third = workspace.add(other); + assert.deepEqual(workspace.getWorkspace().folders.map((f) => [f.uri.path, f.index]), [["/home/dev/proj", 0], ["/home/dev/other", 1]]); + assert.equal(workspace.getWorkspaceFolder(onA)?.uri.path, "/home/dev/proj"); + assert.equal(workspace.isInsideWorkspace(onB), false); + first.dispose(); + first.dispose(); + assert.equal(workspace.getWorkspace().folders.length, 2); + second.dispose(); + assert.deepEqual(workspace.getWorkspace().folders.map((f) => [f.uri.path, f.index]), [["/home/dev/other", 0]]); + third.dispose(); + assert.deepEqual(events, ["+/home/dev/proj -", "+/home/dev/other -", "+ -/home/dev/proj", "+ -/home/dev/other"]); + workspace.dispose(); +}); + +test("a host's vscode.executeHoverProvider and executeDocumentSymbolProvider run on that host's providers, and its workspace symbols are in its own files", async () => { + const { JSDOM } = createRequire(import.meta.url)("jsdom"); + const dom = new JSDOM(""); + for (const key of ["window", "document", "HTMLElement", "HTMLCanvasElement", "Node", "MutationObserver", "Element", "navigator", "customElements", "UIEvent", "MouseEvent", "KeyboardEvent", "FocusEvent"] as const) Object.defineProperty(globalThis, key, { configurable: true, value: dom.window[key] }); + dom.window.matchMedia = () => ({ matches: false, addEventListener() { }, removeEventListener() { } }) as never; + registerHooks({ load: (url, context, next) => (url.endsWith(".css") ? { format: "module", source: "", shortCircuit: true } : next(url, context)) }); + await import("../../../editor/contrib/hover/browser/getHover.js"); + await import("../../../editor/contrib/documentSymbols/browser/documentSymbols.js"); + const model = { id: "a", uri: onA, getLanguageId: () => "typescript", getVersionId: () => 1, isTooLargeForSyncing: () => false }; + const remote = new LanguageFeaturesService(); + const laptopFeatures = new LanguageFeaturesService(); + const range = new Range(1, 1, 1, 5); + remote.hoverProvider.register({ language: "typescript" }, { provideHover: () => ({ range, contents: [{ value: "from host A" }] }) }); + laptopFeatures.hoverProvider.register("*", { provideHover: () => ({ range, contents: [{ value: "the window's" }] }) }); + const symbol = (name: string) => ({ name, detail: "", kind: SymbolKind.Function, tags: [], range, selectionRange: range }); + remote.documentSymbolProvider.register({ language: "typescript" }, { provideDocumentSymbols: () => [symbol("fromHostA")] }); + laptopFeatures.documentSymbolProvider.register("*", { provideDocumentSymbols: () => [symbol("theWindows")] }); + CommandsRegistry.registerCommand("_executeWorkspaceSymbolProvider", () => [onA, onB, laptop].map((uri) => ({ ...symbol(uri.toString()), location: { uri, range } }))); + const activated: string[] = []; + const window = new ServiceCollection( + [ILogService, new NullLogService()], + [ILanguageFeaturesService, laptopFeatures], + [IModelService, { getModel: (uri: URI) => (uri.toString() === onA.toString() ? model : null), onModelRemoved: Event.None }], + [ITextModelService, { createModelReference: async () => ({ object: { textEditorModel: model }, dispose() { } }) }], + [ILanguageFeatureDebounceService, { for: () => ({ get: () => 0, update: () => 0, default: () => 0 }) }], + [IOutlineModelService, new SyncDescriptor(OutlineModelService)], + [IMarkerService, new MarkerService()], + [IExtensionStorageService, {}], + [IFileService, { onDidFilesChange: Event.None, onDidRunOperation: Event.None }], + [IExtensionService, { activateByEvent: async (event: string) => { activated.push(`window ${event}`); } }], + ); + const parent = new InstantiationService(window, true); + const scope = parent.createChild(parent.invokeFunction((accessor) => reviewRemoteScope({ + authority: A, + extensions: [], + activate: async (event) => { activated.push(`host ${event}`); }, + languageFeatures: remote, + workspace: new ReviewRemoteWorkspace("w"), + resolver: {} as IRemoteAuthorityResolverService, + }, accessor))); + + const commands = scope.invokeFunction((accessor) => accessor.get(ICommandService)); + + const hovers = await commands.executeCommand("_executeHoverProvider", onA, { lineNumber: 1, column: 2 }); + const symbols = await commands.executeCommand("_executeDocumentSymbolProvider", onA); + const workspaceSymbols = await commands.executeCommand("_executeWorkspaceSymbolProvider", ""); + + assert.deepEqual(hovers?.map((hover) => hover.contents.map((content) => content.value)), [["from host A"]]); + assert.deepEqual(symbols?.map((item) => item.name), ["fromHostA"]); + assert.deepEqual(workspaceSymbols?.map((item) => item.location.uri.toString()), [onA.toString()]); + assert.deepEqual(activated, ["_executeHoverProvider", "_executeDocumentSymbolProvider", "_executeWorkspaceSymbolProvider"].map((id) => `host onCommand:${id}`)); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteScope.ts b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteScope.ts new file mode 100644 index 000000000..e9372eb3c --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteScope.ts @@ -0,0 +1,220 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Emitter, Event } from "../../../base/common/event.js"; +import { Disposable, type IDisposable } from "../../../base/common/lifecycle.js"; +import type { URI } from "../../../base/common/uri.js"; +import type { ITextModel } from "../../../editor/common/model.js"; +import { ILanguageFeaturesService } from "../../../editor/common/services/languageFeatures.js"; +import { IModelService } from "../../../editor/common/services/model.js"; +import { IOutlineModelService, OutlineModelService } from "../../../editor/contrib/documentSymbols/browser/outlineModel.js"; +import { ICommandService } from "../../../platform/commands/common/commands.js"; +import { IExtensionStorageService } from "../../../platform/extensionManagement/common/extensionStorage.js"; +import type { IExtensionDescription } from "../../../platform/extensions/common/extensions.js"; +import { IFileService } from "../../../platform/files/common/files.js"; +import { SyncDescriptor } from "../../../platform/instantiation/common/descriptors.js"; +import { IInstantiationService, type ServicesAccessor } from "../../../platform/instantiation/common/instantiation.js"; +import { ServiceCollection } from "../../../platform/instantiation/common/serviceCollection.js"; +import { ILogService } from "../../../platform/log/common/log.js"; +import { IMarkerService } from "../../../platform/markers/common/markers.js"; +import { IRemoteAuthorityResolverService, type IRemoteConnectionData, type ResolverResult } from "../../../platform/remote/common/remoteAuthorityResolver.js"; +import { + IWorkspaceContextService, + type IWorkspace, + type IWorkspaceFolder, + type IWorkspaceFoldersChangeEvent, + WorkbenchState, + Workspace, + WorkspaceFolder, +} from "../../../platform/workspace/common/workspace.js"; +import type { IWorkspaceSymbol } from "../../../workbench/contrib/search/common/search.js"; +import { CommandService } from "../../../workbench/services/commands/common/commandService.js"; +import { IExtensionService } from "../../../workbench/services/extensions/common/extensions.js"; +import { ISearchService } from "../../../workbench/services/search/common/search.js"; +import { SearchService } from "../../../workbench/services/search/common/searchService.js"; +import { override, ownsRemoteResource } from "./reviewRemoteAuthority.js"; +import { reviewRemoteExtensionService } from "./reviewRemoteExtensionService.js"; +import { reviewRemoteFileEvents } from "./reviewRemoteFileEvents.js"; + +export function reviewRemoteModelService(base: IModelService, authority: string): IModelService { + const mine = (model: ITextModel) => ownsRemoteResource(authority, model.uri); + return override(base, { + getModels: () => base.getModels().filter(mine), + getModel: (resource: URI) => (ownsRemoteResource(authority, resource) ? base.getModel(resource) : null), + onModelAdded: Event.filter(base.onModelAdded, mine), + onModelRemoved: Event.filter(base.onModelRemoved, mine), + onModelLanguageChanged: Event.filter(base.onModelLanguageChanged, (event) => mine(event.model)), + }); +} + +export function reviewRemoteMarkerService(base: IMarkerService, authority: string): IMarkerService { + const owner = (name: string) => `${authority}/${name}`; + const mine = (resource: URI) => ownsRemoteResource(authority, resource); + return override(base, { + changeOne: (name, resource, markers) => { + if (mine(resource)) base.changeOne(owner(name), resource, markers); + }, + changeAll: (name, data) => base.changeAll(owner(name), data.filter((entry) => mine(entry.resource))), + remove: (name, resources) => base.remove(owner(name), resources.filter(mine)), + read: (filter) => + base.read(filter?.owner ? { ...filter, owner: owner(filter.owner) } : filter).filter((marker) => mine(marker.resource)), + onMarkerChanged: Event.filter( + Event.map(base.onMarkerChanged, (resources) => resources.filter(mine)), + (resources) => resources.length > 0, + ), + }); +} + +/** Upstream keeps `workspaceState` per window, so each host keeps its own under its authority; `globalState` stays shared. */ +export function reviewRemoteExtensionStorage(base: IExtensionStorageService, authority: string): IExtensionStorageService { + const key = (extension: Parameters[0], global: boolean) => + global ? extension : `${authority}/${typeof extension === "string" ? extension : extension.identifier.id}`; + return override(base, { + getExtensionState: (extension, global) => base.getExtensionState(key(extension, global), global), + getExtensionStateRaw: (extension, global) => base.getExtensionStateRaw(key(extension, global), global), + setExtensionState: (extension, state, global) => base.setExtensionState(key(extension, global), state, global), + }); +} + +export function reviewRemoteResolver( + base: IRemoteAuthorityResolverService, + authority: string, + address: () => Promise, +): IRemoteAuthorityResolverService { + let last: IRemoteConnectionData | null = null; + return override(base, { + resolveAuthority: async (name: string): Promise => { + if (name !== authority) return base.resolveAuthority(name); + last = await address(); + return { authority: { authority, ...last }, options: {} }; + }, + getConnectionData: (name: string) => (name === authority ? last : base.getConnectionData(name)), + }); +} + +/** Every host registers into the one workspace-symbol registry, so a host keeps only its own files' symbols. */ +class ReviewRemoteCommandService extends CommandService { + constructor( + private readonly authority: string, + @IInstantiationService instantiationService: IInstantiationService, + @IExtensionService extensionService: IExtensionService, + @ILogService logService: ILogService, + ) { + super(instantiationService, extensionService, logService); + } + + override async executeCommand(id: string, ...args: unknown[]): Promise { + const result = await super.executeCommand(id, ...args); + if (id !== "_executeWorkspaceSymbolProvider") return result; + return (result as IWorkspaceSymbol[]).filter((symbol) => ownsRemoteResource(this.authority, symbol.location.uri)) as T; + } +} + +export class ReviewRemoteWorkspace extends Disposable implements IWorkspaceContextService { + declare readonly _serviceBrand: undefined; + + private readonly counts = new Map(); + private readonly workspace: Workspace; + private readonly foldersChanged = this._register(new Emitter()); + readonly onDidChangeWorkspaceFolders = this.foldersChanged.event; + readonly onWillChangeWorkspaceFolders = Event.None; + readonly onDidChangeWorkbenchState = Event.None; + readonly onDidChangeWorkspaceName = Event.None; + + constructor(id: string) { + super(); + this.workspace = new Workspace(id, [], false, null, () => false); + } + + add(root: URI): IDisposable { + const key = root.toString(); + const entry = this.counts.get(key); + if (entry) entry.count++; + else { + this.counts.set(key, { uri: root, count: 1 }); + this.update(); + } + let disposed = false; + return { + dispose: () => { + if (disposed) return; + disposed = true; + const current = this.counts.get(key); + if (current && --current.count === 0) { + this.counts.delete(key); + this.update(); + } + }, + }; + } + + private update(): void { + const before = this.workspace.folders; + const after = [...this.counts.values()].map( + ({ uri }, index) => new WorkspaceFolder({ uri, index, name: uri.path.split("/").pop() || uri.path }), + ); + const keys = (folders: readonly IWorkspaceFolder[]) => new Set(folders.map((folder) => folder.uri.toString())); + const [had, has] = [keys(before), keys(after)]; + this.workspace.folders = after; + this.foldersChanged.fire({ + added: after.filter((folder) => !had.has(folder.uri.toString())), + removed: before.filter((folder) => !has.has(folder.uri.toString())), + changed: [], + }); + } + + getCompleteWorkspace(): Promise { + return Promise.resolve(this.workspace); + } + + getWorkspace(): IWorkspace { + return this.workspace; + } + + getWorkbenchState(): WorkbenchState { + return WorkbenchState.WORKSPACE; + } + + getWorkspaceFolder(resource: URI): IWorkspaceFolder | null { + return this.workspace.getFolder(resource); + } + + isCurrentWorkspace(): boolean { + return false; + } + + isInsideWorkspace(resource: URI): boolean { + return !!this.workspace.getFolder(resource); + } + + hasWorkspaceData(): boolean { + return true; + } +} + +export function reviewRemoteScope(input: { + authority: string; + extensions: readonly IExtensionDescription[]; + activate: (event: string) => Promise; + languageFeatures: ILanguageFeaturesService; + workspace: ReviewRemoteWorkspace; + resolver: IRemoteAuthorityResolverService; +}, window: ServicesAccessor): ServiceCollection { + const { authority } = input; + return new ServiceCollection( + [ILanguageFeaturesService, input.languageFeatures], + [IModelService, reviewRemoteModelService(window.get(IModelService), authority)], + [IWorkspaceContextService, input.workspace], + [IMarkerService, reviewRemoteMarkerService(window.get(IMarkerService), authority)], + [IExtensionStorageService, reviewRemoteExtensionStorage(window.get(IExtensionStorageService), authority)], + [ISearchService, new SyncDescriptor(SearchService)], + [IRemoteAuthorityResolverService, input.resolver], + [IExtensionService, reviewRemoteExtensionService(window.get(IExtensionService), input.extensions, input.activate)], + // So a host's `vscode.execute*Provider` calls use this scope's registry and models. + [ICommandService, new SyncDescriptor(ReviewRemoteCommandService, [authority])], + [IOutlineModelService, new SyncDescriptor(OutlineModelService)], + [IFileService, reviewRemoteFileEvents(window.get(IFileService), authority)], + ); +} diff --git a/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteWindowExtensionManagement.test.ts b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteWindowExtensionManagement.test.ts new file mode 100644 index 000000000..f25962226 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteWindowExtensionManagement.test.ts @@ -0,0 +1,35 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import type { IChannel } from "../../../base/parts/ipc/common/ipc.js"; +import type { IInstantiationService } from "../../../platform/instantiation/common/instantiation.js"; +import type { ISharedProcessService } from "../../../platform/ipc/electron-browser/services.js"; +import type { ILabelService } from "../../../platform/label/common/label.js"; +import type { IRemoteAgentService } from "../../../workbench/services/remote/common/remoteAgentService.js"; +import { ReviewExtensionManagementServerService, withoutRemoteExtensionManagement } from "./reviewRemoteWindowExtensionManagement.js"; + +function remote(authority: string) { + const asked: string[] = []; + const channel = (name: string) => ({ name, call: async () => { throw new Error(`Unknown channel: ${name}`); }, listen: () => { throw new Error(`Unknown channel: ${name}`); } }); + const connection = { remoteAuthority: authority, getChannel: (name: string) => (asked.push(name), channel(name)) }; + return { asked, service: { getConnection: () => connection } as unknown as IRemoteAgentService }; +} + +function serverService(remoteAgentService: IRemoteAgentService) { + const created: unknown[][] = []; + const instantiation = { createInstance: (...args: unknown[]) => (created.push(args), { dispose() { } }) } as unknown as IInstantiationService; + const sharedProcess = { getChannel: (name: string) => ({ name }) } as unknown as ISharedProcessService; + const service = new ReviewExtensionManagementServerService(sharedProcess, remoteAgentService, { getHostLabel: () => "wb-test-a" } as unknown as ILabelService, instantiation); + return { service, remoteChannel: created[1]?.[1] as IChannel }; +} + +test("a whiteboard+ window manages no extensions on its server and asks it nothing", async () => { + const { asked, service: remoteAgent } = remote("whiteboard+abc-1"); + const { service, remoteChannel } = serverService(remoteAgent); + assert.equal(service.remoteExtensionManagementServer?.label, "wb-test-a"); + assert.deepEqual(await remoteChannel.call("getInstalled", []), []); + assert.doesNotThrow(() => remoteChannel.listen("onDidInstallExtensions")(() => { }).dispose()); + await assert.rejects(remoteChannel.call("installFromGallery", []), /not managed from a Source window \(installFromGallery\)/); + await withoutRemoteExtensionManagement(remoteAgent).getConnection()!.getChannel("extensionGalleryManifest").call("setExtensionGalleryManifest", [null]); + withoutRemoteExtensionManagement(remoteAgent).getConnection()!.getChannel("remoteFilesystem"); + assert.deepEqual(asked, ["remoteFilesystem"]); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteWindowExtensionManagement.ts b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteWindowExtensionManagement.ts new file mode 100644 index 000000000..6b72549a0 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewRemoteWindowExtensionManagement.ts @@ -0,0 +1,46 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Event } from "../../../base/common/event.js"; +import type { IChannel } from "../../../base/parts/ipc/common/ipc.js"; +import { WorkbenchExtensionGalleryManifestService } from "../../../workbench/services/extensionManagement/electron-browser/extensionGalleryManifestService.js"; +import { ExtensionManagementServerService } from "../../../workbench/services/extensionManagement/electron-browser/extensionManagementServerService.js"; +import type { IRemoteAgentService } from "../../../workbench/services/remote/common/remoteAgentService.js"; +import { isReviewRemoteAuthority, override } from "./reviewRemoteAuthority.js"; + +const UNSERVED = new Set(["extensions", "extensionGalleryManifest"]); + +/** The trimmed server manages no extensions. */ +const NO_EXTENSIONS: IChannel = { + call: async (command: string): Promise => { + if (command === "getInstalled") return [] as T; + if (command === "setExtensionGalleryManifest") return undefined as T; + throw new Error(`Extensions on a Whiteboard remote are not managed from a Source window (${command}).`); + }, + listen: () => Event.None as Event, +}; + +export function withoutRemoteExtensionManagement(remoteAgentService: IRemoteAgentService): IRemoteAgentService { + const connection = remoteAgentService.getConnection(); + if (!connection || !isReviewRemoteAuthority(connection.remoteAuthority)) return remoteAgentService; + const quiet = override(connection, { + getChannel: (name: string) => (UNSERVED.has(name) ? NO_EXTENSIONS : connection.getChannel(name)) as T, + }); + return override(remoteAgentService, { getConnection: () => quiet }); +} + +export class ReviewExtensionManagementServerService extends ExtensionManagementServerService { + constructor(...args: ConstructorParameters) { + args[1] = withoutRemoteExtensionManagement(args[1]); + super(...args); + } +} + +export class ReviewExtensionGalleryManifestService extends WorkbenchExtensionGalleryManifestService { + constructor(...args: ConstructorParameters) { + args[5] = withoutRemoteExtensionManagement(args[5]); + super(...args); + } +} diff --git a/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewSourceWindowHostState.test.ts b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewSourceWindowHostState.test.ts new file mode 100644 index 000000000..4609c22b7 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewSourceWindowHostState.test.ts @@ -0,0 +1,121 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from "node:assert/strict"; +import test from "node:test"; +import { Emitter } from "../../../base/common/event.js"; +import type { IMainProcessService } from "../../../platform/ipc/common/mainProcessService.js"; +import type { INotification, INotificationService } from "../../../platform/notification/common/notification.js"; +import { PersistentConnectionEventType, type PersistentConnectionEvent } from "../../../platform/remote/common/remoteAgentConnection.js"; +import type { IStorageService } from "../../../platform/storage/common/storage.js"; +import type { IWorkbenchEnvironmentService } from "../../../workbench/services/environment/common/environmentService.js"; +import type { IRemoteAgentConnection, IRemoteAgentService } from "../../../workbench/services/remote/common/remoteAgentService.js"; +import type { IStatusbarEntry, IStatusbarService } from "../../../workbench/services/statusbar/browser/statusbar.js"; +import { ReviewSourceWindowHostState, showSourceWindowHostState } from "./reviewSourceWindowHostState.js"; +import type { ReviewWindowHosts } from "./reviewWindowAuthorityResolver.js"; + +const OFFLINE = "devbox — offline, reconnecting…"; + +function fakes() { + const entries: IStatusbarEntry[] = []; + const shown: INotification[] = []; + let closed = 0; + const statusbar = { + addEntry: (entry: IStatusbarEntry) => { + entries.push(entry); + return { update: (next: IStatusbarEntry) => void entries.push(next), dispose: () => {} }; + }, + } as unknown as IStatusbarService; + const notifications = { + notify: (notification: INotification) => { + shown.push(notification); + return { close: () => void closed++ }; + }, + } as unknown as INotificationService; + let state = "online"; + const hosts: ReviewWindowHosts = { endpoint: async () => undefined, state: async () => ({ alias: "devbox", state }) }; + const connection = new Emitter(); + return { + statusbar, + notifications, + hosts, + connection: { onDidStateChange: connection.event } as unknown as IRemoteAgentConnection, + fire: (type: PersistentConnectionEventType) => connection.fire({ type } as PersistentConnectionEvent), + set: (next: string) => void (state = next), + last: () => entries.at(-1), + shown, + closed: () => closed, + }; +} + +const tick = () => new Promise((resolve) => setTimeout(resolve, 30)); + +test("shows the alias while online, the offline state within one poll, and restores it", async () => { + const f = fakes(); + const watch = showSourceWindowHostState("abc-1", f.hosts, f.statusbar, f.notifications, f.connection, undefined, 5); + try { + await tick(); + assert.equal(f.last()?.text, "devbox"); + assert.notEqual(f.last()?.kind, "warning"); + assert.equal(f.shown.length, 0); + + f.set("offline"); + await tick(); + assert.equal(f.last()?.text, OFFLINE); + assert.equal(f.last()?.kind, "warning"); + assert.deepEqual(f.shown.map((n) => [n.message, n.actions]), [["devbox is offline. The window reconnects when it is back.", undefined]]); + + f.fire(PersistentConnectionEventType.ConnectionLost); + f.set("online"); + await tick(); + assert.equal(f.last()?.text, OFFLINE, "the window's own connection is still down"); + assert.equal(f.closed(), 0); + + f.fire(PersistentConnectionEventType.ConnectionGain); + assert.equal(f.last()?.text, "devbox"); + assert.notEqual(f.last()?.kind, "warning"); + assert.equal(f.closed(), 1); + + f.set("unreachable"); + await tick(); + assert.equal(f.last()?.text, OFFLINE); + assert.equal(f.shown.length, 1, "no second notification in the same window"); + } finally { + watch.dispose(); + } +}); + +test("a window on another authority shows nothing and never asks main", () => { + for (const remoteAuthority of [undefined, "ssh-remote+x", "Whiteboard+ABC"]) { + const contribution = new ReviewSourceWindowHostState( + { remoteAuthority } as IWorkbenchEnvironmentService, + { getChannel: () => assert.fail("asked main") } as unknown as IMainProcessService, + { addEntry: () => assert.fail("added an entry") } as unknown as IStatusbarService, + { notify: () => assert.fail("notified") } as unknown as INotificationService, + { getConnection: () => null } as unknown as IRemoteAgentService, + { getObject: () => assert.fail("read storage") } as unknown as IStorageService, + ); + contribution.dispose(); + } +}); + +test("a window restored before main knows its host's alias names it by the alias it stored when it opened", async () => { + const f = fakes(); + const contribution = new ReviewSourceWindowHostState( + { remoteAuthority: "whiteboard+3f454168-aaaa" } as IWorkbenchEnvironmentService, + { getChannel: () => ({ call: async (command: string) => (command === "getRemoteHostState" ? { state: "offline" } : undefined) }) } as unknown as IMainProcessService, + f.statusbar, + f.notifications, + { getConnection: () => null } as unknown as IRemoteAgentService, + { getObject: () => ({ side: "head", title: "Remote source", alias: "wb-test-a" }) } as unknown as IStorageService, + ); + try { + await tick(); + assert.equal(f.last()?.text, "wb-test-a — offline, reconnecting…"); + assert.equal(f.shown[0]?.message, "wb-test-a is offline. The window reconnects when it is back."); + } finally { + contribution.dispose(); + } +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewSourceWindowHostState.ts b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewSourceWindowHostState.ts new file mode 100644 index 000000000..d3cb5fd97 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewSourceWindowHostState.ts @@ -0,0 +1,104 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Disposable, DisposableStore, toDisposable, type IDisposable } from "../../../base/common/lifecycle.js"; +import Severity from "../../../base/common/severity.js"; +import { IMainProcessService } from "../../../platform/ipc/common/mainProcessService.js"; +import { INotificationService, type INotificationHandle } from "../../../platform/notification/common/notification.js"; +import { PersistentConnectionEventType } from "../../../platform/remote/common/remoteAgentConnection.js"; +import type { IWorkbenchContribution } from "../../../workbench/common/contributions.js"; +import { IWorkbenchEnvironmentService } from "../../../workbench/services/environment/common/environmentService.js"; +import { IRemoteAgentService, type IRemoteAgentConnection } from "../../../workbench/services/remote/common/remoteAgentService.js"; +import { IStatusbarService, StatusbarAlignment, type IStatusbarEntry, type IStatusbarEntryAccessor } from "../../../workbench/services/statusbar/browser/statusbar.js"; +import { IStorageService, StorageScope } from "../../../platform/storage/common/storage.js"; +import { REVIEW_DESKTOP_CHANNEL } from "../../common/reviewDesktopBootstrap.js"; +import { isReviewSourceTitle, REVIEW_SOURCE_TITLE_KEY } from "../configuration/reviewSourceWindowConfiguration.js"; +import { isReviewRemoteAuthority } from "./reviewRemoteAuthority.js"; +import { reviewWindowHosts, type ReviewWindowHosts } from "./reviewWindowAuthorityResolver.js"; + +export const HOST_STATE_POLL_MS = 2_000; + +/** Shows a Source window's host as main sees it: its alias, or that it is offline and the window is reconnecting. */ +export function showSourceWindowHostState( + serverId: string, + hosts: ReviewWindowHosts, + statusbar: IStatusbarService, + notifications: INotificationService, + connection: Pick | null, + storedAlias: () => string | undefined = () => undefined, + every = HOST_STATE_POLL_MS, +): IDisposable { + const store = new DisposableStore(); + let mainAlias: string | undefined; + let known = false; + let online = true; + let lost = false; + let notified = false; + let entry: IStatusbarEntryAccessor | undefined; + let notice: INotificationHandle | undefined; + const render = () => { + if (!known) return; + const alias = mainAlias ?? storedAlias() ?? serverId.slice(0, 8); + const offline = !online || lost; + const text = offline ? `${alias} — offline, reconnecting…` : alias; + const props: IStatusbarEntry = { name: "Remote Host", text, ariaLabel: text, kind: offline ? "warning" : undefined }; + if (entry) entry.update(props); + else entry = store.add(statusbar.addEntry(props, "review.sourceWindow.host", StatusbarAlignment.LEFT, Number.MAX_VALUE)); + if (offline && !notified) { + notified = true; + notice = notifications.notify({ severity: Severity.Warning, message: `${alias} is offline. The window reconnects when it is back.` }); + } else if (!offline) { + notice?.close(); + notice = undefined; + } + }; + if (connection) { + store.add(connection.onDidStateChange(({ type }) => { + lost = type !== PersistentConnectionEventType.ConnectionGain; + render(); + })); + } + let stopped = false; + let handle: ReturnType | undefined; + const tick = async () => { + const state = await hosts.state(serverId).catch(() => undefined); + if (stopped) return; + if (state) { + mainAlias = state.alias ?? mainAlias; + online = state.state === "online"; + known = true; + render(); + } + handle = setTimeout(() => void tick(), every); + }; + void tick(); + store.add(toDisposable(() => { + stopped = true; + clearTimeout(handle); + })); + return store; +} + +export class ReviewSourceWindowHostState extends Disposable implements IWorkbenchContribution { + static readonly ID = "review.sourceWindow.hostState"; + + constructor( + @IWorkbenchEnvironmentService environment: IWorkbenchEnvironmentService, + @IMainProcessService mainProcess: IMainProcessService, + @IStatusbarService statusbar: IStatusbarService, + @INotificationService notifications: INotificationService, + @IRemoteAgentService remoteAgent: IRemoteAgentService, + @IStorageService storage: IStorageService, + ) { + super(); + const authority = environment.remoteAuthority; + if (!authority || !isReviewRemoteAuthority(authority)) return; + const hosts = reviewWindowHosts(mainProcess.getChannel(REVIEW_DESKTOP_CHANNEL)); + this._register(showSourceWindowHostState(authority.slice("whiteboard+".length), hosts, statusbar, notifications, remoteAgent.getConnection(), () => { + const title = storage.getObject(REVIEW_SOURCE_TITLE_KEY, StorageScope.WORKSPACE); + return isReviewSourceTitle(title) ? title.alias : undefined; + })); + } +} diff --git a/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewWindowAuthorityResolver.test.ts b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewWindowAuthorityResolver.test.ts new file mode 100644 index 000000000..05a4b6984 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewWindowAuthorityResolver.test.ts @@ -0,0 +1,185 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from "node:assert/strict"; +import test from "node:test"; +import { URI } from "../../../base/common/uri.js"; +import type { IChannel } from "../../../base/parts/ipc/common/ipc.js"; +import { + type IRemoteAuthorityResolverService, + RemoteAuthorityResolverError, + RemoteAuthorityResolverErrorCode, + RemoteConnectionType, + type ResolverResult, +} from "../../../platform/remote/common/remoteAuthorityResolver.js"; +import type { ReviewGatewayHostState } from "../../common/reviewProtocol.js"; +import { remoteHostState } from "../../electron-main/remote/reviewRemoteHosts.js"; +import { reloadWhenOnline, reviewWindowAuthorityResolver, reviewWindowHosts, type ReviewWindowHosts } from "./reviewWindowAuthorityResolver.js"; + +const AUTHORITY = "whiteboard+abc-1"; + +/** Main's answers, in order; the last one repeats. */ +function fakeChannel(answers: { endpoint?: { host: string; port: number; connectionToken: string }; state?: { alias: string; state: string } }[]) { + const calls: [string, unknown][] = []; + let index = 0; + const channel = { + call: async (command: string, arg: unknown) => { + calls.push([command, arg]); + const answer = answers[Math.min(index, answers.length - 1)]; + if (command === "getRemoteLanguageEndpoint") return answer.endpoint; + index++; + return answer.state; + }, + listen: () => { + throw new Error("not used"); + }, + } as unknown as IChannel; + return { channel, calls }; +} + +/** + * Main as it behaves: each phase is what the gateway lists and which alias in + * the setting served the machine; the state goes through main's own lookup. + */ +function fakeMain(phases: { endpoint?: number; states: ReviewGatewayHostState[]; configured?: string }[]) { + let index = 0; + const phase = () => phases[Math.min(index, phases.length - 1)]; + const hosts: ReviewWindowHosts = { + endpoint: async () => { + const port = phase().endpoint; + return port === undefined ? undefined : endpoint(port); + }, + state: async (serverId) => { + const { states, configured } = phase(); + index++; + return remoteHostState(serverId, states, configured); + }, + }; + return { hosts, next: (...more: typeof phases) => phases.splice(0, phases.length, ...more) }; +} + +function fakeBase() { + let provider: ((uri: URI) => Promise) | undefined; + const base = { + resolveAuthority: async (authority: string): Promise => ({ + authority: { authority, connectTo: { type: RemoteConnectionType.WebSocket, host: "base", port: 1 }, connectionToken: undefined }, + }), + _setCanonicalURIProvider: (next: (uri: URI) => Promise) => (provider = next), + getCanonicalURI: (uri: URI) => provider!(uri), + } as unknown as IRemoteAuthorityResolverService; + return base; +} + +const endpoint = (port: number) => ({ host: "127.0.0.1", port, connectionToken: "vscode-token" }); +const connectTo = (result: ResolverResult) => { + const { connectTo, connectionToken } = result.authority; + assert.equal(connectTo.type, RemoteConnectionType.WebSocket); + return { host: connectTo.host, port: connectTo.port, connectionToken }; +}; + +test("a whiteboard+ authority resolves to its host's forward, afresh each time", async () => { + const { channel, calls } = fakeChannel([{ endpoint: endpoint(41000) }]); + const resolver = reviewWindowAuthorityResolver(fakeBase(), reviewWindowHosts(channel)); + + assert.deepEqual(connectTo(await resolver.resolveAuthority(AUTHORITY)), { host: "127.0.0.1", port: 41000, connectionToken: "vscode-token" }); + assert.deepEqual(calls, [["getRemoteLanguageEndpoint", "abc-1"]]); +}); + +test("a resolved authority waits through offline and unreachable for the moved forward", async () => { + const online = { alias: "devbox", serverId: "abc-1", state: "online" as const }; + const { hosts, next } = fakeMain([{ endpoint: 41000, states: [online], configured: "devbox" }]); + const resolver = reviewWindowAuthorityResolver(fakeBase(), hosts, { wait: 20, poll: 5 }); + assert.equal(connectTo(await resolver.resolveAuthority(AUTHORITY)).port, 41000); + + // The gateway keeps the id while offline and drops it with Desktop's problem report; each phase outlasts the wait. + const offline = Array.from({ length: 6 }, () => ({ states: [{ ...online, state: "offline" as const }], configured: "devbox" })); + const unreachable = Array.from({ length: 6 }, () => ({ states: [{ alias: "devbox", state: "unreachable" as const }], configured: "devbox" })); + next(...offline, ...unreachable, { endpoint: 42000, states: [online], configured: "devbox" }); + assert.equal(connectTo(await resolver.resolveAuthority(AUTHORITY)).port, 42000); +}); + +test("a resolved authority whose alias left the setting fails after the wait", async () => { + const { hosts, next } = fakeMain([{ endpoint: 41000, states: [{ alias: "devbox", serverId: "abc-1", state: "online" }], configured: "devbox" }]); + const resolver = reviewWindowAuthorityResolver(fakeBase(), hosts, { wait: 20, poll: 5 }); + await resolver.resolveAuthority(AUTHORITY); + + next({ states: [{ alias: "devbox", state: "unreachable" }] }); + const started = Date.now(); + await assert.rejects(resolver.resolveAuthority(AUTHORITY), (error: RemoteAuthorityResolverError) => error._code === RemoteAuthorityResolverErrorCode.NotAvailable); + assert.ok(Date.now() - started >= 20); +}); + +test("a resolved authority whose machine is gone after the wait fails; one still known keeps waiting", async () => { + const answers: Parameters[0] = [{ endpoint: endpoint(41000) }]; + const { channel } = fakeChannel(answers); + const resolver = reviewWindowAuthorityResolver(fakeBase(), reviewWindowHosts(channel), { wait: 20, poll: 5 }); + await resolver.resolveAuthority(AUTHORITY); + + answers.splice(0, 1, {}); + await assert.rejects(resolver.resolveAuthority(AUTHORITY), (error: RemoteAuthorityResolverError) => error._code === RemoteAuthorityResolverErrorCode.NotAvailable); + + answers.splice(0, 1, ...Array.from({ length: 10 }, () => ({ state: { alias: "devbox", state: "offline" } })), { endpoint: endpoint(43000) }); + assert.equal(connectTo(await resolver.resolveAuthority(AUTHORITY)).port, 43000); +}); + +test("a connecting host is waited for, and resolves once its forward appears", async () => { + const connecting = { state: { alias: "devbox", state: "connecting" } }; + const { channel } = fakeChannel([connecting, connecting, { endpoint: endpoint(43000) }]); + const resolver = reviewWindowAuthorityResolver(fakeBase(), reviewWindowHosts(channel), { wait: 1_000, poll: 1 }); + + assert.equal(connectTo(await resolver.resolveAuthority(AUTHORITY)).port, 43000); +}); + +test("before the host list exists, a window waits rather than calling its host offline", async () => { + const { channel } = fakeChannel([{}, {}, { endpoint: endpoint(44000) }]); + const resolver = reviewWindowAuthorityResolver(fakeBase(), reviewWindowHosts(channel), { wait: 1_000, poll: 1 }); + + assert.equal(connectTo(await resolver.resolveAuthority(AUTHORITY)).port, 44000); +}); + +test("a host still connecting after the wait, or not connecting at all, is offline", async () => { + for (const [state, wait] of [["connecting", 20], ["offline", 60_000]] as const) { + const { channel } = fakeChannel([{ state: { alias: "devbox", state } }]); + const resolver = reviewWindowAuthorityResolver(fakeBase(), reviewWindowHosts(channel), { wait, poll: 5 }); + const started = Date.now(); + + await assert.rejects(resolver.resolveAuthority(AUTHORITY), (error: RemoteAuthorityResolverError) => { + assert.ok(error instanceof RemoteAuthorityResolverError); + assert.equal(error._code, RemoteAuthorityResolverErrorCode.NotAvailable); + assert.equal(error._detail, "devbox is offline"); + return true; + }); + assert.ok(Date.now() - started < 1_000); + } +}); + +test("canonical URIs of whiteboard+ are themselves; every other authority is the window's", async () => { + const { channel, calls } = fakeChannel([{ endpoint: endpoint(41000) }]); + const resolver = reviewWindowAuthorityResolver(fakeBase(), reviewWindowHosts(channel)); + resolver._setCanonicalURIProvider(async () => { + throw new Error("no resolver extension"); + }); + + const own = URI.parse(`vscode-remote://${AUTHORITY}/home/dev/repo/f.ts`); + assert.equal(await resolver.getCanonicalURI(own), own); + await assert.rejects(resolver.getCanonicalURI(URI.parse("vscode-remote://ssh-remote+x/f.ts")), /no resolver extension/); + + for (const other of ["ssh-remote+x", "Whiteboard+ABC", "whiteboard+a/b"]) { + assert.equal(connectTo(await resolver.resolveAuthority(other)).host, "base"); + } + assert.deepEqual(calls, []); +}); + +test("a window that could not reach its host reloads once, when the host's forward answers", async () => { + let polls = 0; + const hosts: ReviewWindowHosts = { endpoint: async () => (++polls >= 3 ? endpoint(45000) : undefined), state: async () => undefined }; + let reloads = 0; + const watch = reloadWhenOnline(hosts, "abc-1", () => reloads++, 2); + + await new Promise((resolve) => setTimeout(resolve, 60)); + watch.dispose(); + assert.equal(reloads, 1); + assert.equal(polls, 3); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewWindowAuthorityResolver.ts b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewWindowAuthorityResolver.ts new file mode 100644 index 000000000..7d04cc451 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/review/services/remote/reviewWindowAuthorityResolver.ts @@ -0,0 +1,98 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { timeout } from "../../../base/common/async.js"; +import { toDisposable, type IDisposable } from "../../../base/common/lifecycle.js"; +import type { IChannel } from "../../../base/parts/ipc/common/ipc.js"; +import { + type IRemoteAuthorityResolverService, + RemoteAuthorityResolverError, + RemoteAuthorityResolverErrorCode, + type ResolverResult, + WebSocketRemoteConnection, +} from "../../../platform/remote/common/remoteAuthorityResolver.js"; +import type { ReviewRemoteLanguageEndpoint } from "../reviewDesktopConnectionService.js"; +import { isReviewRemoteAuthority, override } from "./reviewRemoteAuthority.js"; + +/** How long a Source window waits for its host to finish connecting. */ +export const REMOTE_WINDOW_RESOLVE_WAIT_MS = 60_000; + +export interface ReviewWindowHosts { + endpoint(serverId: string): Promise; + state(serverId: string): Promise<{ alias?: string; state: string } | undefined>; +} + +export function parseRemoteLanguageEndpoint(endpoint: unknown): ReviewRemoteLanguageEndpoint | undefined { + if (typeof endpoint !== "object" || endpoint === null) return undefined; + const { host, port, connectionToken } = endpoint as Record; + return typeof host === "string" && Number.isInteger(port) && typeof connectionToken === "string" + ? { host, port: port as number, connectionToken } + : undefined; +} + +/** Reloads a window that could not reach its host, once the host's forward answers. */ +export function reloadWhenOnline(hosts: ReviewWindowHosts, serverId: string, reload: () => void, every = 5_000): IDisposable { + let done = false; + const check = async () => { + const endpoint = await hosts.endpoint(serverId).catch(() => undefined); + if (!endpoint || done) return; + done = true; + clearInterval(handle); + reload(); + }; + const handle = setInterval(() => void check(), every); + return toDisposable(() => { + done = true; + clearInterval(handle); + }); +} + +export function reviewWindowHosts(channel: IChannel): ReviewWindowHosts { + return { + endpoint: async (serverId) => parseRemoteLanguageEndpoint(await channel.call("getRemoteLanguageEndpoint", serverId)), + state: async (serverId) => { + const state: unknown = await channel.call("getRemoteHostState", serverId); + if (typeof state !== "object" || state === null) return undefined; + const { alias, state: name } = state as Record; + return typeof name === "string" ? { alias: typeof alias === "string" ? alias : undefined, state: name } : undefined; + }, + }; +} + +/** Resolves `whiteboard+` to the host's current forward; other authorities go to `base`. */ +export function reviewWindowAuthorityResolver( + base: IRemoteAuthorityResolverService, + hosts: ReviewWindowHosts, + { wait = REMOTE_WINDOW_RESOLVE_WAIT_MS, poll = 1_000 } = {}, +): IRemoteAuthorityResolverService { + const resolved = new Set(); + const resolve = async (authority: string): Promise => { + const serverId = authority.slice("whiteboard+".length); + const again = resolved.has(authority); + const deadline = Date.now() + wait; + for (;;) { + const endpoint = await hosts.endpoint(serverId); + if (endpoint) { + resolved.add(authority); + const { host, port, connectionToken } = endpoint; + return { authority: { authority, connectTo: new WebSocketRemoteConnection(host, port), connectionToken }, options: {} }; + } + const state = await hosts.state(serverId); + const left = deadline - Date.now(); + // A moved forward passes through offline or unreachable; only a machine gone from the setting ends a reconnect. + const waiting = again ? state !== undefined || left > 0 : (state === undefined || state.state === "connecting") && left > 0; + if (!waiting) { + const offline = `${state?.alias ?? serverId.slice(0, 8)} is offline`; + throw new RemoteAuthorityResolverError(offline, RemoteAuthorityResolverErrorCode.NotAvailable, offline); + } + await timeout(left > 0 ? Math.min(poll, left) : poll); + } + }; + return override(base, { + resolveAuthority: (authority) => (isReviewRemoteAuthority(authority) ? resolve(authority) : base.resolveAuthority(authority)), + _setCanonicalURIProvider: (provider) => + base._setCanonicalURIProvider((uri) => (isReviewRemoteAuthority(uri.authority) ? Promise.resolve(uri) : provider(uri))), + }); +} diff --git a/apps/review-desktop/code-oss/src/vs/review/services/reviewCanvasEditorTabsService.test.ts b/apps/review-desktop/code-oss/src/vs/review/services/reviewCanvasEditorTabsService.test.ts index 3476334a2..fbacab108 100644 --- a/apps/review-desktop/code-oss/src/vs/review/services/reviewCanvasEditorTabsService.test.ts +++ b/apps/review-desktop/code-oss/src/vs/review/services/reviewCanvasEditorTabsService.test.ts @@ -4,10 +4,13 @@ *--------------------------------------------------------------------------------------------*/ import assert from "node:assert/strict"; -import test from "node:test"; +import test, { type TestContext } from "node:test"; +import type { VSBuffer } from "../../base/common/buffer.js"; import { Event } from "../../base/common/event.js"; +import { URI } from "../../base/common/uri.js"; import { ReviewCanvasEditorInput } from "../browser/parts/canvas/reviewCanvasEditorInput.js"; +import { apiSourceUri } from "../common/reviewSourceView.js"; import { ReviewCanvasEditorTabsService } from "./reviewCanvasEditorTabsService.js"; async function closeWelcome(updateNeeded: boolean): Promise { @@ -34,6 +37,9 @@ async function closeWelcome(updateNeeded: boolean): Promise { connection as never, {} as never, { warn() {} } as never, + {} as never, + {} as never, + { reviews: [] } as never, ); try { const welcome = await tabs.openWelcome(true); @@ -49,3 +55,78 @@ test("closing Welcome finishes the CLI install update only when one is pending", assert.equal(await closeWelcome(true), 1); assert.equal(await closeWelcome(false), 0); }); + +const HOST = "whiteboard+c0ffee"; +const remote = (path: string) => URI.from({ scheme: "vscode-remote", authority: HOST, path }).toString(); +const view = (generation?: string) => ({ reviewId: "r1", version: 2, generation }); + +function sourceTabs(t: TestContext, answer: (url: URL) => object, reviews: object[] = []) { + const opened: { toOpen: { workspaceUri?: URI; fileUri?: URI; label?: string }[]; options: Record }[] = []; + const written: string[] = []; + t.mock.method(globalThis, "fetch", async (url: string) => Response.json(answer(new URL(url)))); + const tabs = new ReviewCanvasEditorTabsService( + { createInstance: (_ctor: unknown, target: never) => new ReviewCanvasEditorInput(target, {} as never) } as never, + { onDidCloseEditor: Event.None } as never, + {} as never, + { async getConnection() { return { serverUrl: "http://localhost", token: "test" }; } } as never, + { async openWindow(toOpen: never, options: never) { opened.push({ toOpen, options }); } } as never, + { warn() {} } as never, + { async writeFile(resource: URI, content: VSBuffer) { written.push(`${resource.toString()}=${content.toString()}`); } } as never, + { cacheHome: URI.file("/laptop/cache") } as never, + { reviews } as never, + ); + t.after(() => tabs.dispose()); + return { tabs, opened, written }; +} + +const remoteAnswer = { + workspaceUri: `vscode-remote://${HOST}/home/dev/navigator/repo.code-workspace`, + fileUri: "vscode-remote://whiteboard%2Bc0ffee/home/dev/repo/src/a%20b.ts", + remoteAuthority: HOST, +}; + +test("a remote answer opens the host's workspace and file at the position, titled with the review", async (t) => { + const { tabs, opened } = sourceTabs(t, () => remoteAnswer); + tabs.inputFor({ kind: "api", reviewId: "r1", title: "Fix the parser" }); + const resource = apiSourceUri({ view: view(), side: "head", file: "src/a b.ts" }); + assert.equal(await tabs.openSourceEditor({ resource, options: { selection: { startLineNumber: 3, startColumn: 2 } } }), true); + const [{ toOpen, options }] = opened; + assert.deepEqual(toOpen.map(item => (item.workspaceUri ?? item.fileUri)!.toString()), [remote("/home/dev/navigator/repo.code-workspace"), remote("/home/dev/repo/src/a b.ts:3:2")]); + assert.deepEqual(options, { forceNewWindow: true, gotoLineMode: true, diffMode: false, remoteAuthority: HOST, reviewSourceTitle: { side: "head", title: "Fix the parser" } }); +}); + +test("a remote diff opens its empty side from a laptop empty file named by the laptop, never by the host", async (t) => { + for (const file of ["..\\..\\x.ts", "../../x.ts"]) { + const { tabs, opened, written } = sourceTabs(t, url => url.searchParams.has("empty") + ? { workspaceUri: remoteAnswer.workspaceUri, remoteAuthority: HOST, emptySide: true } + : remoteAnswer); + await tabs.openSourceEditor({ + original: { resource: apiSourceUri({ view: view("wt1"), side: "base", file }, true) }, + modified: { resource: apiSourceUri({ view: view("wt1"), side: "head", file }) }, + }); + const empty = URI.file("/laptop/cache/source-empty/empty"); + assert.deepEqual(written, [`${empty.toString()}=`]); + const [{ toOpen, options }] = opened; + assert.deepEqual(toOpen.map(item => (item.workspaceUri ?? item.fileUri)!.toString()), [ + remote("/home/dev/navigator/repo.code-workspace"), + empty.with({ scheme: "vscode-userdata" }).toString(), + remote("/home/dev/repo/src/a b.ts"), + ]); + assert.equal(toOpen.some(item => item.fileUri?.scheme === "file"), false); + assert.deepEqual(options, { forceNewWindow: true, gotoLineMode: true, diffMode: true, remoteAuthority: HOST, reviewSourceTitle: { side: "live", title: "repo" } }); + } +}); + +test("references and the source tree open a remote review's window on its host", async (t) => { + const { tabs, opened } = sourceTabs(t, () => remoteAnswer); + tabs.inputFor({ kind: "api", reviewId: "r1", title: "Fix the parser" }); + await tabs.openSourceReferences(apiSourceUri({ view: view("wt1"), side: "head", file: "src/a b.ts" }), { lineNumber: 4, column: 5 }); + await tabs.openApiSource({ reviewId: "r1", kind: "current" }, "Fix the parser", true); + const [references, tree] = opened; + assert.deepEqual(references.toOpen.map(item => item.workspaceUri!.toString()), [remote("/home/dev/navigator/repo.code-workspace")]); + const { reviewReferencesToShow, ...options } = references.options as { reviewReferencesToShow: { resource: URI; lineNumber: number; column: number } }; + assert.deepEqual(options, { forceNewWindow: true, remoteAuthority: HOST, reviewSourceTitle: { side: "live", title: "Fix the parser" } }); + assert.deepEqual({ ...reviewReferencesToShow, resource: reviewReferencesToShow.resource.toString() }, { resource: remote("/home/dev/repo/src/a b.ts"), lineNumber: 4, column: 5 }); + assert.deepEqual(tree.toOpen.map(item => [item.workspaceUri!.toString(), item.label]), [[remote("/home/dev/navigator/repo.code-workspace"), "Fix the parser"]]); + assert.deepEqual(tree.options, { forceNewWindow: true, remoteAuthority: HOST, reviewSourceTitle: { side: "live", title: "Fix the parser" } }); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/services/reviewCanvasEditorTabsService.ts b/apps/review-desktop/code-oss/src/vs/review/services/reviewCanvasEditorTabsService.ts index 9a4d54687..e8a361d42 100644 --- a/apps/review-desktop/code-oss/src/vs/review/services/reviewCanvasEditorTabsService.ts +++ b/apps/review-desktop/code-oss/src/vs/review/services/reviewCanvasEditorTabsService.ts @@ -3,10 +3,15 @@ * Licensed under the MIT License. See LICENSE in the repository root for license information. *--------------------------------------------------------------------------------------------*/ +import { VSBuffer } from "../../base/common/buffer.js"; import { Event } from "../../base/common/event.js"; +import { Schemas } from "../../base/common/network.js"; import { Disposable } from "../../base/common/lifecycle.js"; +import { basename, joinPath } from "../../base/common/resources.js"; import { URI } from "../../base/common/uri.js"; import type { ITextEditorOptions } from "../../platform/editor/common/editor.js"; +import { IEnvironmentService } from "../../platform/environment/common/environment.js"; +import { IFileService } from "../../platform/files/common/files.js"; import { createDecorator, IInstantiationService } from "../../platform/instantiation/common/instantiation.js"; import { ILogService } from "../../platform/log/common/log.js"; import type { EditorInput } from "../../workbench/common/editor/editorInput.js"; @@ -19,11 +24,20 @@ import { type ReviewCanvasEditorTarget, } from "../browser/parts/canvas/reviewCanvasEditorInput.js"; -import { reviewSourceQuery, type ReviewSourceSelection } from "../common/reviewProtocol.js"; +import { reviewSourceQuery, type ReviewRemoteNavigatorAnswer, type ReviewSourceSelection } from "../common/reviewProtocol.js"; import { REVIEW_LANGUAGE_SOURCE_SCHEME } from "../common/reviewReadonlySource.js"; import { sourceLocation, sourceSelectionIdentity, REVIEW_API_SOURCE_SCHEME } from "../common/reviewSourceView.js"; +import { IReviewApiCatalogService } from "./reviewApiCatalogService.js"; import { IReviewDesktopConnectionService, reviewResponseError } from "./reviewDesktopConnectionService.js"; +type NavigatorAnswer = { workspacePath: string; filePath?: string } | ReviewRemoteNavigatorAnswer; + +function workspace(answer: NavigatorAnswer): { workspaceUri: URI; remote: { remoteAuthority?: string } } { + return "remoteAuthority" in answer + ? { workspaceUri: URI.parse(answer.workspaceUri), remote: { remoteAuthority: answer.remoteAuthority } } + : { workspaceUri: URI.file(answer.workspacePath), remote: {} }; +} + export const IReviewCanvasEditorTabsService = createDecorator( "reviewCanvasEditorTabsService", ); @@ -33,7 +47,7 @@ export interface IReviewCanvasEditorTabsService { inputFor(target: Extract): ReviewCanvasEditorInput; openApiReview(reviewId: string, title: string, active?: boolean): Promise; isActiveReview(reviewId: string): boolean; - openApiSource(selection: ReviewSourceSelection, title: string): Promise; + openApiSource(selection: ReviewSourceSelection, title: string, live: boolean): Promise; openSourceEditor(editor: IUntypedEditorInput): Promise; openSourceReferences(resource: URI, position: { readonly lineNumber: number; readonly column: number }): Promise; openHome(active: boolean): Promise; @@ -59,6 +73,9 @@ export class ReviewCanvasEditorTabsService extends Disposable implements IReview private readonly desktopConnection: IReviewDesktopConnectionService, @IHostService private readonly host: IHostService, @ILogService private readonly logService: ILogService, + @IFileService private readonly files: IFileService, + @IEnvironmentService private readonly environment: IEnvironmentService, + @IReviewApiCatalogService private readonly catalog: IReviewApiCatalogService, ) { super(); this._register( @@ -107,9 +124,10 @@ export class ReviewCanvasEditorTabsService extends Disposable implements IReview return this.openSingleton({ kind: "welcome" }, active); } - async openApiSource(selection: ReviewSourceSelection, title: string): Promise { + async openApiSource(selection: ReviewSourceSelection, title: string, live: boolean): Promise { const result = await this.navigatorWorkspace(selection.reviewId, selection.kind === "version" ? { version: selection.version } : {}); - await this.host.openWindow([{ workspaceUri: URI.file(result.workspacePath), label: title }], { forceNewWindow: true }); + const { workspaceUri, remote } = workspace(result); + await this.host.openWindow([{ workspaceUri, label: title }], { forceNewWindow: true, ...remote, reviewSourceTitle: this.sourceTitle(selection.reviewId, live ? "live" : "head", title, remote) }); } /** Hand source opens to the native workspace before Review creates an editor group. */ @@ -118,27 +136,28 @@ export class ReviewCanvasEditorTabsService extends Disposable implements IReview const resources = diff ? [editor.original.resource, editor.modified.resource] : [isResourceEditorInput(editor) ? editor.resource : undefined]; if (!resources.every((resource): resource is URI => !!resource && [REVIEW_API_SOURCE_SCHEME, REVIEW_LANGUAGE_SOURCE_SCHEME].includes(resource.scheme))) return false; const destinations = await Promise.all(resources.map(resource => this.sourceDestination(resource))); + const { workspaceUri, remote, reviewSourceTitle } = destinations[destinations.length - 1]; + const selection = !diff ? (editor.options as ITextEditorOptions | undefined)?.selection : undefined; await this.host.openWindow([ - { workspaceUri: destinations[destinations.length - 1].workspaceUri }, - ...destinations.map(({ filePath }) => { - const selection = !diff ? (editor.options as ITextEditorOptions | undefined)?.selection : undefined; - return { fileUri: URI.file(selection ? `${filePath}:${selection.startLineNumber}:${selection.startColumn ?? 1}` : filePath) }; - }), - ], { forceNewWindow: true, gotoLineMode: true, diffMode: diff }); + { workspaceUri }, + ...destinations.map(({ fileUri }) => ({ fileUri: selection ? fileUri.with({ path: `${fileUri.path}:${selection.startLineNumber}:${selection.startColumn ?? 1}` }) : fileUri })), + ], { forceNewWindow: true, gotoLineMode: true, diffMode: diff, ...remote, reviewSourceTitle }); return true; } async openSourceReferences(resource: URI, position: { readonly lineNumber: number; readonly column: number }): Promise { if (![REVIEW_API_SOURCE_SCHEME, REVIEW_LANGUAGE_SOURCE_SCHEME].includes(resource.scheme)) return false; - const destination = await this.sourceDestination(resource); - await this.host.openWindow([{ workspaceUri: destination.workspaceUri }], { + const { workspaceUri, fileUri, remote, reviewSourceTitle } = await this.sourceDestination(resource); + await this.host.openWindow([{ workspaceUri }], { forceNewWindow: true, - reviewReferencesToShow: { resource: URI.file(destination.filePath), lineNumber: position.lineNumber, column: position.column }, + ...remote, + reviewSourceTitle, + reviewReferencesToShow: { resource: fileUri, lineNumber: position.lineNumber, column: position.column }, }); return true; } - private async sourceDestination(resource: URI): Promise<{ workspaceUri: URI; filePath: string }> { + private async sourceDestination(resource: URI) { const target = sourceLocation(resource); const local = resource.scheme === REVIEW_LANGUAGE_SOURCE_SCHEME; const result = await this.navigatorWorkspace(target.view.reviewId, { @@ -147,12 +166,30 @@ export class ReviewCanvasEditorTabsService extends Disposable implements IReview file: local ? undefined : target.file, empty: new URLSearchParams(resource.query).has("empty") ? "true" : undefined, }); - const filePath = local ? resource.fsPath : result.filePath; - if (!filePath) throw new Error("The navigator did not resolve the source file."); - return { workspaceUri: URI.file(result.workspacePath), filePath }; + const { workspaceUri, remote } = workspace(result); + const fileUri = "remoteAuthority" in result + ? result.emptySide ? await this.emptyFile() : result.fileUri && URI.parse(result.fileUri) + : local ? URI.file(resource.fsPath) : result.filePath && URI.file(result.filePath); + if (!fileUri) throw new Error("The navigator did not resolve the source file."); + const input = this.inputs.get(`api:${target.view.reviewId}`); + const title = input?.target.kind === "api" ? input.target.title : basename(workspaceUri).replace(/\.code-workspace$/, ""); + const side = target.side === "base" ? "base" : target.view.generation && !target.view.commit && !target.view.pins ? "live" : "head"; + return { workspaceUri, fileUri, remote, reviewSourceTitle: this.sourceTitle(target.view.reviewId, side, title, remote) }; + } + + /** A remote review's Source window also keeps its host's alias, for when main does not know it yet. */ + private sourceTitle(reviewId: string, side: "live" | "base" | "head", title: string, remote: { remoteAuthority?: string }) { + const alias = remote.remoteAuthority && this.catalog.reviews.find(review => review.reviewId === reviewId)?.host; + return { side, title, ...(alias && { alias }) }; + } + + private async emptyFile(): Promise { + const resource = joinPath(this.environment.cacheHome, "source-empty", "empty"); + await this.files.writeFile(resource, VSBuffer.fromString("")); + return resource.with({ scheme: Schemas.vscodeUserData }); } - private async navigatorWorkspace(reviewId: string, values: Record): Promise<{ workspacePath: string; filePath?: string }> { + private async navigatorWorkspace(reviewId: string, values: Record): Promise { const { serverUrl, token } = await this.desktopConnection.getConnection(); const query = new URLSearchParams(Object.entries(values).filter(([key, value]) => key !== "reviewId" && value !== undefined).map(([key, value]) => [key, String(value)])); const response = await fetch(`${serverUrl}/reviews-api/${encodeURIComponent(reviewId)}/navigator${query.size ? `?${query}` : ""}`, { diff --git a/apps/review-desktop/code-oss/src/vs/review/services/reviewDesktopConnectionService.test.ts b/apps/review-desktop/code-oss/src/vs/review/services/reviewDesktopConnectionService.test.ts index 53d673f25..dafe96c9f 100644 --- a/apps/review-desktop/code-oss/src/vs/review/services/reviewDesktopConnectionService.test.ts +++ b/apps/review-desktop/code-oss/src/vs/review/services/reviewDesktopConnectionService.test.ts @@ -144,3 +144,19 @@ test("tutorial deletion suppresses auto-prepare across restarts until explicit o assert.match(requests[2] ?? "", /POST .*\/tutorial\/prepare$/); restoredService.dispose(); }); + +test("reads the remote hosts' states from the Desktop server, and refuses a malformed answer", async (t) => { + const service = serviceWith(); + t.after(() => service.dispose()); + const requests: string[] = []; + let answer: unknown = [{ alias: "devbox", serverId: "s1", state: "online" }, { alias: "box2", state: "not-installed", detail: "Install it.", installCommand: "npm install -g @dev.fast/whiteboard@0.1.6", declined: true }]; + mockFetch(t, async (input, init) => { + requests.push(`${String(input)} ${new Headers(init?.headers).get("x-review-token")}`); + return Response.json(answer); + }); + + assert.deepEqual(await service.readRemoteHosts(), answer); + assert.deepEqual(requests, ["http://127.0.0.1:5000/remote-hosts token"]); + for (answer of [[{ alias: "devbox", state: "asleep" }], [{ alias: "devbox", state: "incompatible", installCommand: 7 }], [{ alias: "devbox", state: "not-installed", declined: "yes" }]]) + await assert.rejects(service.readRemoteHosts(), /remote hosts/); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/services/reviewDesktopConnectionService.ts b/apps/review-desktop/code-oss/src/vs/review/services/reviewDesktopConnectionService.ts index fa2914f01..1e4aef9d3 100644 --- a/apps/review-desktop/code-oss/src/vs/review/services/reviewDesktopConnectionService.ts +++ b/apps/review-desktop/code-oss/src/vs/review/services/reviewDesktopConnectionService.ts @@ -26,6 +26,10 @@ parseReviewDesktopVerbFrame, parseReviewTutorialOpenResponse, type ReviewCliInstallApplyResponse, type ReviewCliInstallStatus, +type ReviewGatewayHostState, +type ReviewRemoteAgent, +type ReviewRemoteAgentId, +type ReviewRemoteAgentResult, type ReviewTutorialOpenResponse, type ReviewVerbResponse } from "../common/reviewProtocol.js"; @@ -33,6 +37,7 @@ import { REVIEW_SERVER_STARTUP_TIMEOUT_MS, reconnectUntilAborted, } from "../common/reviewReconnect.js"; +import { parseRemoteLanguageEndpoint } from "./remote/reviewWindowAuthorityResolver.js"; const REVIEW_TUTORIAL_AUTOPREPARE_SUPPRESSED_KEY = "review.tutorial.autoPrepareSuppressed.v1"; @@ -44,6 +49,12 @@ export interface ReviewServerConnection { } +export interface ReviewRemoteLanguageEndpoint { + readonly host: string; + readonly port: number; + readonly connectionToken: string; +} + export const IReviewDesktopConnectionService = createDecorator( "reviewDesktopConnectionService", ); @@ -65,6 +76,14 @@ export interface IReviewDesktopConnectionService { /** The scratchpad preference: a server preference, since the review server reads it. */ readScratchpadEnabled(): Promise; setScratchpadEnabled(enabled: boolean): Promise; + readRemoteHosts(): Promise; + listSshAliases(): Promise; + retryRemoteHost(alias: string): Promise; + getRemoteLanguageEndpoint(serverId: string): Promise; + installRemoteHost(alias: string): Promise; + detectRemoteAgents(alias: string): Promise; + connectRemoteAgents(alias: string, agents: ReviewRemoteAgentId[]): Promise; + uninstallRemoteHost(alias: string): Promise; getTutorialStatus(): Promise<{ version: 1; reviewUuid: string | null }>; prepareTutorial(): Promise; openTutorial(): Promise; @@ -208,6 +227,46 @@ export class ReviewDesktopConnectionService extends Disposable implements IRevie return parseScratchpadPreference(await response.json()); } + async readRemoteHosts(): Promise { + await this.initialize(); + const response = await fetch(`${this.serverUrl}/remote-hosts`, { + headers: this.authHeaders(), + signal: AbortSignal.timeout(30_000), + }); + await this.requireOk(response, "remote hosts"); + return parseRemoteHostStates(await response.json()); + } + + async listSshAliases(): Promise { + const aliases: unknown = await this.mainProcessService.getChannel(REVIEW_DESKTOP_CHANNEL).call("listSshAliases"); + return Array.isArray(aliases) ? aliases.filter((alias): alias is string => typeof alias === "string") : []; + } + + async retryRemoteHost(alias: string): Promise { + await this.mainProcessService.getChannel(REVIEW_DESKTOP_CHANNEL).call("retryRemoteHost", alias); + } + + async getRemoteLanguageEndpoint(serverId: string): Promise { + return parseRemoteLanguageEndpoint(await this.mainProcessService.getChannel(REVIEW_DESKTOP_CHANNEL).call("getRemoteLanguageEndpoint", serverId)); + } + + async installRemoteHost(alias: string): Promise { + await this.mainProcessService.getChannel(REVIEW_DESKTOP_CHANNEL).call("installRemoteHost", alias); + } + + detectRemoteAgents(alias: string): Promise { + return this.mainProcessService.getChannel(REVIEW_DESKTOP_CHANNEL).call("detectRemoteAgents", alias); + } + + connectRemoteAgents(alias: string, agents: ReviewRemoteAgentId[]): Promise { + return this.mainProcessService.getChannel(REVIEW_DESKTOP_CHANNEL).call("connectRemoteAgents", { alias, agents }); + } + + async uninstallRemoteHost(alias: string): Promise { + const problem: unknown = await this.mainProcessService.getChannel(REVIEW_DESKTOP_CHANNEL).call("uninstallRemoteHost", alias); + if (typeof problem === "string") throw new Error(problem); + } + async saveDiffrSummarizer(input: ReviewDiffrSummarizerInput): Promise { await this.initialize(); const response = await fetch(`${this.serverUrl}/diffr-config/summarizer`, { @@ -550,6 +609,20 @@ export async function reviewResponseError(response: Response, fallback: string): return new Error(typeof payload?.error === "string" && payload.error ? payload.error : fallback); } +const REMOTE_HOST_STATES = new Set(["connecting", "online", "offline", "incompatible", "duplicate", "unreachable", "not-installed", "auth-failed", "unsupported", "installing"]); + +function parseRemoteHostStates(value: unknown): ReviewGatewayHostState[] { + const optionalString = (field: unknown) => field === undefined || typeof field === "string"; + if (!Array.isArray(value) || !value.every((host) => + typeof host === "object" && host !== null && + typeof host.alias === "string" && REMOTE_HOST_STATES.has(host.state) && + optionalString(host.serverId) && optionalString(host.detail) && optionalString(host.installCommand) && + (host.declined === undefined || host.declined === true))) { + throw new Error("remote hosts response is malformed."); + } + return value; +} + function parseScratchpadPreference(value: unknown): boolean { if (typeof value !== "object" || value === null || !("enabled" in value) || typeof value.enabled !== "boolean") { throw new Error("scratchpad preference response is malformed."); diff --git a/apps/review-desktop/code-oss/src/vs/review/services/reviewDiffViewService.ts b/apps/review-desktop/code-oss/src/vs/review/services/reviewDiffViewService.ts index 025931f4c..dc7529210 100644 --- a/apps/review-desktop/code-oss/src/vs/review/services/reviewDiffViewService.ts +++ b/apps/review-desktop/code-oss/src/vs/review/services/reviewDiffViewService.ts @@ -330,6 +330,7 @@ class DiffViewController extends Disposable implements ReviewDiffViewHandle { this.overflowWidgetsDomNode, this.diffLayout, this.spec.fileTreeContainer, this.spec.onToggleViewed, this.spec.onToggleSection, this.spec.document, + this.spec.openFile !== false, ), ); this.view = view; diff --git a/apps/review-desktop/code-oss/src/vs/review/services/reviewEditorResolverService.test.ts b/apps/review-desktop/code-oss/src/vs/review/services/reviewEditorResolverService.test.ts index 179afa0e7..eba0c83bb 100644 --- a/apps/review-desktop/code-oss/src/vs/review/services/reviewEditorResolverService.test.ts +++ b/apps/review-desktop/code-oss/src/vs/review/services/reviewEditorResolverService.test.ts @@ -28,6 +28,9 @@ test("source tree, selected code, definitions and diffs hand off before creating { async getConnection() { return { serverUrl: "http://localhost", token: "test" }; } } as never, { async openWindow(openables: IWindowOpenable[], options: IOpenWindowOptions) { windows.push({ openables, options }); } } as never, { warn() {} } as never, + {} as never, + {} as never, + { reviews: [] } as never, ); const resolver = new ReviewEditorResolverService( { get activeGroup() { throw new Error("Review must not create an editor group"); } } as never, @@ -46,7 +49,7 @@ test("source tree, selected code, definitions and diffs hand off before creating { workspaceUri: URI.file("/navigator/base.code-workspace") }, { fileUri: URI.file("/navigator/base/old-name.ts:42:3") }, ]); - assert.deepEqual(windows[1].options, { forceNewWindow: true, gotoLineMode: true, diffMode: false }); + assert.deepEqual(windows[1].options, { forceNewWindow: true, gotoLineMode: true, diffMode: false, reviewSourceTitle: { side: "base", title: "base" } }); assert.equal(requests[1].searchParams.get("version"), "7"); assert.equal(requests[1].searchParams.get("commit"), "selected-commit"); assert.equal(requests[1].searchParams.get("repositoryId"), "other-repository"); diff --git a/apps/review-desktop/code-oss/src/vs/review/services/reviewFilesDiffView.ts b/apps/review-desktop/code-oss/src/vs/review/services/reviewFilesDiffView.ts index 4ab289a1e..5b7678baa 100644 --- a/apps/review-desktop/code-oss/src/vs/review/services/reviewFilesDiffView.ts +++ b/apps/review-desktop/code-oss/src/vs/review/services/reviewFilesDiffView.ts @@ -213,6 +213,7 @@ export class ReviewFilesDiffView extends Disposable { private readonly onToggleViewed: ((path: string, sectionId?: string) => void) | undefined, private readonly onToggleSection: ((id: string) => void) | undefined, private readonly document: ReviewDiffViewSpec["document"], + openFile: boolean, @IInstantiationService private readonly reviewInstantiationService: IInstantiationService, @IEditorService private readonly editorService: IEditorService, @@ -248,8 +249,7 @@ export class ReviewFilesDiffView extends Disposable { // A binary is folded like a hidden file, with its size as the reason. note: entry.file.status === "unchanged" ? "Unchanged" : entry.file.binary ? this.binaryNote(entry.file.path) : this.hiddenFiles.get(entry.file.path), noteTooltip: entry.file.binary ? { label: "No text to show · binary files stay folded" } : undefined, - // A binary has no text to open. - onDidOpen: entry.file.binary ? undefined : () => { + onDidOpen: entry.file.binary || !(document ? document.onDidOpen : openFile) ? undefined : () => { if (document?.onDidOpen) { document.onDidOpen(); return; } const target = this.widget.tryGetCodeEditor(entry.goToFileResource); const change = isDiffEditor(target?.diffEditor) ? target.diffEditor.getDiffComputationResult()?.changes2[0] : undefined; diff --git a/apps/review-desktop/code-oss/src/vs/review/services/reviewLocalLanguageFeatures.test.ts b/apps/review-desktop/code-oss/src/vs/review/services/reviewLocalLanguageFeatures.test.ts index f0886b759..3e2d0a156 100644 --- a/apps/review-desktop/code-oss/src/vs/review/services/reviewLocalLanguageFeatures.test.ts +++ b/apps/review-desktop/code-oss/src/vs/review/services/reviewLocalLanguageFeatures.test.ts @@ -1,8 +1,11 @@ import assert from "node:assert/strict"; -import test from "node:test"; +import { mock, test } from "node:test"; import { createRequire, registerHooks } from "node:module"; +import { CancellationToken } from "../../base/common/cancellation.js"; import { Disposable } from "../../base/common/lifecycle.js"; import { Position } from "../../editor/common/core/position.js"; +import { Range } from "../../editor/common/core/range.js"; +import { LanguageFeaturesService } from "../../editor/common/services/languageFeaturesService.js"; import { URI } from "../../base/common/uri.js"; const { JSDOM } = createRequire(import.meta.url)("jsdom"); @@ -53,6 +56,25 @@ function model(attached = false) { } as any; } +function codeEditors() { + const handlers: ((input: any, source: unknown, sideBySide?: boolean) => Promise)[] = []; + const sourceWindows: string[] = []; + const opened: { resource: string; selection: unknown }[] = []; + return { + sourceWindows, opened, + service: { registerCodeEditorOpenHandler: (handler: any) => { handlers.unshift(handler); return Disposable.None; } }, + editors: { openEditor: async (input: any) => { opened.push({ resource: input.resource.toString(), selection: input.options?.selection }); return { getControl: () => ({ getEditorType: () => "vs.editor.ICodeEditor" }) }; } }, + async open(input: any) { + for (const handler of handlers) { + const editor = await handler(input, null); + if (editor) return editor; + } + sourceWindows.push(input.resource.toString()); + return null; + }, + }; +} + function setup(input: ReturnType | ReturnType[]) { const sourceModels = Array.isArray(input) ? input : [input]; const added = event(); @@ -69,6 +91,7 @@ function setup(input: ReturnType | ReturnType[]) { getEOL: () => "\n", equalsTextBuffer: (other: string) => other === "same pinned source", }; + const opening = codeEditors(); const service = new ReviewLocalLanguageFeatures( { onDidChangeConnection: () => Disposable.None } as any, { createModelReference: async (uri: URI) => ({ object: { textEditorModel: { ...local, uri } }, dispose() { } }) } as any, @@ -79,8 +102,11 @@ function setup(input: ReturnType | ReturnType[]) { { files: { models: [], resolve: async () => undefined } } as any, { onDidFilesChange: () => Disposable.None, exists: async () => true } as any, { debug() { } } as any, + { host: async () => undefined } as any, + opening.service as any, + opening.editors as any, ); - return { service, sourceModel: sourceModels[0], sourceModels, local }; + return { service, sourceModel: sourceModels[0], sourceModels, local, opening }; } function source(local: any) { @@ -189,3 +215,132 @@ test("disposing a review model or the service releases its warm native source", result.service.dispose(); } }); + +const SERVER_ID = "6F23D55B-8446-437e-afd6-ad3a40eecc4c"; +const AUTHORITY = "whiteboard+6f23d55b-8446-437e-afd6-ad3a40eecc4c"; +const ROOT = "/home/dev/repo/.git/dev-fast/reviews/r/head/c"; +const remoteUri = (path: string, authority = AUTHORITY) => URI.from({ scheme: "vscode-remote", authority, path }); + +function remoteSetup(connect?: () => Promise) { + const window = new LanguageFeaturesService(); + const remote = new LanguageFeaturesService(); + const roots: string[] = []; + const asked: string[] = []; + const activated: string[] = []; + const windowActivations: string[] = []; + const host = { + authority: AUTHORITY, + languageFeatures: remote, + addRoot: async (root: URI) => { roots.push(root.toString()); return Disposable.None; }, + activateByEvent: async (event: string) => { activated.push(event); }, + }; + const text = "same pinned source"; + const textModel = (uri: URI) => ({ + uri, isDisposed: () => false, getVersionId: () => 1, getTextBuffer: () => text, getEOL: () => "\n", + equalsTextBuffer: (other: string) => other === text, getLanguageId: () => "typescript", + isTooLargeForSyncing: () => false, onDidChangeContent: () => Disposable.None, + }); + const review = model(true); + const opening = codeEditors(); + const service = new ReviewLocalLanguageFeatures( + { onDidChangeConnection: () => Disposable.None } as any, + { createModelReference: async (uri: URI) => ({ object: { textEditorModel: textModel(uri) }, dispose() { } }) } as any, + { getModels: () => [], onModelAdded: () => Disposable.None } as any, + window, + { activateByEvent: async (event: string) => { windowActivations.push(event); } } as any, + { addFolders: async () => { throw new Error("a remote root never enters the window's workspace"); } } as any, + { files: { models: [], resolve: async () => undefined } } as any, + { onDidFilesChange: () => Disposable.None, exists: async () => true } as any, + { debug() { }, warn() { } } as any, + { host: async (serverId: string) => { asked.push(serverId); return connect ? connect() : host; } } as any, + opening.service as any, + opening.editors as any, + ); + const internal = service as any; + internal.environment = async () => ({ remoteRootPath: ROOT, identity: "hash", serverId: SERVER_ID }); + return { service, internal, review, window, remote, roots, asked, activated, windowActivations, opening }; +} + +test("a remote review is rooted on its host and asks that host's registry, never the window's", async (t) => { + const { service, internal, review, window, remote, roots, asked, activated, windowActivations } = remoteSetup(); + t.after(() => service.dispose()); + let windowAsked = 0; + window.hoverProvider.register({ language: "typescript" }, { provideHover: () => { windowAsked++; return { range: new Range(1, 1, 1, 5), contents: [{ value: "1" }] }; } }); + remote.hoverProvider.register({ language: "typescript" }, { provideHover: () => ({ range: new Range(1, 1, 1, 5), contents: [{ value: "42" }] }) }); + const hover = await internal.hover(review, new Position(1, 2), CancellationToken.None); + assert.deepEqual(hover.contents.map((content: { value: string }) => content.value), ["42"]); + assert.equal(windowAsked, 0); + assert.deepEqual(asked, [SERVER_ID]); + assert.deepEqual(roots, [remoteUri(ROOT).toString()]); + assert.deepEqual(activated, ["onLanguage:typescript", "onReviewWorkspaceLanguage:typescript"]); + assert.deepEqual(windowActivations, []); +}); + +test("a remote definition inside the review's repository maps to the review's file; one outside stays on its host", async (t) => { + const { service, internal, review, remote } = remoteSetup(); + t.after(() => service.dispose()); + const range = new Range(1, 7, 1, 13); + const library = remoteUri("/usr/lib/node_modules/typescript/lib/lib.es5.d.ts"); + remote.definitionProvider.register({ language: "typescript" }, { provideDefinition: () => [ + { uri: remoteUri(`${ROOT}/src/a.ts`), range }, + { uri: library, range }, + { uri: URI.file(`${ROOT}/src/a.ts`), range }, + { uri: remoteUri(`${ROOT}/src/a.ts`, "whiteboard+b563e17e-6f4f-4552-968e-12c38d75a6ab"), range }, + ] }); + const locations = await internal.locations(review, new Position(1, 2), CancellationToken.None, "definition"); + assert.deepEqual(locations.map((location: { uri: URI }) => location.uri.toString()), [ + review.uri.with({ path: "/src/a.ts" }).toString(), + library.toString(), + ]); +}); + +test("a hover for a review whose host is not connected resolves empty within 5 s, and the next hover asks again", async (t) => { + mock.timers.enable({ apis: ["setTimeout"] }); + t.after(() => mock.timers.reset()); + const flush = async () => { for (let i = 0; i < 50; i++) await Promise.resolve(); }; + + const offline = remoteSetup(async () => undefined); + t.after(() => offline.service.dispose()); + assert.equal(await offline.internal.hover(offline.review, new Position(1, 2), CancellationToken.None), undefined); + assert.equal(await offline.internal.hover(offline.review, new Position(1, 2), CancellationToken.None), undefined); + assert.deepEqual(offline.asked, [SERVER_ID, SERVER_ID]); + + const hung = { remote: new LanguageFeaturesService() }; + hung.remote.hoverProvider.register({ language: "typescript" }, { provideHover: () => new Promise(() => { }) }); + const reconnecting = remoteSetup(async () => ({ authority: AUTHORITY, languageFeatures: hung.remote, addRoot: async () => Disposable.None, activateByEvent: () => new Promise(() => { }) })); + const connecting = remoteSetup(() => new Promise(() => { })); + for (const { service, internal, review } of [reconnecting, connecting]) { + t.after(() => service.dispose()); + let settled = false; + const hover = internal.hover(review, new Position(1, 2), CancellationToken.None).finally(() => { settled = true; }); + await flush(); + mock.timers.tick(4_999); + await flush(); + assert.equal(settled, false); + mock.timers.tick(1); + assert.equal(await hover, undefined); + } +}); + +test("a remote review's in-repository definition opens its host's file at the range, never a source window; a laptop review's still does", async (t) => { + const { service, internal, review, remote, opening } = remoteSetup(); + t.after(() => service.dispose()); + remote.definitionProvider.register({ language: "typescript" }, { provideDefinition: () => [{ uri: remoteUri(`${ROOT}/src/a.ts`), range: new Range(1, 7, 1, 13) }] }); + const [location] = await internal.locations(review, new Position(1, 2), CancellationToken.None, "definition"); + assert.equal(location.uri.scheme, "review-api-source", "results still name the review's own file"); + const selection = { startLineNumber: 1, startColumn: 7, endLineNumber: 1, endColumn: 7 }; + assert.ok(await opening.open({ resource: location.uri, options: { selection } })); + assert.deepEqual(opening.opened, [{ resource: remoteUri(`${ROOT}/src/a.ts`).toString(), selection }]); + assert.deepEqual(opening.sourceWindows, []); + + const laptop = setup(model(true)); + t.after(() => laptop.service.dispose()); + const internalLaptop = laptop.service as any; + internalLaptop.environment = async () => ({ rootPath: "/project", identity: "identity" }); + internalLaptop.acquire = async () => source(laptop.local); + await internalLaptop.localSource(laptop.sourceModel); + const target = laptop.sourceModel.uri.with({ path: "/src/a.ts" }); + assert.equal(await laptop.opening.open({ resource: target, options: { selection } }), null); + assert.deepEqual(laptop.opening.sourceWindows, [target.toString()]); + assert.deepEqual(laptop.opening.opened, []); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/services/reviewLocalLanguageFeatures.ts b/apps/review-desktop/code-oss/src/vs/review/services/reviewLocalLanguageFeatures.ts index 1a56e31f5..7b2639ff2 100644 --- a/apps/review-desktop/code-oss/src/vs/review/services/reviewLocalLanguageFeatures.ts +++ b/apps/review-desktop/code-oss/src/vs/review/services/reviewLocalLanguageFeatures.ts @@ -1,6 +1,9 @@ +import { raceTimeout, timeout } from "../../base/common/async.js"; import type { CancellationToken } from "../../base/common/cancellation.js"; import { Disposable, DisposableStore, RefCountedDisposable, toDisposable, type IDisposable, type IReference } from "../../base/common/lifecycle.js"; import { URI } from "../../base/common/uri.js"; +import { getCodeEditor } from "../../editor/browser/editorBrowser.js"; +import { ICodeEditorService } from "../../editor/browser/services/codeEditorService.js"; import { Position } from "../../editor/common/core/position.js"; import type { Hover, LocationLink } from "../../editor/common/languages.js"; import { EndOfLinePreference, type ITextModel } from "../../editor/common/model.js"; @@ -12,23 +15,30 @@ import { getHoversPromise } from "../../editor/contrib/hover/browser/getHover.js import { IFileService } from "../../platform/files/common/files.js"; import { ILogService } from "../../platform/log/common/log.js"; import { registerWorkbenchContribution2, WorkbenchPhase } from "../../workbench/common/contributions.js"; +import { ACTIVE_GROUP, IEditorService, SIDE_GROUP } from "../../workbench/services/editor/common/editorService.js"; import { IExtensionService } from "../../workbench/services/extensions/common/extensions.js"; import { ITextFileService } from "../../workbench/services/textfile/common/textfiles.js"; import { IWorkspaceEditingService } from "../../workbench/services/workspaces/common/workspaceEditing.js"; -import { reviewSourceQuery, type ReviewLanguageEnvironment } from "../common/reviewProtocol.js"; +import { reviewSourceQuery, type ReviewLanguageEnvironment, type ReviewRemoteLanguageEnvironment } from "../common/reviewProtocol.js"; import { sourceLocation } from "../common/reviewSourceView.js"; import { reviewWorkspaceLanguageEvent } from "../common/reviewWorkspaceLanguageActivation.js"; import { REVIEW_LANGUAGE_SOURCE_SCHEME } from "../common/reviewReadonlySource.js"; import { REVIEW_API_SOURCE_SCHEME } from "./reviewApiSourceService.js"; import { IReviewDesktopConnectionService } from "./reviewDesktopConnectionService.js"; import { withCurrentLocalContext } from "./reviewLocalRequest.js"; -import { acquireReviewLanguageRoot } from "./reviewLocalWorkspace.js"; +import { acquireReviewLanguageRoot, reviewLanguageRoot } from "./reviewLocalWorkspace.js"; import { ReviewLanguageEnvironmentRequests } from "./reviewLanguageEnvironmentRequests.js"; import { watchAttachedReviewModels, withRetainedSource } from "./reviewSourceModelLifecycle.js"; +import { ownsRemoteResource } from "./remote/reviewRemoteAuthority.js"; +import type { IReviewRemoteHost } from "./remote/reviewRemoteHost.js"; +import { IReviewRemoteHostsService } from "./remote/reviewRemoteHosts.js"; + +const REMOTE_ANSWER_MS = 5_000; interface LocalSource { identity: string; root: URI; + remote?: IReviewRemoteHost; reference: IReference; retain(): IDisposable | undefined; dispose(): void; @@ -37,7 +47,9 @@ interface LocalSource { /** Review bytes remain pinned; language queries use the resolved project environment. */ export class ReviewLocalLanguageFeatures extends Disposable { static readonly ID = "review.localLanguageFeatures"; - private readonly sources = new Map }>(); + private readonly sources = new Map }>(); + private readonly remoteReviews = new Set(); + private readonly remoteRoots = new Map(); private readonly environments = new ReviewLanguageEnvironmentRequests(); private readonly roots = new Map(); private readonly uncertainRoots = new Set(); @@ -53,13 +65,22 @@ export class ReviewLocalLanguageFeatures extends Disposable { @ITextFileService private readonly textFiles: ITextFileService, @IFileService private readonly files: IFileService, @ILogService private readonly log: ILogService, + @IReviewRemoteHostsService private readonly remoteHosts: IReviewRemoteHostsService, + @ICodeEditorService codeEditors: ICodeEditorService, + @IEditorService private readonly editors: IEditorService, ) { super(); + this._register(codeEditors.registerCodeEditorOpenHandler(async (input, _source, sideBySide) => { + const root = input.resource.scheme === REVIEW_API_SOURCE_SCHEME ? this.remoteRoots.get(input.resource.with({ path: "/" }).toString()) : undefined; + if (!root) return null; + const pane = await this.editors.openEditor({ resource: URI.joinPath(root, input.resource.path), options: input.options }, sideBySide ? SIDE_GROUP : ACTIVE_GROUP); + return getCodeEditor(pane?.getControl()); + })); this._register(connection.onDidChangeConnection(() => { this.environments.invalidate(); this.generation++; for (const entry of this.sources.values()) { - this.uncertainRoots.add(URI.file(entry.rootPath).toString()); + this.uncertainRoots.add(entry.root); void entry.pending.then(source => source?.dispose()); } this.sources.clear(); @@ -83,14 +104,14 @@ export class ReviewLocalLanguageFeatures extends Disposable { this._register(languages.implementationProvider.register(target, { provideImplementation: (model, position, token) => this.locations(model, position, token, "implementation") })); this._register(languages.referenceProvider.register(target, { provideReferences: (model, position, context, token) => this.withSource(model, position, token, async (local, at, pinned, source) => { - const results = await Promise.all(languages.referenceProvider.ordered(local).map(provider => provider.provideReferences(local, at, context, token))); + const results = await Promise.all(this.registry(source).referenceProvider.ordered(local).map(provider => provider.provideReferences(local, at, context, token))); return this.reviewLocations(pinned, results.flatMap(result => result ?? []), token, source); }), })); } } - private async environment(model: ITextModel, validate = false): Promise { + private async environment(model: ITextModel, validate = false): Promise { const { serverUrl, token } = await this.connection.getConnection(); const target = sourceLocation(model.uri); return this.environments.read(JSON.stringify([serverUrl, token]), target.view, target.side, async () => { @@ -118,20 +139,26 @@ export class ReviewLocalLanguageFeatures extends Disposable { const epoch = this.environments.generation; const context = await this.environment(model); if (epoch !== this.environments.generation || this._store.isDisposed || model.isDisposed() || (warming && !model.isAttachedToEditor())) return undefined; + if (context && "remoteRootPath" in context) this.remoteReviews.add(model.uri.authority); + else if (context) this.remoteReviews.delete(model.uri.authority); + const target = reviewLanguageRoot(context); + const side = model.uri.with({ path: "/" }).toString(); + if (target?.serverId !== undefined) this.remoteRoots.set(side, target.root); + else if (context) this.remoteRoots.delete(side); const cached = this.sources.get(model); - if (cached && cached.identity === context?.identity && cached.rootPath === context.rootPath) return cached.pending; + if (cached && cached.identity === context?.identity && cached.root === target?.root.toString()) return cached.pending; if (cached) { - this.uncertainRoots.add(URI.file(cached.rootPath).toString()); + this.uncertainRoots.add(cached.root); void cached.pending.then(source => source?.dispose()); this.sources.delete(model); this.generation++; } - if (!context?.rootPath) return undefined; - const pending = this.acquire(model, { rootPath: context.rootPath, identity: context.identity }).catch(error => { + if (!context || !target) return undefined; + const pending = this.acquire(model, target, context.identity).catch(error => { this.log.debug("[Whiteboard] Language model unavailable", error); return undefined; }); - const entry = { identity: context.identity, rootPath: context.rootPath, pending }; + const entry = { identity: context.identity, root: target.root.toString(), pending }; this.sources.set(model, entry); const result = await pending; if (this.sources.get(model) !== entry || epoch !== this.environments.generation) { result?.dispose(); return undefined; } @@ -139,21 +166,23 @@ export class ReviewLocalLanguageFeatures extends Disposable { return result; } catch (error) { const cached = this.sources.get(model); - if (cached) this.uncertainRoots.add(URI.file(cached.rootPath).toString()); + if (cached) this.uncertainRoots.add(cached.root); this.log.debug("[Whiteboard] Language environment unavailable", error); return undefined; } } - private async acquire(model: ITextModel, context: { rootPath: string; identity: string }): Promise { - const root = URI.file(context.rootPath); + private async acquire(model: ITextModel, { root, serverId }: { root: URI; serverId?: string }, identity: string): Promise { const relative = model.uri.path.slice(1); if (!relative || relative.split(/[\\/]/).some(part => part === "..")) return undefined; + if (serverId !== undefined && model.uri.scheme === REVIEW_LANGUAGE_SOURCE_SCHEME) return undefined; const resource = model.uri.scheme === REVIEW_LANGUAGE_SOURCE_SCHEME ? URI.file(model.uri.path) : URI.joinPath(root, relative); + const host = serverId === undefined ? undefined : await raceTimeout(this.remoteHosts.host(serverId), REMOTE_ANSWER_MS); + if (serverId !== undefined && !host) return undefined; if (!await this.files.exists(resource) || model.isDisposed()) return undefined; const owned = new DisposableStore(); try { - owned.add(await acquireReviewLanguageRoot(this.workspace, root)); + owned.add(host ? await host.addRoot(root) : await acquireReviewLanguageRoot(this.workspace, root)); this.roots.set(root.toString(), (this.roots.get(root.toString()) ?? 0) + 1); owned.add({ dispose: () => { @@ -166,14 +195,16 @@ export class ReviewLocalLanguageFeatures extends Disposable { owned.add(reference.object.textEditorModel.onDidChangeContent(() => this.generation++)); if (model.isDisposed()) { owned.dispose(); return undefined; } const languageId = reference.object.textEditorModel.getLanguageId(); - await this.extensions.activateByEvent(`onLanguage:${languageId}`); + const events = [`onLanguage:${languageId}`, reviewWorkspaceLanguageEvent(languageId)]; + if (host) await raceTimeout((async () => { for (const event of events) await host.activateByEvent(event); })(), REMOTE_ANSWER_MS); // Folder changes reach the extension host before this activation request. - await this.extensions.activateByEvent(reviewWorkspaceLanguageEvent(languageId)); + else for (const event of events) await this.extensions.activateByEvent(event); if (model.isDisposed()) { owned.dispose(); return undefined; } const lifetime = new RefCountedDisposable(owned); const owner = toDisposable(() => lifetime.release()); return { - root, reference, identity: context.identity, + root, reference, identity, + remote: host, retain: () => { if (owned.isDisposed) return undefined; lifetime.acquire(); @@ -185,6 +216,16 @@ export class ReviewLocalLanguageFeatures extends Disposable { } private async withSource(model: ITextModel, position: Position, token: CancellationToken, run: (local: ITextModel, at: Position, review: ITextModel, source: LocalSource | undefined) => Promise): Promise { + const answer = this.query(model, position, token, run); + const limit = timeout(REMOTE_ANSWER_MS); + try { + return await Promise.race([answer, limit.then(() => this.remoteReviews.has(model.uri.authority) ? undefined : answer)]); + } finally { + limit.cancel(); + } + } + + private async query(model: ITextModel, position: Position, token: CancellationToken, run: (local: ITextModel, at: Position, review: ITextModel, source: LocalSource | undefined) => Promise): Promise { if (token.isCancellationRequested || model.isDisposed()) return undefined; if (model.uri.scheme === "file") return withCurrentLocalContext([model], token, () => this.generation, async () => run(model, position, model, undefined)); const epoch = this.environments.generation; @@ -202,7 +243,7 @@ export class ReviewLocalLanguageFeatures extends Disposable { // watcher updates. Until watcher readiness is authoritative, revalidate // clean native buffers before querying; never replace a dirty buffer. const prefix = source.root.path.replace(/\/$/, "") + "/"; - await Promise.all(this.textFiles.files.models.filter(file => !file.isDirty() && file.resource.scheme === "file" && file.resource.path.startsWith(prefix)) + await Promise.all(this.textFiles.files.models.filter(file => !file.isDirty() && file.resource.scheme === source.root.scheme && file.resource.authority === source.root.authority && file.resource.path.startsWith(prefix)) .map(file => this.textFiles.files.resolve(file.resource, { reload: { async: false } }))); } // Resolve current disk contents, preserving any unsaved local editor buffer. @@ -212,11 +253,11 @@ export class ReviewLocalLanguageFeatures extends Disposable { if (!sameSource(model, local)) return undefined; const result = await run(local, position, model, source); const current = await this.environment(model, true).catch(() => undefined); - if (epoch === this.environments.generation && isSameRoot(current?.rootPath, source.root) && current?.identity === source.identity) return result; + if (epoch === this.environments.generation && reviewLanguageRoot(current)?.root.toString() === source.root.toString() && current?.identity === source.identity) return result; // Failed validation must also release the old workspace/watchers. Keeping // them after deletion can leave the language server blind to later edits. const cached = this.sources.get(model); - if (cached?.identity === source.identity && isSameRoot(cached.rootPath, source.root)) { + if (cached?.identity === source.identity && cached.root === source.root.toString()) { this.uncertainRoots.add(source.root.toString()); this.sources.delete(model); void cached.pending.then(value => value?.dispose()); @@ -230,9 +271,13 @@ export class ReviewLocalLanguageFeatures extends Disposable { } } + private registry(source: LocalSource | undefined): ILanguageFeaturesService { + return source?.remote?.languageFeatures ?? this.languages; + } + private hover(model: ITextModel, position: Position, token: CancellationToken): Promise { - return this.withSource(model, position, token, async (local, at) => { - const hovers = await getHoversPromise(this.languages.hoverProvider, local, at, token); + return this.withSource(model, position, token, async (local, at, _review, source) => { + const hovers = await getHoversPromise(this.registry(source).hoverProvider, local, at, token); if (!hovers.length) return undefined; const range = hovers[0].range; if (!range) return undefined; @@ -242,9 +287,10 @@ export class ReviewLocalLanguageFeatures extends Disposable { private async locations(model: ITextModel, position: Position, token: CancellationToken, kind: "definition" | "type" | "implementation"): Promise { return this.withSource(model, position, token, async (local, at, pinned, source) => { - const results = kind === "definition" ? await getDefinitionsAtPosition(this.languages.definitionProvider, local, at, false, token) - : kind === "type" ? await getTypeDefinitionsAtPosition(this.languages.typeDefinitionProvider, local, at, false, token) - : await getImplementationsAtPosition(this.languages.implementationProvider, local, at, false, token); + const registry = this.registry(source); + const results = kind === "definition" ? await getDefinitionsAtPosition(registry.definitionProvider, local, at, false, token) + : kind === "type" ? await getTypeDefinitionsAtPosition(registry.typeDefinitionProvider, local, at, false, token) + : await getImplementationsAtPosition(registry.implementationProvider, local, at, false, token); return this.reviewLocations(pinned, results, token, source); }); } @@ -265,17 +311,22 @@ export class ReviewLocalLanguageFeatures extends Disposable { const mapped = new Map(); await Promise.all([...groups.values()].map(async group => { const target = group[0].uri; - if (target.scheme !== "file") return; + const remote = source.remote; + if (remote ? !ownsRemoteResource(remote.authority, target) : target.scheme !== "file") return; + const inside = (remote || target.authority === source.root.authority) && target.path.startsWith(prefix); + const onHost = () => { for (const location of group) mapped.set(location, location); }; + if (remote && !inside) return onHost(); const owned = new DisposableStore(); try { const local = owned.add(await this.models.createModelReference(target)).object.textEditorModel; await this.textFiles.files.resolve(target, { reload: { async: false } }); let original: ITextModel | undefined; - if (target.authority === source.root.authority && target.path.startsWith(prefix)) { + if (inside) { const candidate = pinned.uri.with({ scheme: REVIEW_API_SOURCE_SCHEME, path: "/" + target.path.slice(prefix.length) }); try { original = owned.add(await this.models.createModelReference(candidate)).object.textEditorModel; } catch { /* Dependencies and generated files may have no review counterpart. */ } } + if (remote && (!original || !sameSource(original, local))) return onHost(); if (!original || !sameSource(original, local)) { const candidate = pinned.uri.with({ scheme: REVIEW_LANGUAGE_SOURCE_SCHEME, path: target.path }); original = owned.add(await this.models.createModelReference(candidate)).object.textEditorModel; @@ -313,8 +364,3 @@ function sameSource(review: ITextModel, local: ITextModel): boolean { if (review.getEOL() === local.getEOL()) return review.equalsTextBuffer(local.getTextBuffer()); return review.getLineCount() === local.getLineCount() && review.getValue(EndOfLinePreference.LF) === local.getValue(EndOfLinePreference.LF); } - -/** Compare as URIs: Windows fsPath lowercases the drive letter the server reported. */ -function isSameRoot(rootPath: string | null | undefined, root: URI): boolean { - return typeof rootPath === "string" && URI.file(rootPath).toString() === root.toString(); -} diff --git a/apps/review-desktop/code-oss/src/vs/review/services/reviewLocalWorkspace.test.ts b/apps/review-desktop/code-oss/src/vs/review/services/reviewLocalWorkspace.test.ts index 32cde1203..431b05f82 100644 --- a/apps/review-desktop/code-oss/src/vs/review/services/reviewLocalWorkspace.test.ts +++ b/apps/review-desktop/code-oss/src/vs/review/services/reviewLocalWorkspace.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { URI } from '../../base/common/uri.js'; -import { acquireReviewLanguageRoot } from './reviewLocalWorkspace.js'; +import { acquireReviewLanguageRoot, reviewLanguageRoot } from './reviewLocalWorkspace.js'; function workspace() { const folders: string[] = []; @@ -60,3 +60,12 @@ test('a shared checkout stays until its last owner releases it', async () => { await settled(target); assert.ok(!target.folders.includes(a.toString())); }); + +const SERVER_ID = '6F23D55B-8446-437e-afd6-ad3a40eecc4c'; + +test('no root while a checkout is prepared, or for a server id that is not an id', () => { + assert.equal(reviewLanguageRoot({ remoteRootPath: null, identity: 'hash', serverId: SERVER_ID }), undefined); + assert.equal(reviewLanguageRoot({ remoteRootPath: '/repo', identity: 'hash', serverId: '../x' }), undefined); + assert.equal(reviewLanguageRoot({ rootPath: null, identity: 'id' }), undefined); + assert.equal(reviewLanguageRoot(undefined), undefined); +}); diff --git a/apps/review-desktop/code-oss/src/vs/review/services/reviewLocalWorkspace.ts b/apps/review-desktop/code-oss/src/vs/review/services/reviewLocalWorkspace.ts index 01f150ed5..35aaca7d5 100644 --- a/apps/review-desktop/code-oss/src/vs/review/services/reviewLocalWorkspace.ts +++ b/apps/review-desktop/code-oss/src/vs/review/services/reviewLocalWorkspace.ts @@ -1,7 +1,10 @@ import { Queue } from "../../base/common/async.js"; import type { IDisposable } from "../../base/common/lifecycle.js"; +import { Schemas } from "../../base/common/network.js"; import { URI } from "../../base/common/uri.js"; import type { IWorkspaceEditingService } from "../../workbench/services/workspaces/common/workspaceEditing.js"; +import type { ReviewLanguageEnvironment, ReviewRemoteLanguageEnvironment } from "../common/reviewProtocol.js"; +import { reviewRemoteAuthority } from "./remote/reviewRemoteAuthority.js"; type ReviewLanguageRoots = Pick; @@ -42,3 +45,11 @@ export async function acquireReviewLanguageRoot(workspace: ReviewLanguageRoots, } }; } + +export function reviewLanguageRoot(context: ReviewLanguageEnvironment | ReviewRemoteLanguageEnvironment | undefined): { root: URI; serverId?: string } | undefined { + if (!context) return undefined; + if (!("remoteRootPath" in context)) return context.rootPath ? { root: URI.file(context.rootPath) } : undefined; + const authority = reviewRemoteAuthority(context.serverId); + if (!context.remoteRootPath || !authority) return undefined; + return { root: URI.from({ scheme: Schemas.vscodeRemote, authority, path: context.remoteRootPath }), serverId: context.serverId }; +} diff --git a/apps/review-desktop/code-oss/src/vs/server/node/extensionHostConnection.ts b/apps/review-desktop/code-oss/src/vs/server/node/extensionHostConnection.ts new file mode 100644 index 000000000..c54caa639 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/server/node/extensionHostConnection.ts @@ -0,0 +1,373 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as cp from 'child_process'; +import * as net from 'net'; +import { VSBuffer } from '../../base/common/buffer.js'; +import { Emitter, Event } from '../../base/common/event.js'; +import { Disposable, DisposableStore, toDisposable } from '../../base/common/lifecycle.js'; +import { FileAccess } from '../../base/common/network.js'; +import { delimiter, join } from '../../base/common/path.js'; +import { IProcessEnvironment, isWindows } from '../../base/common/platform.js'; +import { removeDangerousEnvVariables } from '../../base/common/processes.js'; +import { createRandomIPCHandle, NodeSocket, WebSocketNodeSocket } from '../../base/parts/ipc/node/ipc.net.js'; +import { IConfigurationService } from '../../platform/configuration/common/configuration.js'; +import { ILogService } from '../../platform/log/common/log.js'; +import { IRemoteExtensionHostStartParams } from '../../platform/remote/common/remoteAgentConnection.js'; +import { getResolvedShellEnv } from '../../platform/shell/node/shellEnv.js'; +import { IExtensionHostStatusService } from './extensionHostStatusService.js'; +import { getNLSConfiguration } from './remoteLanguagePacks.js'; +import { IServerEnvironmentService } from './serverEnvironmentService.js'; +import { IPCExtHostConnection, SocketExtHostConnection, writeExtHostConnection } from '../../workbench/services/extensions/common/extensionHostEnv.js'; +import { IExtHostReadyMessage, IExtHostReduceGraceTimeMessage, IExtHostSocketMessage } from '../../workbench/services/extensions/common/extensionHostProtocol.js'; + +export async function buildUserEnvironment(startParamsEnv: { [key: string]: string | null } = {}, withUserShellEnvironment: boolean, language: string, environmentService: IServerEnvironmentService, logService: ILogService, configurationService: IConfigurationService): Promise { + const nlsConfig = await getNLSConfiguration(language, environmentService.userDataPath); + + let userShellEnv: typeof process.env = {}; + if (withUserShellEnvironment) { + try { + userShellEnv = await getResolvedShellEnv(configurationService, logService, environmentService.args, process.env); + } catch (error) { + logService.error('ExtensionHostConnection#buildUserEnvironment resolving shell environment failed', error); + } + } + + const processEnv = process.env; + + const env: IProcessEnvironment = { + ...processEnv, + ...userShellEnv, + ...{ + VSCODE_ESM_ENTRYPOINT: 'vs/workbench/api/node/extensionHostProcess', + VSCODE_HANDLES_UNCAUGHT_ERRORS: 'true', + VSCODE_NLS_CONFIG: JSON.stringify(nlsConfig) + }, + ...startParamsEnv + }; + + const binFolder = environmentService.isBuilt ? join(environmentService.appRoot, 'bin') : join(environmentService.appRoot, 'resources', 'server', 'bin-dev'); + const remoteCliBinFolder = join(binFolder, 'remote-cli'); // contains the `code` command that can talk to the remote server + + let PATH = readCaseInsensitive(env, 'PATH'); + if (PATH) { + PATH = remoteCliBinFolder + delimiter + PATH; + } else { + PATH = remoteCliBinFolder; + } + setCaseInsensitive(env, 'PATH', PATH); + + if (!environmentService.args['without-browser-env-var']) { + env.BROWSER = join(binFolder, 'helpers', isWindows ? 'browser.cmd' : 'browser.sh'); // a command that opens a browser on the local machine + } + + env.VSCODE_RECONNECTION_GRACE_TIME = String(environmentService.reconnectionGraceTime); + logService.trace(`[reconnection-grace-time] Setting VSCODE_RECONNECTION_GRACE_TIME env var for extension host: ${environmentService.reconnectionGraceTime}ms (${Math.floor(environmentService.reconnectionGraceTime / 1000)}s)`); + + removeNulls(env); + return env; +} + +class ConnectionData { + constructor( + public readonly socket: NodeSocket | WebSocketNodeSocket, + public readonly initialDataChunk: VSBuffer + ) { } + + public socketDrain(): Promise { + return this.socket.drain(); + } + + public toIExtHostSocketMessage(): IExtHostSocketMessage { + + let skipWebSocketFrames: boolean; + let permessageDeflate: boolean; + let inflateBytes: VSBuffer; + + if (this.socket instanceof NodeSocket) { + skipWebSocketFrames = true; + permessageDeflate = false; + inflateBytes = VSBuffer.alloc(0); + } else { + skipWebSocketFrames = false; + permessageDeflate = this.socket.permessageDeflate; + inflateBytes = this.socket.recordedInflateBytes; + this.socket.setRecordInflateBytes(false); + } + + return { + type: 'VSCODE_EXTHOST_IPC_SOCKET', + initialDataChunk: (this.initialDataChunk.buffer).toString('base64'), + skipWebSocketFrames: skipWebSocketFrames, + permessageDeflate: permessageDeflate, + inflateBytes: (inflateBytes.buffer).toString('base64'), + }; + } +} + +export class ExtensionHostConnection extends Disposable { + + private _onClose = this._register(new Emitter()); + readonly onClose: Event = this._onClose.event; + + private readonly _canSendSocket: boolean; + private _disposed: boolean; + private _remoteAddress: string; + private _extensionHostProcess: cp.ChildProcess | null; + private _connectionData: ConnectionData | null; + + constructor( + private readonly _reconnectionToken: string, + remoteAddress: string, + socket: NodeSocket | WebSocketNodeSocket, + initialDataChunk: VSBuffer, + @IServerEnvironmentService private readonly _environmentService: IServerEnvironmentService, + @ILogService private readonly _logService: ILogService, + @IExtensionHostStatusService private readonly _extensionHostStatusService: IExtensionHostStatusService, + @IConfigurationService private readonly _configurationService: IConfigurationService + ) { + super(); + this._canSendSocket = (!isWindows || !this._environmentService.args['socket-path']); + this._disposed = false; + this._remoteAddress = remoteAddress; + this._extensionHostProcess = null; + this._connectionData = new ConnectionData(socket, initialDataChunk); + if (!this._canSendSocket && socket instanceof WebSocketNodeSocket) { + socket.setRecordInflateBytes(false); + } + + this._log(`New connection established.`); + } + + override dispose(): void { + this._cleanResources(); + super.dispose(); + } + + private get _logPrefix(): string { + return `[${this._remoteAddress}][${this._reconnectionToken.substr(0, 8)}][ExtensionHostConnection] `; + } + + private _log(_str: string): void { + this._logService.info(`${this._logPrefix}${_str}`); + } + + private _logError(_str: string): void { + this._logService.error(`${this._logPrefix}${_str}`); + } + + private async _pipeSockets(extHostSocket: net.Socket, connectionData: ConnectionData): Promise { + + const disposables = new DisposableStore(); + disposables.add(connectionData.socket); + disposables.add(toDisposable(() => { + if (!extHostSocket.destroyed && !extHostSocket.writableEnded) { + extHostSocket.end(); + } + })); + + const stopAndCleanup = () => { + disposables.dispose(); + }; + + disposables.add(connectionData.socket.onEnd(stopAndCleanup)); + disposables.add(connectionData.socket.onClose(stopAndCleanup)); + + disposables.add(Event.fromNodeEventEmitter(extHostSocket, 'end')(stopAndCleanup)); + disposables.add(Event.fromNodeEventEmitter(extHostSocket, 'close')(stopAndCleanup)); + disposables.add(Event.fromNodeEventEmitter(extHostSocket, 'error')(stopAndCleanup)); + + disposables.add(connectionData.socket.onData((e) => extHostSocket.write(e.buffer))); + disposables.add(Event.fromNodeEventEmitter(extHostSocket, 'data')((e) => { + connectionData.socket.write(VSBuffer.wrap(e)); + })); + + if (connectionData.initialDataChunk.byteLength > 0) { + extHostSocket.write(connectionData.initialDataChunk.buffer); + } + } + + private async _sendSocketToExtensionHost(extensionHostProcess: cp.ChildProcess, connectionData: ConnectionData): Promise { + // Make sure all outstanding writes have been drained before sending the socket + await connectionData.socketDrain(); + const msg = connectionData.toIExtHostSocketMessage(); + let socket: net.Socket; + if (connectionData.socket instanceof NodeSocket) { + socket = connectionData.socket.socket; + } else { + socket = connectionData.socket.socket.socket; + } + extensionHostProcess.send(msg, socket); + } + + public shortenReconnectionGraceTimeIfNecessary(): void { + if (!this._extensionHostProcess) { + return; + } + const msg: IExtHostReduceGraceTimeMessage = { + type: 'VSCODE_EXTHOST_IPC_REDUCE_GRACE_TIME' + }; + this._extensionHostProcess.send(msg); + } + + public acceptReconnection(remoteAddress: string, _socket: NodeSocket | WebSocketNodeSocket, initialDataChunk: VSBuffer): void { + this._remoteAddress = remoteAddress; + this._log(`The client has reconnected.`); + if (!this._canSendSocket && _socket instanceof WebSocketNodeSocket) { + _socket.setRecordInflateBytes(false); + } + const connectionData = new ConnectionData(_socket, initialDataChunk); + + if (!this._extensionHostProcess) { + // The extension host didn't even start up yet + this._connectionData = connectionData; + return; + } + + this._sendSocketToExtensionHost(this._extensionHostProcess, connectionData); + } + + private _cleanResources(): void { + if (this._disposed) { + // already called + return; + } + this._disposed = true; + if (this._connectionData) { + this._connectionData.socket.end(); + this._connectionData = null; + } + if (this._extensionHostProcess) { + this._extensionHostProcess.kill(); + this._extensionHostProcess = null; + } + this._onClose.fire(undefined); + } + + public async start(startParams: IRemoteExtensionHostStartParams): Promise { + try { + let execArgv: string[] = process.execArgv ? process.execArgv.filter(a => !/^--inspect(-brk)?=/.test(a)) : []; + // eslint-disable-next-line local/code-no-any-casts, @typescript-eslint/no-explicit-any + if (startParams.port && !(process).pkg) { + execArgv = [ + `--inspect${startParams.break ? '-brk' : ''}=${startParams.port}`, + '--experimental-network-inspection' + ]; + } + + this._log(`Starting extension host process...`); + + const env = await buildUserEnvironment(startParams.env, true, startParams.language, this._environmentService, this._logService, this._configurationService); + removeDangerousEnvVariables(env); + + let extHostNamedPipeServer: net.Server | null; + + if (this._canSendSocket) { + writeExtHostConnection(new SocketExtHostConnection(), env); + extHostNamedPipeServer = null; + } else { + const { namedPipeServer, pipeName } = await this._listenOnPipe(); + writeExtHostConnection(new IPCExtHostConnection(pipeName), env); + extHostNamedPipeServer = namedPipeServer; + } + + const opts = { + env, + execArgv, + silent: true + }; + + // Refs https://github.com/microsoft/vscode/issues/189805 + opts.execArgv.unshift('--dns-result-order=ipv4first'); + + // Run Extension Host as fork of current process + const args = ['--type=extensionHost', `--transformURIs`]; + const useHostProxy = this._environmentService.args['use-host-proxy']; + args.push(`--useHostProxy=${useHostProxy ? 'true' : 'false'}`); + if (this._configurationService.getValue('extensions.supportNodeGlobalNavigator')) { + args.push('--supportGlobalNavigator'); + } + this._extensionHostProcess = cp.fork(FileAccess.asFileUri('bootstrap-fork').fsPath, args, opts); + const pid = this._extensionHostProcess.pid; + this._log(`<${pid}> Launched Extension Host Process.`); + + // Catch all output coming from the extension host process + this._extensionHostProcess.stdout!.setEncoding('utf8'); + this._extensionHostProcess.stderr!.setEncoding('utf8'); + const onStdout = Event.fromNodeEventEmitter(this._extensionHostProcess.stdout!, 'data'); + const onStderr = Event.fromNodeEventEmitter(this._extensionHostProcess.stderr!, 'data'); + this._register(onStdout((e) => this._log(`<${pid}> ${e}`))); + this._register(onStderr((e) => this._log(`<${pid}> ${e}`))); + + // Lifecycle + this._extensionHostProcess.on('error', (err) => { + this._logError(`<${pid}> Extension Host Process had an error`); + this._logService.error(err); + this._cleanResources(); + }); + + this._extensionHostProcess.on('exit', (code: number, signal: string) => { + this._extensionHostStatusService.setExitInfo(this._reconnectionToken, { code, signal }); + this._log(`<${pid}> Extension Host Process exited with code: ${code}, signal: ${signal}.`); + this._cleanResources(); + }); + + if (extHostNamedPipeServer) { + extHostNamedPipeServer.on('connection', (socket) => { + extHostNamedPipeServer.close(); + this._pipeSockets(socket, this._connectionData!); + }); + } else { + const messageListener = (msg: IExtHostReadyMessage) => { + if (msg.type === 'VSCODE_EXTHOST_IPC_READY') { + this._extensionHostProcess!.removeListener('message', messageListener); + this._sendSocketToExtensionHost(this._extensionHostProcess!, this._connectionData!); + this._connectionData = null; + } + }; + this._extensionHostProcess.on('message', messageListener); + } + + } catch (error) { + this._logError(`Failed to start extension host process`); + this._logService.error(error); + this._cleanResources(); + } + } + + private _listenOnPipe(): Promise<{ pipeName: string; namedPipeServer: net.Server }> { + return new Promise<{ pipeName: string; namedPipeServer: net.Server }>((resolve, reject) => { + const pipeName = createRandomIPCHandle(); + + const namedPipeServer = net.createServer(); + namedPipeServer.on('error', reject); + namedPipeServer.listen(pipeName, () => { + namedPipeServer?.removeListener('error', reject); + resolve({ pipeName, namedPipeServer }); + }); + }); + } +} + +function readCaseInsensitive(env: { [key: string]: string | undefined }, key: string): string | undefined { + const pathKeys = Object.keys(env).filter(k => k.toLowerCase() === key.toLowerCase()); + const pathKey = pathKeys.length > 0 ? pathKeys[0] : key; + return env[pathKey]; +} + +function setCaseInsensitive(env: { [key: string]: unknown }, key: string, value: string): void { + const pathKeys = Object.keys(env).filter(k => k.toLowerCase() === key.toLowerCase()); + const pathKey = pathKeys.length > 0 ? pathKeys[0] : key; + env[pathKey] = value; +} + +function removeNulls(env: { [key: string]: unknown | null }): void { + // Don't delete while iterating the object itself + for (const key of Object.keys(env)) { + if (env[key] === null) { + delete env[key]; + } + } +} diff --git a/apps/review-desktop/code-oss/src/vs/server/node/extensionHostStatusService.ts b/apps/review-desktop/code-oss/src/vs/server/node/extensionHostStatusService.ts new file mode 100644 index 000000000..79426739b --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/server/node/extensionHostStatusService.ts @@ -0,0 +1,30 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { createDecorator } from '../../platform/instantiation/common/instantiation.js'; +import { IExtensionHostExitInfo } from '../../workbench/services/remote/common/remoteAgentService.js'; + +export const IExtensionHostStatusService = createDecorator('extensionHostStatusService'); + +export interface IExtensionHostStatusService { + readonly _serviceBrand: undefined; + + setExitInfo(reconnectionToken: string, info: IExtensionHostExitInfo): void; + getExitInfo(reconnectionToken: string): IExtensionHostExitInfo | null; +} + +export class ExtensionHostStatusService implements IExtensionHostStatusService { + _serviceBrand: undefined; + + private readonly _exitInfo = new Map(); + + setExitInfo(reconnectionToken: string, info: IExtensionHostExitInfo): void { + this._exitInfo.set(reconnectionToken, info); + } + + getExitInfo(reconnectionToken: string): IExtensionHostExitInfo | null { + return this._exitInfo.get(reconnectionToken) || null; + } +} diff --git a/apps/review-desktop/code-oss/src/vs/server/node/extensionsScannerService.ts b/apps/review-desktop/code-oss/src/vs/server/node/extensionsScannerService.ts new file mode 100644 index 000000000..5a191557c --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/server/node/extensionsScannerService.ts @@ -0,0 +1,50 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { joinPath } from '../../base/common/resources.js'; +import { URI } from '../../base/common/uri.js'; +import { INativeEnvironmentService } from '../../platform/environment/common/environment.js'; +import { IExtensionsProfileScannerService } from '../../platform/extensionManagement/common/extensionsProfileScannerService.js'; +import { AbstractExtensionsScannerService, IExtensionsScannerService, Translations } from '../../platform/extensionManagement/common/extensionsScannerService.js'; +import { IFileService } from '../../platform/files/common/files.js'; +import { IInstantiationService } from '../../platform/instantiation/common/instantiation.js'; +import { ILogService } from '../../platform/log/common/log.js'; +import { IProductService } from '../../platform/product/common/productService.js'; +import { IUriIdentityService } from '../../platform/uriIdentity/common/uriIdentity.js'; +import { IUserDataProfilesService } from '../../platform/userDataProfile/common/userDataProfile.js'; +import { getNLSConfiguration } from './remoteLanguagePacks.js'; + +export class ExtensionsScannerService extends AbstractExtensionsScannerService implements IExtensionsScannerService { + + constructor( + @IUserDataProfilesService userDataProfilesService: IUserDataProfilesService, + @IExtensionsProfileScannerService extensionsProfileScannerService: IExtensionsProfileScannerService, + @IFileService fileService: IFileService, + @ILogService logService: ILogService, + @INativeEnvironmentService private readonly nativeEnvironmentService: INativeEnvironmentService, + @IProductService productService: IProductService, + @IUriIdentityService uriIdentityService: IUriIdentityService, + @IInstantiationService instantiationService: IInstantiationService, + ) { + super( + URI.file(nativeEnvironmentService.builtinExtensionsPath), + URI.file(nativeEnvironmentService.extensionsPath), + joinPath(nativeEnvironmentService.userHome, '.vscode-oss-dev', 'extensions', 'control.json'), + userDataProfilesService.defaultProfile, + userDataProfilesService, extensionsProfileScannerService, fileService, logService, nativeEnvironmentService, productService, uriIdentityService, instantiationService); + } + + protected async getTranslations(language: string): Promise { + const config = await getNLSConfiguration(language, this.nativeEnvironmentService.userDataPath); + if (config.languagePack) { + try { + const content = await this.fileService.readFile(URI.file(config.languagePack.translationsConfigFile)); + return JSON.parse(content.value.toString()); + } catch (err) { /* Ignore error */ } + } + return Object.create(null); + } + +} diff --git a/apps/review-desktop/code-oss/src/vs/server/node/remoteAgentEnvironmentImpl.ts b/apps/review-desktop/code-oss/src/vs/server/node/remoteAgentEnvironmentImpl.ts new file mode 100644 index 000000000..640c74695 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/server/node/remoteAgentEnvironmentImpl.ts @@ -0,0 +1,138 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Event } from '../../base/common/event.js'; +import * as platform from '../../base/common/platform.js'; +import * as performance from '../../base/common/performance.js'; +import { URI } from '../../base/common/uri.js'; +import { createURITransformer } from '../../base/common/uriTransformer.js'; +import { IRemoteAgentEnvironmentDTO, IGetEnvironmentDataArguments, IGetExtensionHostExitInfoArguments } from '../../workbench/services/remote/common/remoteAgentEnvironmentChannel.js'; +import { IServerEnvironmentService } from './serverEnvironmentService.js'; +import { IServerChannel } from '../../base/parts/ipc/common/ipc.js'; +import { transformOutgoingURIs } from '../../base/common/uriIpc.js'; +import { listProcesses } from '../../base/node/ps.js'; +import { getMachineInfo, collectWorkspaceStats } from '../../platform/diagnostics/node/diagnosticsService.js'; +import { IDiagnosticInfoOptions, IDiagnosticInfo } from '../../platform/diagnostics/common/diagnostics.js'; +import { basename } from '../../base/common/path.js'; +import { ProcessItem } from '../../base/common/processes.js'; +import { ServerConnectionToken, ServerConnectionTokenType } from './serverConnectionToken.js'; +import { IExtensionHostStatusService } from './extensionHostStatusService.js'; +import { IUserDataProfilesService } from '../../platform/userDataProfile/common/userDataProfile.js'; +import { joinPath } from '../../base/common/resources.js'; +import { ILogService } from '../../platform/log/common/log.js'; + +export class RemoteAgentEnvironmentChannel implements IServerChannel { + + private static _namePool = 1; + + constructor( + private readonly _connectionToken: ServerConnectionToken, + private readonly _environmentService: IServerEnvironmentService, + private readonly _userDataProfilesService: IUserDataProfilesService, + private readonly _extensionHostStatusService: IExtensionHostStatusService, + private readonly _logService: ILogService, + ) { + } + + async call(_: any, command: string, arg?: any): Promise { + switch (command) { + + case 'getEnvironmentData': { + const args = arg; + const uriTransformer = createURITransformer(args.remoteAuthority); + + let environmentData = await this._getEnvironmentData(args.profile); + environmentData = transformOutgoingURIs(environmentData, uriTransformer); + + return environmentData; + } + + case 'getExtensionHostExitInfo': { + const args = arg; + return this._extensionHostStatusService.getExitInfo(args.reconnectionToken); + } + + case 'getDiagnosticInfo': { + const options = arg; + const diagnosticInfo: IDiagnosticInfo = { + machineInfo: getMachineInfo() + }; + + const processesPromise: Promise = options.includeProcesses ? listProcesses(process.pid) : Promise.resolve(); + + let workspaceMetadataPromises: Promise[] = []; + const workspaceMetadata: { [key: string]: any } = {}; + if (options.folders) { + // only incoming paths are transformed, so remote authority is unneeded. + const uriTransformer = createURITransformer(''); + const folderPaths = options.folders + .map(folder => URI.revive(uriTransformer.transformIncoming(folder))) + .filter(uri => uri.scheme === 'file'); + + workspaceMetadataPromises = folderPaths.map(folder => { + return collectWorkspaceStats(folder.fsPath, ['node_modules', '.git']) + .then(stats => { + workspaceMetadata[basename(folder.fsPath)] = stats; + }); + }); + } + + return Promise.all([processesPromise, ...workspaceMetadataPromises]).then(([processes, _]) => { + diagnosticInfo.processes = processes || undefined; + diagnosticInfo.workspaceMetadata = options.folders ? workspaceMetadata : undefined; + return diagnosticInfo; + }); + } + } + + throw new Error(`IPC Command ${command} not found`); + } + + listen(_: any, event: string, arg: any): Event { + throw new Error('Not supported'); + } + + private async _getEnvironmentData(profile?: string): Promise { + if (profile && !this._userDataProfilesService.profiles.some(p => p.id === profile)) { + await this._userDataProfilesService.createProfile(profile, profile); + } + type ProcessWithGlibc = NodeJS.Process & { + glibcVersion?: string; + }; + let isUnsupportedGlibc = false; + if (process.platform === 'linux') { + const glibcVersion = (process as ProcessWithGlibc).glibcVersion; + const minorVersion = glibcVersion ? parseInt(glibcVersion.split('.')[1]) : 28; + isUnsupportedGlibc = (minorVersion <= 27) || !!process.env['VSCODE_SERVER_CUSTOM_GLIBC_LINKER']; + } + this._logService.trace(`[reconnection-grace-time] Server sending grace time to client: ${this._environmentService.reconnectionGraceTime}ms (${Math.floor(this._environmentService.reconnectionGraceTime / 1000)}s)`); + return { + pid: process.pid, + connectionToken: (this._connectionToken.type !== ServerConnectionTokenType.None ? this._connectionToken.value : ''), + appRoot: URI.file(this._environmentService.appRoot), + execPath: process.execPath, + tmpDir: this._environmentService.tmpDir, + settingsPath: this._environmentService.machineSettingsResource, + mcpResource: this._environmentService.mcpResource, + logsPath: this._environmentService.logsHome, + extensionHostLogsPath: joinPath(this._environmentService.logsHome, `exthost${RemoteAgentEnvironmentChannel._namePool++}`), + globalStorageHome: this._userDataProfilesService.defaultProfile.globalStorageHome, + workspaceStorageHome: this._environmentService.workspaceStorageHome, + localHistoryHome: this._environmentService.localHistoryHome, + userHome: this._environmentService.userHome, + os: platform.OS, + arch: process.arch, + marks: performance.getMarks(), + useHostProxy: !!this._environmentService.args['use-host-proxy'], + profiles: { + home: this._userDataProfilesService.profilesHome, + all: [...this._userDataProfilesService.profiles].map(profile => ({ ...profile })) + }, + isUnsupportedGlibc, + reconnectionGraceTime: this._environmentService.reconnectionGraceTime + }; + } + +} diff --git a/apps/review-desktop/code-oss/src/vs/server/node/remoteExtensionHostAgentServer.ts b/apps/review-desktop/code-oss/src/vs/server/node/remoteExtensionHostAgentServer.ts new file mode 100644 index 000000000..284094ee4 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/server/node/remoteExtensionHostAgentServer.ts @@ -0,0 +1,736 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as fs from 'fs'; +import type * as http from 'http'; +import * as net from 'net'; +import { createRequire } from 'node:module'; +import { performance } from 'perf_hooks'; +import * as url from 'url'; +import { VSBuffer } from '../../base/common/buffer.js'; +import { CharCode } from '../../base/common/charCode.js'; +import { isSigPipeError, onUnexpectedError, setUnexpectedErrorHandler } from '../../base/common/errors.js'; +import { Disposable, DisposableMap, DisposableStore } from '../../base/common/lifecycle.js'; +import { FileAccess, getServerProductSegment } from '../../base/common/network.js'; +import { dirname, join } from '../../base/common/path.js'; +import * as perf from '../../base/common/performance.js'; +import * as platform from '../../base/common/platform.js'; +import { generateUuid } from '../../base/common/uuid.js'; +import { getOSReleaseInfo } from '../../base/node/osReleaseInfo.js'; +import { findFreePort } from '../../base/node/ports.js'; +import { addUNCHostToAllowlist, disableUNCAccessRestrictions } from '../../base/node/unc.js'; +import { PersistentProtocol } from '../../base/parts/ipc/common/ipc.net.js'; +import { NodeSocket, upgradeToISocket, WebSocketNodeSocket } from '../../base/parts/ipc/node/ipc.net.js'; +import { IConfigurationService } from '../../platform/configuration/common/configuration.js'; +import { IInstantiationService } from '../../platform/instantiation/common/instantiation.js'; +import { ILogService } from '../../platform/log/common/log.js'; +import { IProductService } from '../../platform/product/common/productService.js'; +import { ConnectionType, ConnectionTypeRequest, ErrorMessage, HandshakeMessage, IRemoteExtensionHostStartParams, ITunnelConnectionStartParams, OKMessage, SignRequest } from '../../platform/remote/common/remoteAgentConnection.js'; +import { RemoteAgentConnectionContext } from '../../platform/remote/common/remoteAgentEnvironment.js'; +import { ITelemetryService } from '../../platform/telemetry/common/telemetry.js'; +import { ExtensionHostConnection } from './extensionHostConnection.js'; +import { ManagementConnection } from './remoteExtensionManagement.js'; +import { determineServerConnectionToken, requestHasValidConnectionToken as httpRequestHasValidConnectionToken, ServerConnectionToken, ServerConnectionTokenParseError, ServerConnectionTokenType } from './serverConnectionToken.js'; +import { IServerEnvironmentService, ServerParsedArgs } from './serverEnvironmentService.js'; +import { IServerLifetimeService } from './serverLifetimeService.js'; +import { setupServerServices, SocketServer } from './serverServices.js'; +const require = createRequire(import.meta.url); + +declare namespace vsda { + // the signer is a native module that for historical reasons uses a lower case class name + // eslint-disable-next-line @typescript-eslint/naming-convention + export class signer { + sign(arg: string): string; + } + + // eslint-disable-next-line @typescript-eslint/naming-convention + export class validator { + createNewMessage(arg: string): string; + validate(arg: string): 'ok' | 'error'; + } +} + +class RemoteExtensionHostAgentServer extends Disposable implements IServerAPI { + + private readonly _extHostConnections: { [reconnectionToken: string]: ExtensionHostConnection }; + private readonly _managementConnections: { [reconnectionToken: string]: ManagementConnection }; + private readonly _allReconnectionTokens: Set; + private readonly _extHostLifetimeTokens = this._register(new DisposableMap()); + private readonly _reconnectionGraceTime: number; + + private readonly _serverBasePath: string | undefined; + private readonly _serverProductPath: string; + + constructor( + private readonly _socketServer: SocketServer, + private readonly _connectionToken: ServerConnectionToken, + private readonly _vsdaMod: typeof vsda | null, + serverBasePath: string | undefined, + @IServerEnvironmentService private readonly _environmentService: IServerEnvironmentService, + @IProductService private readonly _productService: IProductService, + @ILogService private readonly _logService: ILogService, + @IInstantiationService private readonly _instantiationService: IInstantiationService, + @IServerLifetimeService private readonly _serverLifetimeService: IServerLifetimeService, + ) { + super(); + if (serverBasePath !== undefined && serverBasePath.charCodeAt(serverBasePath.length - 1) === CharCode.Slash) { + // Remove trailing slash from base path + serverBasePath = serverBasePath.substring(0, serverBasePath.length - 1); + } + this._serverBasePath = serverBasePath; // undefined or starts with a slash + this._serverProductPath = `/${getServerProductSegment(_productService)}`; // starts with a slash + this._extHostConnections = Object.create(null); + this._managementConnections = Object.create(null); + this._allReconnectionTokens = new Set(); + this._logService.info(`Extension host agent started.`); + this._reconnectionGraceTime = this._environmentService.reconnectionGraceTime; + } + + public async handleRequest(req: http.IncomingMessage, res: http.ServerResponse): Promise { + // Only serve GET requests + if (req.method !== 'GET') { + return serveError(req, res, 405, `Unsupported method ${req.method}`); + } + + if (!req.url) { + return serveError(req, res, 400, `Bad request.`); + } + + const parsedUrl = url.parse(req.url, true); + let pathname = parsedUrl.pathname; + + if (!pathname) { + return serveError(req, res, 400, `Bad request.`); + } + + // Serve from both '/' and serverBasePath + if (this._serverBasePath !== undefined && pathname.startsWith(this._serverBasePath)) { + pathname = pathname.substring(this._serverBasePath.length) || '/'; + } + // for now accept all paths, with or without server product path + if (pathname.startsWith(this._serverProductPath) && pathname.charCodeAt(this._serverProductPath.length) === CharCode.Slash) { + pathname = pathname.substring(this._serverProductPath.length); + } + + // Version + if (pathname === '/version') { + res.writeHead(200, { 'Content-Type': 'text/plain' }); + return void res.end(this._productService.commit || ''); + } + + // Delay shutdown + if (pathname === '/delay-shutdown') { + this._serverLifetimeService.delay(); + res.writeHead(200); + return void res.end('OK'); + } + + if (!httpRequestHasValidConnectionToken(this._connectionToken, req, parsedUrl)) { + // invalid connection token + return serveError(req, res, 403, `Forbidden.`); + } + + // Whiteboard: the web client and `/vscode-remote-resource` are cut. + res.writeHead(404, { 'Content-Type': 'text/plain' }); + return void res.end('Not found'); + } + + public handleUpgrade(req: http.IncomingMessage, socket: net.Socket) { + let reconnectionToken = generateUuid(); + let isReconnection = false; + let skipWebSocketFrames = false; + + if (req.url) { + const query = url.parse(req.url, true).query; + if (typeof query.reconnectionToken === 'string') { + reconnectionToken = query.reconnectionToken; + } + if (query.reconnection === 'true') { + isReconnection = true; + } + if (query.skipWebSocketFrames === 'true') { + skipWebSocketFrames = true; + } + } + + const upgraded = upgradeToISocket(req, socket, { + debugLabel: `server-connection-${reconnectionToken}`, + skipWebSocketFrames, + disableWebSocketCompression: this._environmentService.args['disable-websocket-compression'] + }); + + if (!upgraded) { + return; + } + + this._handleWebSocketConnection(upgraded, isReconnection, reconnectionToken); + } + + public handleServerError(err: Error): void { + this._logService.error(`Error occurred in server`); + this._logService.error(err); + } + + // Eventually cleanup + + private _getRemoteAddress(socket: NodeSocket | WebSocketNodeSocket): string { + let _socket: net.Socket; + if (socket instanceof NodeSocket) { + _socket = socket.socket; + } else { + _socket = socket.socket.socket; + } + return _socket.remoteAddress || ``; + } + + private async _rejectWebSocketConnection(logPrefix: string, protocol: PersistentProtocol, reason: string): Promise { + const socket = protocol.getSocket(); + this._logService.error(`${logPrefix} ${reason}.`); + const errMessage: ErrorMessage = { + type: 'error', + reason: reason + }; + protocol.sendControl(VSBuffer.fromString(JSON.stringify(errMessage))); + protocol.dispose(); + await socket.drain(); + socket.dispose(); + } + + /** + * NOTE: Avoid using await in this method! + * The problem is that await introduces a process.nextTick due to the implicit Promise.then + * This can lead to some bytes being received and interpreted and a control message being emitted before the next listener has a chance to be registered. + */ + private _handleWebSocketConnection(socket: NodeSocket | WebSocketNodeSocket, isReconnection: boolean, reconnectionToken: string): void { + const remoteAddress = this._getRemoteAddress(socket); + const logPrefix = `[${remoteAddress}][${reconnectionToken.substr(0, 8)}]`; + const protocol = new PersistentProtocol({ socket }); + + const validator = this._vsdaMod ? new this._vsdaMod.validator() : null; + const signer = this._vsdaMod ? new this._vsdaMod.signer() : null; + + const enum State { + WaitingForAuth, + WaitingForConnectionType, + Done, + Error + } + let state = State.WaitingForAuth; + + const rejectWebSocketConnection = (msg: string) => { + state = State.Error; + listener.dispose(); + this._rejectWebSocketConnection(logPrefix, protocol, msg); + }; + + const listener = protocol.onControlMessage((raw) => { + if (state === State.WaitingForAuth) { + let msg1: HandshakeMessage; + try { + msg1 = JSON.parse(raw.toString()); + } catch (err) { + return rejectWebSocketConnection(`Malformed first message`); + } + if (msg1.type !== 'auth') { + return rejectWebSocketConnection(`Invalid first message`); + } + + if (this._connectionToken.type === ServerConnectionTokenType.Mandatory && !this._connectionToken.validate(msg1.auth)) { + return rejectWebSocketConnection(`Unauthorized client refused: auth mismatch`); + } + + // Send `sign` request + let signedData = generateUuid(); + if (signer) { + try { + signedData = signer.sign(msg1.data); + } catch (e) { + } + } + let someText = generateUuid(); + if (validator) { + try { + someText = validator.createNewMessage(someText); + } catch (e) { + } + } + const signRequest: SignRequest = { + type: 'sign', + data: someText, + signedData: signedData + }; + protocol.sendControl(VSBuffer.fromString(JSON.stringify(signRequest))); + + state = State.WaitingForConnectionType; + + } else if (state === State.WaitingForConnectionType) { + + let msg2: HandshakeMessage; + try { + msg2 = JSON.parse(raw.toString()); + } catch (err) { + return rejectWebSocketConnection(`Malformed second message`); + } + if (msg2.type !== 'connectionType') { + return rejectWebSocketConnection(`Invalid second message`); + } + if (typeof msg2.signedData !== 'string') { + return rejectWebSocketConnection(`Invalid second message field type`); + } + + const rendererCommit = msg2.commit; + const myCommit = this._productService.commit; + if (rendererCommit && myCommit) { + // Running in the built version where commits are defined + if (rendererCommit !== myCommit) { + return rejectWebSocketConnection(`Client refused: version mismatch`); + } + } + + let valid = false; + if (!validator) { + valid = true; + } else if (this._connectionToken.validate(msg2.signedData)) { + // web client + valid = true; + } else { + try { + valid = validator.validate(msg2.signedData) === 'ok'; + } catch (e) { + } + } + + if (!valid) { + if (this._environmentService.isBuilt) { + return rejectWebSocketConnection(`Unauthorized client refused`); + } else { + this._logService.error(`${logPrefix} Unauthorized client handshake failed but we proceed because of dev mode.`); + } + } + + // We have received a new connection. + // This indicates that the server owner has connectivity. + // Therefore we will shorten the reconnection grace period for disconnected connections! + for (const key in this._managementConnections) { + const managementConnection = this._managementConnections[key]; + managementConnection.shortenReconnectionGraceTimeIfNecessary(); + } + for (const key in this._extHostConnections) { + const extHostConnection = this._extHostConnections[key]; + extHostConnection.shortenReconnectionGraceTimeIfNecessary(); + } + + state = State.Done; + listener.dispose(); + this._handleConnectionType(remoteAddress, logPrefix, protocol, socket, isReconnection, reconnectionToken, msg2); + } + }); + } + + private async _handleConnectionType(remoteAddress: string, _logPrefix: string, protocol: PersistentProtocol, socket: NodeSocket | WebSocketNodeSocket, isReconnection: boolean, reconnectionToken: string, msg: ConnectionTypeRequest): Promise { + const logPrefix = ( + msg.desiredConnectionType === ConnectionType.Management + ? `${_logPrefix}[ManagementConnection]` + : msg.desiredConnectionType === ConnectionType.ExtensionHost + ? `${_logPrefix}[ExtensionHostConnection]` + : _logPrefix + ); + + if (msg.desiredConnectionType === ConnectionType.Management) { + // This should become a management connection + if (socket instanceof WebSocketNodeSocket) { + socket.setRecordInflateBytes(false); + } + + if (isReconnection) { + // This is a reconnection + if (!this._managementConnections[reconnectionToken]) { + if (!this._allReconnectionTokens.has(reconnectionToken)) { + // This is an unknown reconnection token + return this._rejectWebSocketConnection(logPrefix, protocol, `Unknown reconnection token (never seen)`); + } else { + // This is a connection that was seen in the past, but is no longer valid + return this._rejectWebSocketConnection(logPrefix, protocol, `Unknown reconnection token (seen before)`); + } + } + + protocol.sendControl(VSBuffer.fromString(JSON.stringify({ type: 'ok' }))); + const dataChunk = protocol.readEntireBuffer(); + protocol.dispose(); + this._managementConnections[reconnectionToken].acceptReconnection(remoteAddress, socket, dataChunk); + + } else { + // This is a fresh connection + if (this._managementConnections[reconnectionToken]) { + // Cannot have two concurrent connections using the same reconnection token + return this._rejectWebSocketConnection(logPrefix, protocol, `Duplicate reconnection token`); + } + + protocol.sendControl(VSBuffer.fromString(JSON.stringify({ type: 'ok' }))); + const con = new ManagementConnection(this._logService, reconnectionToken, remoteAddress, protocol, this._reconnectionGraceTime); + this._socketServer.acceptConnection(con.protocol, con.onClose); + this._managementConnections[reconnectionToken] = con; + this._allReconnectionTokens.add(reconnectionToken); + con.onClose(() => { + delete this._managementConnections[reconnectionToken]; + }); + + } + + } else if (msg.desiredConnectionType === ConnectionType.ExtensionHost) { + + // This should become an extension host connection + const startParams0 = msg.args || { language: 'en' }; + const startParams = await this._updateWithFreeDebugPort(startParams0); + + if (startParams.port) { + this._logService.trace(`${logPrefix} - startParams debug port ${startParams.port}`); + } + this._logService.trace(`${logPrefix} - startParams language: ${startParams.language}`); + this._logService.trace(`${logPrefix} - startParams env: ${JSON.stringify(startParams.env)}`); + + if (isReconnection) { + // This is a reconnection + if (!this._extHostConnections[reconnectionToken]) { + if (!this._allReconnectionTokens.has(reconnectionToken)) { + // This is an unknown reconnection token + return this._rejectWebSocketConnection(logPrefix, protocol, `Unknown reconnection token (never seen)`); + } else { + // This is a connection that was seen in the past, but is no longer valid + return this._rejectWebSocketConnection(logPrefix, protocol, `Unknown reconnection token (seen before)`); + } + } + + protocol.sendPause(); + protocol.sendControl(VSBuffer.fromString(JSON.stringify(startParams.port ? { debugPort: startParams.port } : {}))); + const dataChunk = protocol.readEntireBuffer(); + protocol.dispose(); + this._extHostConnections[reconnectionToken].acceptReconnection(remoteAddress, socket, dataChunk); + + } else { + // This is a fresh connection + if (this._extHostConnections[reconnectionToken]) { + // Cannot have two concurrent connections using the same reconnection token + return this._rejectWebSocketConnection(logPrefix, protocol, `Duplicate reconnection token`); + } + + protocol.sendPause(); + protocol.sendControl(VSBuffer.fromString(JSON.stringify(startParams.port ? { debugPort: startParams.port } : {}))); + const dataChunk = protocol.readEntireBuffer(); + protocol.dispose(); + const con = this._instantiationService.createInstance(ExtensionHostConnection, reconnectionToken, remoteAddress, socket, dataChunk); + this._extHostConnections[reconnectionToken] = con; + this._allReconnectionTokens.add(reconnectionToken); + this._extHostLifetimeTokens.set(reconnectionToken, this._serverLifetimeService.active(`ExtensionHost:${reconnectionToken.substring(0, 8)}`)); + con.onClose(() => { + con.dispose(); + delete this._extHostConnections[reconnectionToken]; + this._extHostLifetimeTokens.deleteAndDispose(reconnectionToken); + }); + con.start(startParams).catch(error => { + this._logService.error(`${logPrefix} Failed to start extension host connection:`, error); + }); + } + + } else if (msg.desiredConnectionType === ConnectionType.Tunnel) { + if (socket instanceof WebSocketNodeSocket) { + socket.setRecordInflateBytes(false); + } + + const tunnelStartParams = msg.args; + this._createTunnel(protocol, tunnelStartParams); + + } else { + + return this._rejectWebSocketConnection(logPrefix, protocol, `Unknown initial data received`); + + } + } + + private async _createTunnel(protocol: PersistentProtocol, tunnelStartParams: ITunnelConnectionStartParams): Promise { + let localSocket: net.Socket; + try { + localSocket = await this._connectTunnelSocket(tunnelStartParams.host, tunnelStartParams.port); + } catch (err) { + this._logService.error(`[remote-connection] Failed to connect tunnel to ${tunnelStartParams.host}:${tunnelStartParams.port}:`, err); + const reason = (err instanceof Error ? err.message : String(err)); + const errorMessage: ErrorMessage = { type: 'error', reason }; + protocol.sendControl(VSBuffer.fromString(JSON.stringify(errorMessage))); + const socket = protocol.getSocket(); + protocol.dispose(); + await socket.drain(); + socket.dispose(); + return; + } + + const okMessage: OKMessage = { type: 'ok' }; + protocol.sendControl(VSBuffer.fromString(JSON.stringify(okMessage))); + + const remoteNodeSocket = protocol.getSocket(); + const remoteSocket = remoteNodeSocket.socket; + const dataChunk = protocol.readEntireBuffer(); + protocol.dispose(); + remoteNodeSocket.dispose(false); // `false` prevents the underlying socket from being closed + + if (dataChunk.byteLength > 0) { + localSocket.write(dataChunk.buffer); + } + + localSocket.on('end', () => remoteSocket.end()); + localSocket.on('close', () => remoteSocket.end()); + localSocket.on('error', () => remoteSocket.destroy()); + remoteSocket.on('end', () => localSocket.end()); + remoteSocket.on('close', () => localSocket.end()); + remoteSocket.on('error', () => localSocket.destroy()); + + localSocket.pipe(remoteSocket); + remoteSocket.pipe(localSocket); + } + + private _connectTunnelSocket(host: string, port: number): Promise { + return new Promise((c, e) => { + const socket = net.createConnection( + { + host: host, + port: port, + autoSelectFamily: true + }, () => { + socket.removeListener('error', e); + socket.pause(); + c(socket); + } + ); + + socket.once('error', e); + }); + } + + private _updateWithFreeDebugPort(startParams: IRemoteExtensionHostStartParams): Thenable { + if (typeof startParams.port === 'number') { + return findFreePort(startParams.port, 10 /* try 10 ports */, 5000 /* try up to 5 seconds */).then(freePort => { + startParams.port = freePort; + return startParams; + }); + } + // No port clear debug configuration. + startParams.debugId = undefined; + startParams.port = undefined; + startParams.break = undefined; + return Promise.resolve(startParams); + } +} + +export interface IServerAPI { + /** + * Do not remove!!. Called from server-main.js + */ + handleRequest(req: http.IncomingMessage, res: http.ServerResponse): Promise; + /** + * Do not remove!!. Called from server-main.js + */ + handleUpgrade(req: http.IncomingMessage, socket: net.Socket): void; + /** + * Do not remove!!. Called from server-main.js + */ + handleServerError(err: Error): void; + /** + * Do not remove!!. Called from server-main.js + */ + dispose(): void; +} + +export async function createServer(address: string | net.AddressInfo | null, args: ServerParsedArgs, REMOTE_DATA_FOLDER: string): Promise { + + const connectionToken = await determineServerConnectionToken(args); + if (connectionToken instanceof ServerConnectionTokenParseError) { + console.warn(connectionToken.message); + process.exit(1); + } + + // setting up error handlers, first with console.error, then, once available, using the log service + + function initUnexpectedErrorHandler(handler: (err: any) => void) { + setUnexpectedErrorHandler(err => { + // See https://github.com/microsoft/vscode-remote-release/issues/6481 + // In some circumstances, console.error will throw an asynchronous error. This asynchronous error + // will end up here, and then it will be logged again, thus creating an endless asynchronous loop. + // Here we try to break the loop by ignoring EPIPE errors that include our own unexpected error handler in the stack. + if (isSigPipeError(err) && err.stack && /unexpectedErrorHandler/.test(err.stack)) { + return; + } + handler(err); + }); + } + + const unloggedErrors: any[] = []; + initUnexpectedErrorHandler((error: any) => { + unloggedErrors.push(error); + console.error(error); + }); + let didLogAboutSIGPIPE = false; + process.on('SIGPIPE', () => { + // See https://github.com/microsoft/vscode-remote-release/issues/6543 + // We would normally install a SIGPIPE listener in bootstrap-node.js + // But in certain situations, the console itself can be in a broken pipe state + // so logging SIGPIPE to the console will cause an infinite async loop + if (!didLogAboutSIGPIPE) { + didLogAboutSIGPIPE = true; + onUnexpectedError(new Error(`Unexpected SIGPIPE`)); + } + }); + + const disposables = new DisposableStore(); + const { socketServer, instantiationService } = await setupServerServices(connectionToken, args, REMOTE_DATA_FOLDER, disposables); + + // Set the unexpected error handler after the services have been initialized, to avoid having + // the telemetry service overwrite our handler + instantiationService.invokeFunction((accessor) => { + const logService = accessor.get(ILogService); + unloggedErrors.forEach(error => logService.error(error)); + unloggedErrors.length = 0; + + initUnexpectedErrorHandler((error: any) => logService.error(error)); + }); + + // On Windows, configure the UNC allow list based on settings + instantiationService.invokeFunction((accessor) => { + const configurationService = accessor.get(IConfigurationService); + + if (platform.isWindows) { + if (configurationService.getValue('security.restrictUNCAccess') === false) { + disableUNCAccessRestrictions(); + } else { + addUNCHostToAllowlist(configurationService.getValue('security.allowedUNCHosts')); + } + } + }); + + // + // On Windows, exit early with warning message to users about potential security issue + // if there is node_modules folder under home drive or Users folder. + // + instantiationService.invokeFunction((accessor) => { + const logService = accessor.get(ILogService); + + if (platform.isWindows && process.env.HOMEDRIVE && process.env.HOMEPATH) { + const homeDirModulesPath = join(process.env.HOMEDRIVE, 'node_modules'); + const userDir = dirname(join(process.env.HOMEDRIVE, process.env.HOMEPATH)); + const userDirModulesPath = join(userDir, 'node_modules'); + if (fs.existsSync(homeDirModulesPath) || fs.existsSync(userDirModulesPath)) { + const message = ` + +* +* !!!! Server terminated due to presence of CVE-2020-1416 !!!! +* +* Please remove the following directories and re-try +* ${homeDirModulesPath} +* ${userDirModulesPath} +* +* For more information on the vulnerability https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-1416 +* + +`; + logService.warn(message); + console.warn(message); + process.exit(0); + } + } + }); + + const vsdaMod = instantiationService.invokeFunction((accessor) => { + const logService = accessor.get(ILogService); + const hasVSDA = fs.existsSync(join(FileAccess.asFileUri('').fsPath, '../node_modules/vsda')); + if (hasVSDA) { + try { + return require('vsda'); + } catch (err) { + logService.error(err); + } + } + return null; + }); + + let serverBasePath = args['server-base-path']; + if (serverBasePath && !serverBasePath.startsWith('/')) { + serverBasePath = `/${serverBasePath}`; + } + + const remoteExtensionHostAgentServer = instantiationService.createInstance(RemoteExtensionHostAgentServer, socketServer, connectionToken, vsdaMod, serverBasePath); + + perf.mark('code/server/ready'); + const currentTime = performance.now(); + // eslint-disable-next-line local/code-no-any-casts + const vscodeServerStartTime: number = (global).vscodeServerStartTime; + // eslint-disable-next-line local/code-no-any-casts + const vscodeServerListenTime: number = (global).vscodeServerListenTime; + // eslint-disable-next-line local/code-no-any-casts + const vscodeServerCodeLoadedTime: number = (global).vscodeServerCodeLoadedTime; + + instantiationService.invokeFunction(async (accessor) => { + const telemetryService = accessor.get(ITelemetryService); + + type ServerStartClassification = { + owner: 'alexdima'; + comment: 'The server has started up'; + startTime: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The time the server started at.' }; + startedTime: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The time the server began listening for connections.' }; + codeLoadedTime: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The time which the code loaded on the server' }; + readyTime: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The time when the server was completely ready' }; + }; + type ServerStartEvent = { + startTime: number; + startedTime: number; + codeLoadedTime: number; + readyTime: number; + }; + telemetryService.publicLog2('serverStart', { + startTime: vscodeServerStartTime, + startedTime: vscodeServerListenTime, + codeLoadedTime: vscodeServerCodeLoadedTime, + readyTime: currentTime + }); + + if (platform.isLinux) { + const logService = accessor.get(ILogService); + const releaseInfo = await getOSReleaseInfo(logService.error.bind(logService)); + if (releaseInfo) { + type ServerPlatformInfoClassification = { + platformId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'A string identifying the operating system without any version information.' }; + platformVersionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'A string identifying the operating system version excluding any name information or release code.' }; + platformIdLike: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'A string identifying the operating system the current OS derivate is closely related to.' }; + owner: 'deepak1556'; + comment: 'Provides insight into the distro information on Linux.'; + }; + type ServerPlatformInfoEvent = { + platformId: string; + platformVersionId: string | undefined; + platformIdLike: string | undefined; + }; + telemetryService.publicLog2('serverPlatformInfo', { + platformId: releaseInfo.id, + platformVersionId: releaseInfo.version_id, + platformIdLike: releaseInfo.id_like + }); + } + } + }); + + if (args['print-startup-performance']) { + let output = ''; + output += `Start-up time: ${vscodeServerListenTime - vscodeServerStartTime}\n`; + output += `Code loading time: ${vscodeServerCodeLoadedTime - vscodeServerStartTime}\n`; + output += `Initialized time: ${currentTime - vscodeServerStartTime}\n`; + output += `\n`; + console.log(output); + } + + return remoteExtensionHostAgentServer; +} + +// Whiteboard: replaces the web client server's `serveError`. +function serveError(req: http.IncomingMessage, res: http.ServerResponse, errorCode: number, errorMessage: string): void { + res.writeHead(errorCode, { 'Content-Type': 'text/plain' }); + res.end(errorMessage); +} diff --git a/apps/review-desktop/code-oss/src/vs/server/node/remoteExtensionManagement.ts b/apps/review-desktop/code-oss/src/vs/server/node/remoteExtensionManagement.ts new file mode 100644 index 000000000..4a38e83ea --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/server/node/remoteExtensionManagement.ts @@ -0,0 +1,128 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { PersistentProtocol, ISocket, ProtocolConstants } from '../../base/parts/ipc/common/ipc.net.js'; +import { ILogService } from '../../platform/log/common/log.js'; +import { Emitter, Event } from '../../base/common/event.js'; +import { VSBuffer } from '../../base/common/buffer.js'; +import { ProcessTimeRunOnceScheduler } from '../../base/common/async.js'; +import { IDisposable } from '../../base/common/lifecycle.js'; + +function printTime(ms: number): string { + let h = 0; + let m = 0; + let s = 0; + if (ms >= 1000) { + s = Math.floor(ms / 1000); + ms -= s * 1000; + } + if (s >= 60) { + m = Math.floor(s / 60); + s -= m * 60; + } + if (m >= 60) { + h = Math.floor(m / 60); + m -= h * 60; + } + const _h = h ? `${h}h` : ``; + const _m = m ? `${m}m` : ``; + const _s = s ? `${s}s` : ``; + const _ms = ms ? `${ms}ms` : ``; + return `${_h}${_m}${_s}${_ms}`; +} + +export class ManagementConnection { + + private _onClose = new Emitter(); + public readonly onClose: Event = this._onClose.event; + + private readonly _reconnectionGraceTime: number; + private readonly _reconnectionShortGraceTime: number; + private _remoteAddress: string; + + public readonly protocol: PersistentProtocol; + private _disposed: boolean; + private _disconnectRunner1: ProcessTimeRunOnceScheduler; + private _disconnectRunner2: ProcessTimeRunOnceScheduler; + private readonly _socketCloseListener: IDisposable; + + constructor( + private readonly _logService: ILogService, + private readonly _reconnectionToken: string, + remoteAddress: string, + protocol: PersistentProtocol, + reconnectionGraceTime: number + ) { + this._reconnectionGraceTime = reconnectionGraceTime; + const defaultShortGrace = ProtocolConstants.ReconnectionShortGraceTime; + this._reconnectionShortGraceTime = reconnectionGraceTime > 0 ? Math.min(defaultShortGrace, reconnectionGraceTime) : 0; + this._remoteAddress = remoteAddress; + + this.protocol = protocol; + this._disposed = false; + this._disconnectRunner1 = new ProcessTimeRunOnceScheduler(() => { + this._log(`The reconnection grace time of ${printTime(this._reconnectionGraceTime)} has expired, so the connection will be disposed.`); + this._cleanResources(); + }, this._reconnectionGraceTime); + this._disconnectRunner2 = new ProcessTimeRunOnceScheduler(() => { + this._log(`The reconnection short grace time of ${printTime(this._reconnectionShortGraceTime)} has expired, so the connection will be disposed.`); + this._cleanResources(); + }, this._reconnectionShortGraceTime); + + Event.once(this.protocol.onDidDispose)(() => { + this._log(`The client has disconnected gracefully, so the connection will be disposed.`); + this._cleanResources(); + }); + this._socketCloseListener = this.protocol.onSocketClose(() => { + this._log(`The client has disconnected, will wait for reconnection ${printTime(this._reconnectionGraceTime)} before disposing...`); + // The socket has closed, let's give the renderer a certain amount of time to reconnect + this._disconnectRunner1.schedule(); + }); + + this._log(`New connection established.`); + } + + private _log(_str: string): void { + this._logService.info(`[${this._remoteAddress}][${this._reconnectionToken.substr(0, 8)}][ManagementConnection] ${_str}`); + } + + public shortenReconnectionGraceTimeIfNecessary(): void { + if (this._disconnectRunner2.isScheduled()) { + // we are disconnected and already running the short reconnection timer + return; + } + if (this._disconnectRunner1.isScheduled()) { + this._log(`Another client has connected, will shorten the wait for reconnection ${printTime(this._reconnectionShortGraceTime)} before disposing...`); + // we are disconnected and running the long reconnection timer + this._disconnectRunner2.schedule(); + } + } + + private _cleanResources(): void { + if (this._disposed) { + // already called + return; + } + this._disposed = true; + this._disconnectRunner1.dispose(); + this._disconnectRunner2.dispose(); + this._socketCloseListener.dispose(); + const socket = this.protocol.getSocket(); + this.protocol.sendDisconnect(); + this.protocol.dispose(); + socket.end(); + this._onClose.fire(undefined); + this._onClose.dispose(); + } + + public acceptReconnection(remoteAddress: string, socket: ISocket, initialDataChunk: VSBuffer): void { + this._remoteAddress = remoteAddress; + this._log(`The client has reconnected.`); + this._disconnectRunner1.cancel(); + this._disconnectRunner2.cancel(); + this.protocol.beginAcceptReconnection(socket, initialDataChunk); + this.protocol.endAcceptReconnection(); + } +} diff --git a/apps/review-desktop/code-oss/src/vs/server/node/remoteExtensionsScanner.ts b/apps/review-desktop/code-oss/src/vs/server/node/remoteExtensionsScanner.ts new file mode 100644 index 000000000..7fc7bbe93 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/server/node/remoteExtensionsScanner.ts @@ -0,0 +1,250 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { resolve } from '../../base/common/path.js'; +import * as platform from '../../base/common/platform.js'; +import { URI } from '../../base/common/uri.js'; +import * as performance from '../../base/common/performance.js'; +import { Event } from '../../base/common/event.js'; +import { IURITransformer, transformOutgoingURIs } from '../../base/common/uriIpc.js'; +import { IServerChannel } from '../../base/parts/ipc/common/ipc.js'; +import { ContextKeyDefinedExpr, ContextKeyEqualsExpr, ContextKeyExpr, ContextKeyExpression, ContextKeyGreaterEqualsExpr, ContextKeyGreaterExpr, ContextKeyInExpr, ContextKeyNotEqualsExpr, ContextKeyNotExpr, ContextKeyNotInExpr, ContextKeyRegexExpr, ContextKeySmallerEqualsExpr, ContextKeySmallerExpr, ContextKeyValue, IContextKeyExprMapper } from '../../platform/contextkey/common/contextkey.js'; +import { InstallExtensionSummary } from '../../platform/extensionManagement/common/extensionManagement.js'; +import { IExtensionsScannerService, toExtensionDescription } from '../../platform/extensionManagement/common/extensionsScannerService.js'; +import { ExtensionType, IExtensionDescription } from '../../platform/extensions/common/extensions.js'; +import { ILogService } from '../../platform/log/common/log.js'; +import { IUserDataProfilesService } from '../../platform/userDataProfile/common/userDataProfile.js'; +import { dedupExtensions } from '../../workbench/services/extensions/common/extensionsUtil.js'; +import { Schemas } from '../../base/common/network.js'; +import { IRemoteExtensionsScannerService } from '../../platform/remote/common/remoteExtensionsScanner.js'; + +export class RemoteExtensionsScannerService implements IRemoteExtensionsScannerService { + + readonly _serviceBrand: undefined; + + // Whiteboard: install-on-start, the gallery and language packs are cut. + private readonly _whenExtensionsReady = Promise.resolve({ failed: [] }); + + constructor( + private readonly _userDataProfilesService: IUserDataProfilesService, + private readonly _extensionsScannerService: IExtensionsScannerService, + private readonly _logService: ILogService, + ) { } + + whenExtensionsReady(): Promise { + return this._whenExtensionsReady; + } + + async scanExtensions( + language?: string, + profileLocation?: URI, + workspaceExtensionLocations?: URI[], + extensionDevelopmentLocations?: URI[], + languagePackId?: string + ): Promise { + performance.mark('code/server/willScanExtensions'); + this._logService.trace(`Scanning extensions using UI language: ${language}`); + + const extensionDevelopmentPaths = extensionDevelopmentLocations ? extensionDevelopmentLocations.filter(url => url.scheme === Schemas.file).map(url => url.fsPath) : undefined; + profileLocation = profileLocation ?? this._userDataProfilesService.defaultProfile.extensionsResource; + + const extensions = await this._scanExtensions(profileLocation, language ?? platform.language, workspaceExtensionLocations, extensionDevelopmentPaths, languagePackId); + + this._logService.trace('Scanned Extensions', extensions); + this._massageWhenConditions(extensions); + + performance.mark('code/server/didScanExtensions'); + return extensions; + } + + private async _scanExtensions(profileLocation: URI, language: string, workspaceInstalledExtensionLocations: URI[] | undefined, extensionDevelopmentPath: string[] | undefined, languagePackId: string | undefined): Promise { + const [builtinExtensions, installedExtensions, workspaceInstalledExtensions, developedExtensions] = await Promise.all([ + this._scanBuiltinExtensions(language), + this._scanInstalledExtensions(profileLocation, language), + this._scanWorkspaceInstalledExtensions(language, workspaceInstalledExtensionLocations), + this._scanDevelopedExtensions(language, extensionDevelopmentPath) + ]); + + return dedupExtensions(builtinExtensions, installedExtensions, workspaceInstalledExtensions, developedExtensions, this._logService); + } + + private async _scanDevelopedExtensions(language: string, extensionDevelopmentPaths?: string[]): Promise { + if (extensionDevelopmentPaths) { + return (await Promise.all(extensionDevelopmentPaths.map(extensionDevelopmentPath => this._extensionsScannerService.scanOneOrMultipleExtensions(URI.file(resolve(extensionDevelopmentPath)), ExtensionType.User, { language })))) + .flat() + .map(e => toExtensionDescription(e, true)); + } + return []; + } + + private async _scanWorkspaceInstalledExtensions(language: string, workspaceInstalledExtensions?: URI[]): Promise { + const result: IExtensionDescription[] = []; + if (workspaceInstalledExtensions?.length) { + const scannedExtensions = await Promise.all(workspaceInstalledExtensions.map(location => this._extensionsScannerService.scanExistingExtension(location, ExtensionType.User, { language }))); + for (const scannedExtension of scannedExtensions) { + if (scannedExtension) { + result.push(toExtensionDescription(scannedExtension, false)); + } + } + } + return result; + } + + private async _scanBuiltinExtensions(language: string): Promise { + const scannedExtensions = await this._extensionsScannerService.scanSystemExtensions({ language }); + return scannedExtensions.map(e => toExtensionDescription(e, false)); + } + + private async _scanInstalledExtensions(profileLocation: URI, language: string): Promise { + const scannedExtensions = await this._extensionsScannerService.scanUserExtensions({ profileLocation, language, useCache: true }); + return scannedExtensions.map(e => toExtensionDescription(e, false)); + } + + private _massageWhenConditions(extensions: IExtensionDescription[]): void { + // Massage "when" conditions which mention `resourceScheme` + + interface WhenUser { when?: string } + + interface LocWhenUser { [loc: string]: WhenUser[] } + + const _mapResourceSchemeValue = (value: string, isRegex: boolean): string => { + // console.log(`_mapResourceSchemeValue: ${value}, ${isRegex}`); + return value.replace(/file/g, 'vscode-remote'); + }; + + const _mapResourceRegExpValue = (value: RegExp): RegExp => { + let flags = ''; + flags += value.global ? 'g' : ''; + flags += value.ignoreCase ? 'i' : ''; + flags += value.multiline ? 'm' : ''; + return new RegExp(_mapResourceSchemeValue(value.source, true), flags); + }; + + const _exprKeyMapper = new class implements IContextKeyExprMapper { + mapDefined(key: string): ContextKeyExpression { + return ContextKeyDefinedExpr.create(key); + } + mapNot(key: string): ContextKeyExpression { + return ContextKeyNotExpr.create(key); + } + mapEquals(key: string, value: ContextKeyValue): ContextKeyExpression { + if (key === 'resourceScheme' && typeof value === 'string') { + return ContextKeyEqualsExpr.create(key, _mapResourceSchemeValue(value, false)); + } else { + return ContextKeyEqualsExpr.create(key, value); + } + } + mapNotEquals(key: string, value: ContextKeyValue): ContextKeyExpression { + if (key === 'resourceScheme' && typeof value === 'string') { + return ContextKeyNotEqualsExpr.create(key, _mapResourceSchemeValue(value, false)); + } else { + return ContextKeyNotEqualsExpr.create(key, value); + } + } + mapGreater(key: string, value: ContextKeyValue): ContextKeyExpression { + return ContextKeyGreaterExpr.create(key, value); + } + mapGreaterEquals(key: string, value: ContextKeyValue): ContextKeyExpression { + return ContextKeyGreaterEqualsExpr.create(key, value); + } + mapSmaller(key: string, value: ContextKeyValue): ContextKeyExpression { + return ContextKeySmallerExpr.create(key, value); + } + mapSmallerEquals(key: string, value: ContextKeyValue): ContextKeyExpression { + return ContextKeySmallerEqualsExpr.create(key, value); + } + mapRegex(key: string, regexp: RegExp | null): ContextKeyRegexExpr { + if (key === 'resourceScheme' && regexp) { + return ContextKeyRegexExpr.create(key, _mapResourceRegExpValue(regexp)); + } else { + return ContextKeyRegexExpr.create(key, regexp); + } + } + mapIn(key: string, valueKey: string): ContextKeyInExpr { + return ContextKeyInExpr.create(key, valueKey); + } + mapNotIn(key: string, valueKey: string): ContextKeyNotInExpr { + return ContextKeyNotInExpr.create(key, valueKey); + } + }; + + const _massageWhenUser = (element: WhenUser) => { + if (!element || !element.when || !/resourceScheme/.test(element.when)) { + return; + } + + const expr = ContextKeyExpr.deserialize(element.when); + if (!expr) { + return; + } + + const massaged = expr.map(_exprKeyMapper); + element.when = massaged.serialize(); + }; + + const _massageWhenUserArr = (elements: WhenUser[] | WhenUser) => { + if (Array.isArray(elements)) { + for (const element of elements) { + _massageWhenUser(element); + } + } else { + _massageWhenUser(elements); + } + }; + + const _massageLocWhenUser = (target: LocWhenUser) => { + for (const loc in target) { + _massageWhenUserArr(target[loc]); + } + }; + + extensions.forEach((extension) => { + if (extension.contributes) { + if (extension.contributes.menus) { + _massageLocWhenUser(extension.contributes.menus); + } + if (extension.contributes.keybindings) { + _massageWhenUserArr(extension.contributes.keybindings); + } + if (extension.contributes.views) { + _massageLocWhenUser(extension.contributes.views); + } + } + }); + } +} + +export class RemoteExtensionsScannerChannel implements IServerChannel { + + constructor(private service: RemoteExtensionsScannerService, private getUriTransformer: (requestContext: any) => IURITransformer) { } + + listen(context: any, event: string): Event { + throw new Error('Invalid listen'); + } + + async call(context: any, command: string, args?: any): Promise { + const uriTransformer = this.getUriTransformer(context); + switch (command) { + case 'whenExtensionsReady': return await this.service.whenExtensionsReady(); + + case 'scanExtensions': { + const language = args[0]; + const profileLocation = args[1] ? URI.revive(uriTransformer.transformIncoming(args[1])) : undefined; + const workspaceExtensionLocations = Array.isArray(args[2]) ? args[2].map(u => URI.revive(uriTransformer.transformIncoming(u))) : undefined; + const extensionDevelopmentPath = Array.isArray(args[3]) ? args[3].map(u => URI.revive(uriTransformer.transformIncoming(u))) : undefined; + const languagePackId: string | undefined = args[4]; + const extensions = await this.service.scanExtensions( + language, + profileLocation, + workspaceExtensionLocations, + extensionDevelopmentPath, + languagePackId + ); + return extensions.map(extension => transformOutgoingURIs(extension, uriTransformer)); + } + } + throw new Error('Invalid call'); + } +} diff --git a/apps/review-desktop/code-oss/src/vs/server/node/remoteFileSystemProviderServer.ts b/apps/review-desktop/code-oss/src/vs/server/node/remoteFileSystemProviderServer.ts new file mode 100644 index 000000000..f97d2cc97 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/server/node/remoteFileSystemProviderServer.ts @@ -0,0 +1,98 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Emitter } from '../../base/common/event.js'; +import { URI, UriComponents } from '../../base/common/uri.js'; +import { IURITransformer } from '../../base/common/uriIpc.js'; +import { IFileChange } from '../../platform/files/common/files.js'; +import { ILogService } from '../../platform/log/common/log.js'; +import { createURITransformer } from '../../base/common/uriTransformer.js'; +import { RemoteAgentConnectionContext } from '../../platform/remote/common/remoteAgentEnvironment.js'; +import { DiskFileSystemProvider } from '../../platform/files/node/diskFileSystemProvider.js'; +import { posix, delimiter } from '../../base/common/path.js'; +import { IServerEnvironmentService } from './serverEnvironmentService.js'; +import { AbstractDiskFileSystemProviderChannel, AbstractSessionFileWatcher, ISessionFileWatcher } from '../../platform/files/node/diskFileSystemProviderServer.js'; +import { IRecursiveWatcherOptions } from '../../platform/files/common/watcher.js'; +import { IConfigurationService } from '../../platform/configuration/common/configuration.js'; + +export class RemoteAgentFileSystemProviderChannel extends AbstractDiskFileSystemProviderChannel { + + private readonly uriTransformerCache = new Map(); + + constructor( + logService: ILogService, + private readonly environmentService: IServerEnvironmentService, + private readonly configurationService: IConfigurationService + ) { + super(new DiskFileSystemProvider(logService), logService); + + this._register(this.provider); + } + + protected override getUriTransformer(ctx: RemoteAgentConnectionContext): IURITransformer { + let transformer = this.uriTransformerCache.get(ctx.remoteAuthority); + if (!transformer) { + transformer = createURITransformer(ctx.remoteAuthority); + this.uriTransformerCache.set(ctx.remoteAuthority, transformer); + } + + return transformer; + } + + protected override transformIncoming(uriTransformer: IURITransformer, _resource: UriComponents, supportVSCodeResource = false): URI { + if (supportVSCodeResource && _resource.path === '/vscode-resource' && _resource.query) { + const requestResourcePath = JSON.parse(_resource.query).requestResourcePath; + + return URI.from({ scheme: 'file', path: requestResourcePath }); + } + + return URI.revive(uriTransformer.transformIncoming(_resource)); + } + + //#region File Watching + + protected createSessionFileWatcher(uriTransformer: IURITransformer, emitter: Emitter): ISessionFileWatcher { + return new SessionFileWatcher(uriTransformer, emitter, this.logService, this.environmentService, this.configurationService); + } + + //#endregion +} + +class SessionFileWatcher extends AbstractSessionFileWatcher { + + constructor( + uriTransformer: IURITransformer, + sessionEmitter: Emitter, + logService: ILogService, + environmentService: IServerEnvironmentService, + configurationService: IConfigurationService + ) { + super(uriTransformer, sessionEmitter, logService, environmentService); + } + + protected override getRecursiveWatcherOptions(environmentService: IServerEnvironmentService): IRecursiveWatcherOptions | undefined { + const fileWatcherPolling = environmentService.args['file-watcher-polling']; + if (fileWatcherPolling) { + const segments = fileWatcherPolling.split(delimiter); + const pollingInterval = Number(segments[0]); + if (pollingInterval > 0) { + const usePolling = segments.length > 1 ? segments.slice(1) : true; + return { usePolling, pollingInterval }; + } + } + + return undefined; + } + + protected override getExtraExcludes(environmentService: IServerEnvironmentService): string[] | undefined { + if (environmentService.extensionsPath) { + // when opening the $HOME folder, we end up watching the extension folder + // so simply exclude watching the extensions folder + return [posix.join(environmentService.extensionsPath, '**')]; + } + + return undefined; + } +} diff --git a/apps/review-desktop/code-oss/src/vs/server/node/remoteLanguagePacks.ts b/apps/review-desktop/code-oss/src/vs/server/node/remoteLanguagePacks.ts new file mode 100644 index 000000000..1e6db87fb --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/server/node/remoteLanguagePacks.ts @@ -0,0 +1,39 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { FileAccess } from '../../base/common/network.js'; +import { join } from '../../base/common/path.js'; +import type { INLSConfiguration } from '../../nls.js'; +import { resolveNLSConfiguration } from '../../base/node/nls.js'; +import { Promises } from '../../base/node/pfs.js'; +import product from '../../platform/product/common/product.js'; + +const nlsMetadataPath = join(FileAccess.asFileUri('').fsPath); +const defaultMessagesFile = join(nlsMetadataPath, 'nls.messages.json'); +const nlsConfigurationCache = new Map>(); + +export async function getNLSConfiguration(language: string, userDataPath: string): Promise { + if (!product.commit || !(await Promises.exists(defaultMessagesFile))) { + return { + userLocale: 'en', + osLocale: 'en', + resolvedLanguage: 'en', + defaultMessagesFile, + + // NLS: below 2 are a relic from old times only used by vscode-nls and deprecated + locale: 'en', + availableLanguages: {} + }; + } + + const cacheKey = `${language}||${userDataPath}`; + let result = nlsConfigurationCache.get(cacheKey); + if (!result) { + result = resolveNLSConfiguration({ userLocale: language, osLocale: language, commit: product.commit, userDataPath, nlsMetadataPath }); + nlsConfigurationCache.set(cacheKey, result); + } + + return result; +} diff --git a/apps/review-desktop/code-oss/src/vs/server/node/reviewServerHandshake.test.ts b/apps/review-desktop/code-oss/src/vs/server/node/reviewServerHandshake.test.ts new file mode 100644 index 000000000..9bb92e665 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/server/node/reviewServerHandshake.test.ts @@ -0,0 +1,203 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) dev.fast. All rights reserved. + * Licensed under the MIT License. See LICENSE in the repository root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'node:assert/strict'; +import { ChildProcess, execFileSync, spawn } from 'node:child_process'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { after, before, test } from 'node:test'; +import { Schemas } from '../../base/common/network.js'; +import { URI } from '../../base/common/uri.js'; +import { LoadEstimator } from '../../base/parts/ipc/common/ipc.net.js'; +import { NullLogService } from '../../platform/log/common/log.js'; +import { connectRemoteAgentExtensionHost, connectRemoteAgentManagement, IConnectionOptions } from '../../platform/remote/common/remoteAgentConnection.js'; +import { RemoteConnectionType, WebSocketRemoteConnection } from '../../platform/remote/common/remoteAuthorityResolver.js'; +import { RemoteSocketFactoryService } from '../../platform/remote/common/remoteSocketFactoryService.js'; +import { nodeSocketFactory } from '../../platform/remote/node/nodeSocketFactory.js'; +import { SignService } from '../../platform/sign/node/signService.js'; +import { createMessageOfType, isMessageOfType, MessageType } from '../../workbench/services/extensions/common/extensionHostProtocol.js'; +import { RemoteExtensionEnvironmentChannelClient } from '../../workbench/services/remote/common/remoteAgentEnvironmentChannel.js'; + +const COMMIT = '0123456789abcdef0123456789abcdef01234567'; +const AUTHORITY = 'wb-test+handshake'; +const scripts = join(import.meta.dirname, '../../../../../scripts'); + +// The client's LoadEstimator ticks every second for the life of the process. +// Create it with an unreferenced interval so this test process can exit. +// The client's 10 s handshake timeouts are never cleared either; the package's +// test command runs this directory with --test-force-exit so they are not waited out. +const setIntervalReferenced = globalThis.setInterval; +globalThis.setInterval = ((...args: Parameters) => (setIntervalReferenced(...args) as unknown as NodeJS.Timeout).unref()) as unknown as typeof setInterval; +LoadEstimator.getInstance(); +globalThis.setInterval = setIntervalReferenced; + +interface Target { + readonly host: string; + readonly port: number; + readonly token: string; +} + +function connectionOptions(target: Target, overrides: { commit?: string; token?: string } = {}): IConnectionOptions { + const remoteSocketFactoryService = new RemoteSocketFactoryService(); + remoteSocketFactoryService.register(RemoteConnectionType.WebSocket, nodeSocketFactory); + return { + commit: overrides.commit ?? COMMIT, + quality: undefined, + addressProvider: { getAddress: async () => ({ connectTo: new WebSocketRemoteConnection(target.host, target.port), connectionToken: overrides.token ?? target.token }) }, + remoteSocketFactoryService, + signService: new SignService(), + logService: new NullLogService(), + ipcLogger: null, + }; +} + +async function handshake(target: Target, commit = COMMIT) { + const options = connectionOptions(target, { commit }); + const management = await connectRemoteAgentManagement(options, AUTHORITY, 'renderer'); + const environment = await RemoteExtensionEnvironmentChannelClient.getEnvironmentData(management.client.getChannel('remoteextensionsenvironment'), AUTHORITY, undefined); + const extensionHost = await connectRemoteAgentExtensionHost(options, { language: 'en' }); + await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error('the extension host did not send Ready')), 30_000); + const listener = extensionHost.protocol.onMessage(message => { + if (isMessageOfType(message, MessageType.Ready)) { + clearTimeout(timer); + listener.dispose(); + resolve(); + } + }); + }); + return { management, environment, extensionHost }; +} + +function close({ management, extensionHost }: Awaited>): void { + extensionHost.protocol.send(createMessageOfType(MessageType.Terminate)); + extensionHost.protocol.sendDisconnect(); + extensionHost.protocol.getSocket().end(); + extensionHost.protocol.dispose(); + extensionHost.dispose(); + management.dispose(); +} + +async function version(target: Target): Promise { + const response = await fetch(`http://${target.host}:${target.port}/version`); + return response.text(); +} + +const external = process.env['REVIEW_SERVER_HANDSHAKE_TARGET']; +if (external) { + test('completes the handshake with a server started elsewhere, which survives the client leaving', async () => { + const [host, port] = external.split(':'); + const tokenFile = process.env['REVIEW_SERVER_HANDSHAKE_TOKEN_FILE']; + assert.ok(tokenFile, 'REVIEW_SERVER_HANDSHAKE_TOKEN_FILE is required'); + const target = { host, port: Number(port), token: readFileSync(tokenFile, 'utf8').trim() }; + const started = Date.now(); + const connection = await handshake(target, await version(target)); + console.log(`extension host Ready after ${Date.now() - started} ms; server pid ${connection.environment.pid}, ${connection.environment.arch}`); + await new Promise(resolve => setTimeout(resolve, Number(process.env['REVIEW_SERVER_HANDSHAKE_HOLD_MS'] ?? 0))); + close(connection); + await new Promise(resolve => setTimeout(resolve, 2_000)); + assert.match(await version(target), /^[0-9a-f]{40}$/); + }); +} else { + let root: string; + let server: ChildProcess; + let output = ''; + let target: Target; + const servers: ChildProcess[] = []; + + async function startServer(extraArgs: string[] = []) { + const tokenFile = join(root, 'token'); + writeFileSync(tokenFile, 'test-token', { mode: 0o600 }); + const child = spawn(process.execPath, [ + join(root, 'remote-runtime', 'out/server-main.js'), + '--host', '127.0.0.1', + '--port', '0', + '--connection-token-file', tokenFile, + '--server-data-dir', join(root, 'data'), + '--extensions-dir', join(root, 'extensions'), + ...extraArgs, + ], { stdio: ['ignore', 'pipe', 'pipe'] }); + servers.push(child); + let text = ''; + child.stdout!.on('data', chunk => { text += chunk; output += chunk; }); + child.stderr!.on('data', chunk => output += chunk); + const port = await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`the server did not start:\n${output}`)), 30_000); + child.stdout!.on('data', () => { + const match = /Extension host agent listening on (\d+)/.exec(text); + if (match) { + clearTimeout(timer); + resolve(Number(match[1])); + } + }); + }); + return { child, target: { host: '127.0.0.1', port, token: 'test-token' } }; + } + + before(async () => { + root = mkdtempSync(join(tmpdir(), 'wb-server-')); + execFileSync(process.execPath, [join(scripts, 'build-remote-runtime.mjs'), '--out', join(root, 'remote-runtime'), '--commit', COMMIT], { stdio: 'pipe' }); + ({ child: server, target } = await startServer()); + }); + + after(async () => { + for (const child of servers) { + child.kill(); + } + for (const match of output.matchAll(/<(\d+)> Launched Extension Host Process/g)) { + try { + process.kill(Number(match[1])); + } catch { + } + } + rmSync(root, { recursive: true, force: true }); + }); + + test('completes a Management then an ExtensionHost connection, and the extension host sends Ready', async () => { + const connection = await handshake(target); + assert.equal(connection.environment.pid, server.pid); + assert.match(output, /Launched Extension Host Process/); + close(connection); + }); + + test('a client that disconnects cleanly leaves the server running', async () => { + const connection = await handshake(target); + const files = connection.management.client.getChannel('remoteFilesystem'); + const session = 'handshake-test'; + const listener = files.listen('fileChange', [session])(() => { }); + await files.call('watch', [session, 1, URI.from({ scheme: Schemas.vscodeRemote, authority: AUTHORITY, path: root }), { recursive: true, excludes: [] }]); + listener.dispose(); + close(connection); + await new Promise(resolve => setTimeout(resolve, 1_500)); + assert.equal(server.exitCode, null, output); + assert.equal(await version(target), COMMIT); + close(await handshake(target)); + }); + + test('with auto-shutdown, the server exits once the last extension host leaves', async () => { + const idle = await startServer(['--enable-remote-auto-shutdown', '--reconnection-grace-time', '600', '--remote-auto-shutdown-without-delay']); + const connection = await handshake(idle.target); + await new Promise(resolve => setTimeout(resolve, 500)); + assert.equal(idle.child.exitCode, null, 'it runs while a client is connected'); + close(connection); + const exited = await new Promise(resolve => { + const timer = setTimeout(() => resolve(false), 10_000); + idle.child.once('exit', () => { + clearTimeout(timer); + resolve(true); + }); + }); + assert.ok(exited, output); + }); + + test('refuses a wrong connection token', async () => { + await assert.rejects(connectRemoteAgentManagement(connectionOptions(target, { token: 'wrong' }), AUTHORITY, 'renderer'), /Unauthorized client refused: auth mismatch/); + }); + + test('refuses a client built from another commit', async () => { + await assert.rejects(connectRemoteAgentManagement(connectionOptions(target, { commit: 'f'.repeat(40) }), AUTHORITY, 'renderer'), /Client refused: version mismatch/); + }); +} diff --git a/apps/review-desktop/code-oss/src/vs/server/node/server.main.ts b/apps/review-desktop/code-oss/src/vs/server/node/server.main.ts new file mode 100644 index 000000000..d74eb5d41 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/server/node/server.main.ts @@ -0,0 +1,72 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as os from 'os'; +import * as fs from 'fs'; +import * as net from 'net'; +import { FileAccess } from '../../base/common/network.js'; +import { createServer as doCreateServer, IServerAPI } from './remoteExtensionHostAgentServer.js'; +import { parseArgs, ErrorReporter } from '../../platform/environment/node/argv.js'; +import { join, dirname } from '../../base/common/path.js'; +import { performance } from 'perf_hooks'; +import { serverOptions } from './serverEnvironmentService.js'; +import product from '../../platform/product/common/product.js'; +import * as perf from '../../base/common/performance.js'; + +perf.mark('code/server/codeLoaded'); +(global as unknown as { vscodeServerCodeLoadedTime?: number }).vscodeServerCodeLoadedTime = performance.now(); + +const errorReporter: ErrorReporter = { + onMultipleValues: (id: string, usedValue: string) => { + console.error(`Option '${id}' can only be defined once. Using value ${usedValue}.`); + }, + onEmptyValue: (id) => { + console.error(`Ignoring option '${id}': Value must not be empty.`); + }, + onUnknownOption: (id: string) => { + console.error(`Ignoring option '${id}': not supported for server.`); + }, + onDeprecatedOption: (deprecatedOption: string, message) => { + console.warn(`Option '${deprecatedOption}' is deprecated: ${message}`); + } +}; + +const args = parseArgs(process.argv.slice(2), serverOptions, errorReporter); + +const REMOTE_DATA_FOLDER = args['server-data-dir'] || process.env['VSCODE_AGENT_FOLDER'] || join(os.homedir(), product.serverDataFolderName || '.vscode-remote'); +const USER_DATA_PATH = join(REMOTE_DATA_FOLDER, 'data'); +const APP_SETTINGS_HOME = join(USER_DATA_PATH, 'User'); +const GLOBAL_STORAGE_HOME = join(APP_SETTINGS_HOME, 'globalStorage'); +const LOCAL_HISTORY_HOME = join(APP_SETTINGS_HOME, 'History'); +const MACHINE_SETTINGS_HOME = join(USER_DATA_PATH, 'Machine'); +args['user-data-dir'] = USER_DATA_PATH; +const APP_ROOT = dirname(FileAccess.asFileUri('').fsPath); +const BUILTIN_EXTENSIONS_FOLDER_PATH = join(APP_ROOT, 'extensions'); +args['builtin-extensions-dir'] = BUILTIN_EXTENSIONS_FOLDER_PATH; +args['extensions-dir'] = args['extensions-dir'] || join(REMOTE_DATA_FOLDER, 'extensions'); + +[REMOTE_DATA_FOLDER, args['extensions-dir'], USER_DATA_PATH, APP_SETTINGS_HOME, MACHINE_SETTINGS_HOME, GLOBAL_STORAGE_HOME, LOCAL_HISTORY_HOME].forEach(f => { + try { + if (!fs.existsSync(f)) { + fs.mkdirSync(f, { mode: 0o700, recursive: true }); + } + } catch (err) { console.error(err); } +}); + +/** + * invoked by server-main.js + */ +export function spawnCli() { + // Whiteboard: the extension management CLI is cut. + console.error('This server does not include the extension management CLI.'); + process.exit(1); +} + +/** + * invoked by server-main.js + */ +export function createServer(address: string | net.AddressInfo | null): Promise { + return doCreateServer(address, args, REMOTE_DATA_FOLDER); +} diff --git a/apps/review-desktop/code-oss/src/vs/server/node/serverConnectionToken.ts b/apps/review-desktop/code-oss/src/vs/server/node/serverConnectionToken.ts new file mode 100644 index 000000000..978d1b5eb --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/server/node/serverConnectionToken.ts @@ -0,0 +1,132 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as cookie from 'cookie'; +import * as fs from 'fs'; +import type * as http from 'http'; +import * as url from 'url'; +import * as path from '../../base/common/path.js'; +import { generateUuid } from '../../base/common/uuid.js'; +import { connectionTokenCookieName, connectionTokenQueryName } from '../../base/common/network.js'; +import { ServerParsedArgs } from './serverEnvironmentService.js'; +import { Promises } from '../../base/node/pfs.js'; + +const connectionTokenRegex = /^[0-9A-Za-z_-]+$/; + +export const enum ServerConnectionTokenType { + None, + Optional,// TODO: Remove this soon + Mandatory +} + +export class NoneServerConnectionToken { + public readonly type = ServerConnectionTokenType.None; + + public validate(connectionToken: unknown): boolean { + return true; + } +} + +export class MandatoryServerConnectionToken { + public readonly type = ServerConnectionTokenType.Mandatory; + + constructor(public readonly value: string) { + } + + public validate(connectionToken: unknown): boolean { + return (connectionToken === this.value); + } +} + +export type ServerConnectionToken = NoneServerConnectionToken | MandatoryServerConnectionToken; + +export class ServerConnectionTokenParseError { + constructor( + public readonly message: string + ) { } +} + +export async function parseServerConnectionToken(args: ServerParsedArgs, defaultValue: () => Promise): Promise { + const withoutConnectionToken = args['without-connection-token']; + const connectionToken = args['connection-token']; + const connectionTokenFile = args['connection-token-file']; + + if (withoutConnectionToken) { + if (typeof connectionToken !== 'undefined' || typeof connectionTokenFile !== 'undefined') { + return new ServerConnectionTokenParseError(`Please do not use the argument '--connection-token' or '--connection-token-file' at the same time as '--without-connection-token'.`); + } + return new NoneServerConnectionToken(); + } + + if (typeof connectionTokenFile !== 'undefined') { + if (typeof connectionToken !== 'undefined') { + return new ServerConnectionTokenParseError(`Please do not use the argument '--connection-token' at the same time as '--connection-token-file'.`); + } + + let rawConnectionToken: string; + try { + rawConnectionToken = fs.readFileSync(connectionTokenFile).toString().replace(/\r?\n$/, ''); + } catch (e) { + return new ServerConnectionTokenParseError(`Unable to read the connection token file at '${connectionTokenFile}'.`); + } + + if (!connectionTokenRegex.test(rawConnectionToken)) { + return new ServerConnectionTokenParseError(`The connection token defined in '${connectionTokenFile} does not adhere to the characters 0-9, a-z, A-Z, _, or -.`); + } + + return new MandatoryServerConnectionToken(rawConnectionToken); + } + + if (typeof connectionToken !== 'undefined') { + if (!connectionTokenRegex.test(connectionToken)) { + return new ServerConnectionTokenParseError(`The connection token '${connectionToken} does not adhere to the characters 0-9, a-z, A-Z or -.`); + } + + return new MandatoryServerConnectionToken(connectionToken); + } + + return new MandatoryServerConnectionToken(await defaultValue()); +} + +export async function determineServerConnectionToken(args: ServerParsedArgs): Promise { + const readOrGenerateConnectionToken = async () => { + if (!args['user-data-dir']) { + // No place to store it! + return generateUuid(); + } + const storageLocation = path.join(args['user-data-dir'], 'token'); + + // First try to find a connection token + try { + const fileContents = await fs.promises.readFile(storageLocation); + const connectionToken = fileContents.toString().replace(/\r?\n$/, ''); + if (connectionTokenRegex.test(connectionToken)) { + return connectionToken; + } + } catch (err) { } + + // No connection token found, generate one + const connectionToken = generateUuid(); + + try { + // Try to store it + await Promises.writeFile(storageLocation, connectionToken, { mode: 0o600 }); + } catch (err) { } + + return connectionToken; + }; + return parseServerConnectionToken(args, readOrGenerateConnectionToken); +} + +export function requestHasValidConnectionToken(connectionToken: ServerConnectionToken, req: http.IncomingMessage, parsedUrl: url.UrlWithParsedQuery) { + // First check if there is a valid query parameter + if (connectionToken.validate(parsedUrl.query[connectionTokenQueryName])) { + return true; + } + + // Otherwise, check if there is a valid cookie + const cookies = cookie.parse(req.headers.cookie || ''); + return connectionToken.validate(cookies[connectionTokenCookieName]); +} diff --git a/apps/review-desktop/code-oss/src/vs/server/node/serverEnvironmentService.ts b/apps/review-desktop/code-oss/src/vs/server/node/serverEnvironmentService.ts new file mode 100644 index 000000000..4407fcb60 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/server/node/serverEnvironmentService.ts @@ -0,0 +1,284 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as nls from '../../nls.js'; + +import { NativeEnvironmentService } from '../../platform/environment/node/environmentService.js'; +import { OPTIONS, OptionDescriptions } from '../../platform/environment/node/argv.js'; +import { refineServiceDecorator } from '../../platform/instantiation/common/instantiation.js'; +import { IEnvironmentService, INativeEnvironmentService } from '../../platform/environment/common/environment.js'; +import { memoize } from '../../base/common/decorators.js'; +import { URI } from '../../base/common/uri.js'; +import { joinPath } from '../../base/common/resources.js'; +import { join } from '../../base/common/path.js'; +import { ProtocolConstants } from '../../base/parts/ipc/common/ipc.net.js'; + +export const serverOptions: OptionDescriptions> = { + + /* ----- server setup ----- */ + + 'host': { type: 'string', cat: 'o', args: 'ip-address', description: nls.localize('host', "The host name or IP address the server should listen to. If not set, defaults to 'localhost'.") }, + 'port': { type: 'string', cat: 'o', args: 'port | port range', description: nls.localize('port', "The port the server should listen to. If 0 is passed a random free port is picked. If a range in the format num-num is passed, a free port from the range (end inclusive) is selected.") }, + 'socket-path': { type: 'string', cat: 'o', args: 'path', description: nls.localize('socket-path', "The path to a socket file for the server to listen to.") }, + 'server-base-path': { type: 'string', cat: 'o', args: 'path', description: nls.localize('server-base-path', "The path under which the web UI and the code server is provided. Defaults to '/'.`") }, + 'connection-token': { type: 'string', cat: 'o', args: 'token', deprecates: ['connectionToken'], description: nls.localize('connection-token', "A secret that must be included with all requests.") }, + 'connection-token-file': { type: 'string', cat: 'o', args: 'path', deprecates: ['connection-secret', 'connectionTokenFile'], description: nls.localize('connection-token-file', "Path to a file that contains the connection token.") }, + 'without-connection-token': { type: 'boolean', cat: 'o', description: nls.localize('without-connection-token', "Run without a connection token. Only use this if the connection is secured by other means.") }, + 'disable-websocket-compression': { type: 'boolean' }, + 'print-startup-performance': { type: 'boolean' }, + 'print-ip-address': { type: 'boolean' }, + 'accept-server-license-terms': { type: 'boolean', cat: 'o', description: nls.localize('acceptLicenseTerms', "If set, the user accepts the server license terms and the server will be started without a user prompt.") }, + 'server-data-dir': { type: 'string', cat: 'o', description: nls.localize('serverDataDir', "Specifies the directory that server data is kept in.") }, + 'telemetry-level': { type: 'string', cat: 'o', args: 'level', description: nls.localize('telemetry-level', "Sets the initial telemetry level. Valid levels are: 'off', 'crash', 'error' and 'all'. If not specified, the server will send telemetry until a client connects, it will then use the clients telemetry setting. Setting this to 'off' is equivalent to --disable-telemetry") }, + + /* ----- vs code options --- -- */ + + 'user-data-dir': OPTIONS['user-data-dir'], + 'enable-smoke-test-driver': OPTIONS['enable-smoke-test-driver'], + 'disable-telemetry': OPTIONS['disable-telemetry'], + 'disable-experiments': OPTIONS['disable-experiments'], + 'disable-workspace-trust': OPTIONS['disable-workspace-trust'], + 'file-watcher-polling': { type: 'string', deprecates: ['fileWatcherPolling'] }, + 'log': OPTIONS['log'], + 'logsPath': OPTIONS['logsPath'], + 'force-disable-user-env': OPTIONS['force-disable-user-env'], + 'enable-proposed-api': OPTIONS['enable-proposed-api'], + + /* ----- vs code web options ----- */ + + 'folder': { type: 'string', deprecationMessage: 'No longer supported. Folder needs to be provided in the browser URL or with `default-folder`.' }, + 'workspace': { type: 'string', deprecationMessage: 'No longer supported. Workspace needs to be provided in the browser URL or with `default-workspace`.' }, + + 'default-folder': { type: 'string', description: nls.localize('default-folder', 'The workspace folder to open when no input is specified in the browser URL. A relative or absolute path resolved against the current working directory.') }, + 'default-workspace': { type: 'string', description: nls.localize('default-workspace', 'The workspace to open when no input is specified in the browser URL. A relative or absolute path resolved against the current working directory.') }, + + 'enable-sync': { type: 'boolean' }, + 'github-auth': { type: 'string' }, + 'use-test-resolver': { type: 'boolean' }, + + /* ----- extension management ----- */ + + 'extensions-dir': OPTIONS['extensions-dir'], + 'extensions-download-dir': OPTIONS['extensions-download-dir'], + 'builtin-extensions-dir': OPTIONS['builtin-extensions-dir'], + 'install-extension': OPTIONS['install-extension'], + 'install-builtin-extension': OPTIONS['install-builtin-extension'], + 'update-extensions': OPTIONS['update-extensions'], + 'uninstall-extension': OPTIONS['uninstall-extension'], + 'list-extensions': OPTIONS['list-extensions'], + 'locate-extension': OPTIONS['locate-extension'], + + 'show-versions': OPTIONS['show-versions'], + 'category': OPTIONS['category'], + 'force': OPTIONS['force'], + 'do-not-sync': OPTIONS['do-not-sync'], + 'do-not-include-pack-dependencies': OPTIONS['do-not-include-pack-dependencies'], + 'pre-release': OPTIONS['pre-release'], + 'start-server': { type: 'boolean', cat: 'e', description: nls.localize('start-server', "Start the server when installing or uninstalling extensions. To be used in combination with 'install-extension', 'install-builtin-extension' and 'uninstall-extension'.") }, + + + /* ----- remote development options ----- */ + + 'enable-remote-auto-shutdown': { type: 'boolean' }, + 'remote-auto-shutdown-without-delay': { type: 'boolean' }, + 'inspect-ptyhost': { type: 'string', allowEmptyValue: true }, + + 'agent-host-port': { type: 'string', cat: 'o', args: 'port', description: nls.localize('agent-host-port', "The port the agent host WebSocket server should listen on.") }, + 'agent-host-path': { type: 'string', cat: 'o', args: 'path', description: nls.localize('agent-host-path', "The path to a socket file for the agent host WebSocket server to listen on.") }, + 'agent-host-bridge-port': { type: 'string', cat: 'o', args: 'port', description: nls.localize('agent-host-bridge-port', "Bridge renderer agent-host traffic to an already-running agent host listening on this port. Does not spawn an agent host.") }, + 'agent-host-bridge-path': { type: 'string', cat: 'o', args: 'path', description: nls.localize('agent-host-bridge-path', "Bridge renderer agent-host traffic to an already-running agent host listening on this socket path. Does not spawn an agent host.") }, + 'agent-host-bridge-host': { type: 'string', cat: 'o', args: 'host', description: nls.localize('agent-host-bridge-host', "Host the externally-running agent host is reachable at when used with --agent-host-bridge-port. Defaults to localhost.") }, + 'agent-host-bridge-connection-token': { type: 'string', cat: 'o', args: 'token', description: nls.localize('agent-host-bridge-connection-token', "Connection token required by the externally-running agent host when used with --agent-host-bridge-port.") }, + + 'use-host-proxy': { type: 'boolean' }, + 'without-browser-env-var': { type: 'boolean' }, + 'reconnection-grace-time': { type: 'string', cat: 'o', args: 'seconds', description: nls.localize('reconnection-grace-time', "Override the reconnection grace time window in seconds. Defaults to 10800 (3 hours).") }, + + /* ----- server cli ----- */ + + 'help': OPTIONS['help'], + 'version': OPTIONS['version'], + 'locate-shell-integration-path': OPTIONS['locate-shell-integration-path'], + + 'compatibility': { type: 'string' }, + + _: OPTIONS['_'] +}; + +export interface ServerParsedArgs { + + /* ----- server setup ----- */ + + host?: string; + /** + * A port or a port range + */ + port?: string; + 'socket-path'?: string; + + /** + * The path under which the web UI and the code server is provided. + * By defaults it is '/'.` + */ + 'server-base-path'?: string; + + /** + * A secret token that must be provided by the web client with all requests. + * Use only `[0-9A-Za-z\-]`. + * + * By default, a UUID will be generated every time the server starts up. + * + * If the server is running on a multi-user system, then consider + * using `--connection-token-file` which has the advantage that the token cannot + * be seen by other users using `ps` or similar commands. + */ + 'connection-token'?: string; + /** + * A path to a filename which will be read on startup. + * Consider placing this file in a folder readable only by the same user (a `chmod 0700` directory). + * + * The contents of the file will be used as the connection token. Use only `[0-9A-Z\-]` as contents in the file. + * The file can optionally end in a `\n` which will be ignored. + * + * This secret must be communicated to any vscode instance via the resolver or embedder API. + */ + 'connection-token-file'?: string; + + /** + * Run the server without a connection token + */ + 'without-connection-token'?: boolean; + + 'disable-websocket-compression'?: boolean; + + 'print-startup-performance'?: boolean; + 'print-ip-address'?: boolean; + + 'accept-server-license-terms': boolean; + + 'server-data-dir'?: string; + + 'telemetry-level'?: string; + + 'disable-workspace-trust'?: boolean; + + /* ----- vs code options ----- */ + + 'user-data-dir'?: string; + + 'enable-smoke-test-driver'?: boolean; + + 'disable-telemetry'?: boolean; + 'disable-experiments'?: boolean; + 'file-watcher-polling'?: string; + + 'log'?: string[]; + 'logsPath'?: string; + + 'force-disable-user-env'?: boolean; + 'enable-proposed-api'?: string[]; + + /* ----- vs code web options ----- */ + + 'default-workspace'?: string; + 'default-folder'?: string; + + /** @deprecated use default-workspace instead */ + workspace: string; + /** @deprecated use default-folder instead */ + folder: string; + + + 'enable-sync'?: boolean; + 'github-auth'?: string; + 'use-test-resolver'?: boolean; + + /* ----- extension management ----- */ + + 'extensions-dir'?: string; + 'extensions-download-dir'?: string; + 'builtin-extensions-dir'?: string; + 'install-extension'?: string[]; + 'install-builtin-extension'?: string[]; + 'update-extensions'?: boolean; + 'uninstall-extension'?: string[]; + 'list-extensions'?: boolean; + 'locate-extension'?: string[]; + 'show-versions'?: boolean; + 'category'?: string; + force?: boolean; // used by install-extension + 'do-not-sync'?: boolean; // used by install-extension + 'pre-release'?: boolean; // used by install-extension + 'do-not-include-pack-dependencies'?: boolean; // used by install-extension + + + 'start-server'?: boolean; + + /* ----- remote development options ----- */ + + 'enable-remote-auto-shutdown'?: boolean; + 'remote-auto-shutdown-without-delay'?: boolean; + 'inspect-ptyhost'?: string; + + 'agent-host-port'?: string; + 'agent-host-path'?: string; + 'agent-host-bridge-port'?: string; + 'agent-host-bridge-path'?: string; + 'agent-host-bridge-host'?: string; + 'agent-host-bridge-connection-token'?: string; + + 'use-host-proxy'?: boolean; + 'without-browser-env-var'?: boolean; + 'reconnection-grace-time'?: string; + + /* ----- server cli ----- */ + help: boolean; + version: boolean; + 'locate-shell-integration-path'?: string; + + compatibility: string; + + _: string[]; +} + +export const IServerEnvironmentService = refineServiceDecorator(IEnvironmentService); + +export interface IServerEnvironmentService extends INativeEnvironmentService { + readonly machineSettingsResource: URI; + readonly mcpResource: URI; + readonly args: ServerParsedArgs; + readonly reconnectionGraceTime: number; +} + +export class ServerEnvironmentService extends NativeEnvironmentService implements IServerEnvironmentService { + @memoize + override get userRoamingDataHome(): URI { return this.appSettingsHome; } + @memoize + get machineSettingsResource(): URI { return joinPath(URI.file(join(this.userDataPath, 'Machine')), 'settings.json'); } + @memoize + get mcpResource(): URI { return joinPath(URI.file(join(this.userDataPath, 'User')), 'mcp.json'); } + override get args(): ServerParsedArgs { return super.args as ServerParsedArgs; } + @memoize + get reconnectionGraceTime(): number { return parseGraceTime(this.args['reconnection-grace-time'], ProtocolConstants.ReconnectionGraceTime); } +} + +function parseGraceTime(rawValue: string | undefined, fallback: number): number { + if (typeof rawValue !== 'string' || rawValue.trim().length === 0) { + console.log(`[reconnection-grace-time] No CLI argument provided, using default: ${fallback}ms (${Math.floor(fallback / 1000)}s)`); + return fallback; + } + const parsedSeconds = Number(rawValue); + if (!isFinite(parsedSeconds) || parsedSeconds < 0) { + console.log(`[reconnection-grace-time] Invalid value '${rawValue}', using default: ${fallback}ms (${Math.floor(fallback / 1000)}s)`); + return fallback; + } + const millis = Math.floor(parsedSeconds * 1000); + if (!isFinite(millis) || millis > Number.MAX_SAFE_INTEGER) { + console.log(`[reconnection-grace-time] Value too large '${rawValue}', using default: ${fallback}ms (${Math.floor(fallback / 1000)}s)`); + return fallback; + } + console.log(`[reconnection-grace-time] Parsed CLI argument: ${parsedSeconds}s -> ${millis}ms`); + return millis; +} diff --git a/apps/review-desktop/code-oss/src/vs/server/node/serverLifetimeService.ts b/apps/review-desktop/code-oss/src/vs/server/node/serverLifetimeService.ts new file mode 100644 index 000000000..397c3eed3 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/server/node/serverLifetimeService.ts @@ -0,0 +1,145 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Disposable, IDisposable, toDisposable } from '../../base/common/lifecycle.js'; +import { createDecorator } from '../../platform/instantiation/common/instantiation.js'; +import { ILogService } from '../../platform/log/common/log.js'; + +export const IServerLifetimeService = createDecorator('serverLifetimeService'); + +export const SHUTDOWN_TIMEOUT = 5 * 60 * 1000; + +/** Options controlling the auto-shutdown behaviour. */ +export interface IServerLifetimeOptions { + /** When `false` (default), the server never auto-shuts down. */ + readonly enableAutoShutdown?: boolean; + /** When `true`, skip the 5-minute grace period on non-initial shutdowns. */ + readonly shutdownWithoutDelay?: boolean; +} + +/** + * Tracks active consumers (extension hosts, agent sessions, etc.) that keep + * the server alive. When auto-shutdown is enabled, the service manages a + * shutdown timer and fires {@link onDidShutdownRequested} when it is time for + * the process to exit. + */ +export interface IServerLifetimeService { + readonly _serviceBrand: undefined; + + /** + * Marks a consumer as active. The server will not auto-shutdown until the + * returned {@link IDisposable} is disposed. + */ + active(consumer: string): IDisposable; + + /** + * Delays the auto-shutdown timer. If the server is currently in a shutdown + * timeout (all consumers inactive), the timer is reset. + */ + delay(): void; + + /** Whether any consumer is currently active. */ + readonly hasActiveConsumers: boolean; +} + +export class ServerLifetimeService extends Disposable implements IServerLifetimeService { + declare readonly _serviceBrand: undefined; + + private readonly _consumers = new Map(); + private _totalCount = 0; + private _shutdownTimer: ReturnType | undefined; + + constructor( + private readonly _options: IServerLifetimeOptions, + @ILogService private readonly _logService: ILogService, + ) { + super(); + + if (this._options.enableAutoShutdown) { + // Start initial shutdown timer (no clients connected yet) + this._scheduleShutdown(true); + } + } + + get hasActiveConsumers(): boolean { + return this._totalCount > 0; + } + + active(consumer: string): IDisposable { + const wasEmpty = this._totalCount === 0; + const current = this._consumers.get(consumer) ?? 0; + this._consumers.set(consumer, current + 1); + this._totalCount++; + + this._logService.debug(`ServerLifetime: consumer '${consumer}' active (total: ${this._totalCount})`); + + if (wasEmpty) { + this._cancelShutdown(); + } + + let disposed = false; + return toDisposable(() => { + if (disposed) { + return; + } + disposed = true; + + const count = this._consumers.get(consumer); + if (count !== undefined) { + if (count <= 1) { + this._consumers.delete(consumer); + } else { + this._consumers.set(consumer, count - 1); + } + } + this._totalCount--; + + this._logService.debug(`ServerLifetime: consumer '${consumer}' inactive (total: ${this._totalCount})`); + + if (this._totalCount === 0 && this._options.enableAutoShutdown) { + this._scheduleShutdown(false); + } + }); + } + + delay(): void { + if (this._shutdownTimer) { + this._logService.debug('ServerLifetime: delay requested, resetting shutdown timer'); + this._cancelShutdown(); + this._scheduleShutdown(false); + } + } + + private _scheduleShutdown(initial: boolean): void { + if (this._options.shutdownWithoutDelay && !initial) { + this._tryShutdown(); + } else { + this._logService.debug('ServerLifetime: scheduling shutdown timer'); + this._shutdownTimer = setTimeout(() => { + this._shutdownTimer = undefined; + this._tryShutdown(); + }, SHUTDOWN_TIMEOUT); + } + } + + private _tryShutdown(): void { + if (this._totalCount > 0) { + this._logService.debug('ServerLifetime: consumer became active, aborting shutdown'); + return; + } + console.log('All consumers inactive, shutting down'); + this._logService.info('ServerLifetime: all consumers inactive, shutting down'); + this.dispose(); + process.exit(0); + } + + private _cancelShutdown(): void { + if (this._shutdownTimer) { + this._logService.debug('ServerLifetime: cancelling shutdown timer'); + clearTimeout(this._shutdownTimer); + this._shutdownTimer = undefined; + } + } +} diff --git a/apps/review-desktop/code-oss/src/vs/server/node/serverServices.ts b/apps/review-desktop/code-oss/src/vs/server/node/serverServices.ts new file mode 100644 index 000000000..08489b711 --- /dev/null +++ b/apps/review-desktop/code-oss/src/vs/server/node/serverServices.ts @@ -0,0 +1,321 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { isCancellationError, isSigPipeError, onUnexpectedError } from '../../base/common/errors.js'; +import { Emitter, Event } from '../../base/common/event.js'; +import { DisposableStore } from '../../base/common/lifecycle.js'; +import { Schemas } from '../../base/common/network.js'; +import * as path from '../../base/common/path.js'; +import { IURITransformer } from '../../base/common/uriIpc.js'; +import { Promises } from '../../base/node/pfs.js'; +import { ClientConnectionEvent, IMessagePassingProtocol, IPCServer, IServerChannel } from '../../base/parts/ipc/common/ipc.js'; +import { IConfigurationService } from '../../platform/configuration/common/configuration.js'; +import { ConfigurationService } from '../../platform/configuration/common/configurationService.js'; +import { ExtensionHostDebugBroadcastChannel } from '../../platform/debug/common/extensionHostDebugIpc.js'; +import { IEnvironmentService, INativeEnvironmentService } from '../../platform/environment/common/environment.js'; +import { IFileService } from '../../platform/files/common/files.js'; +import { FileService } from '../../platform/files/common/fileService.js'; +import { DiskFileSystemProvider } from '../../platform/files/node/diskFileSystemProvider.js'; +import { SyncDescriptor } from '../../platform/instantiation/common/descriptors.js'; +import { IInstantiationService } from '../../platform/instantiation/common/instantiation.js'; +import { InstantiationService } from '../../platform/instantiation/common/instantiationService.js'; +import { ServiceCollection } from '../../platform/instantiation/common/serviceCollection.js'; +import { AbstractLogger, DEFAULT_LOG_LEVEL, getLogLevel, ILoggerService, ILogService, log, LogLevel, LogLevelToString } from '../../platform/log/common/log.js'; +import product from '../../platform/product/common/product.js'; +import { IProductService } from '../../platform/product/common/productService.js'; +import { RemoteAgentConnectionContext } from '../../platform/remote/common/remoteAgentEnvironment.js'; +import { ITelemetryService } from '../../platform/telemetry/common/telemetry.js'; +import { NullTelemetryService } from '../../platform/telemetry/common/telemetryUtils.js'; +import { IUriIdentityService } from '../../platform/uriIdentity/common/uriIdentity.js'; +import { UriIdentityService } from '../../platform/uriIdentity/common/uriIdentityService.js'; +import { RemoteAgentEnvironmentChannel } from './remoteAgentEnvironmentImpl.js'; +import { RemoteAgentFileSystemProviderChannel } from './remoteFileSystemProviderServer.js'; +import { createURITransformer } from '../../base/common/uriTransformer.js'; +import { ServerConnectionToken } from './serverConnectionToken.js'; +import { ServerEnvironmentService, ServerParsedArgs } from './serverEnvironmentService.js'; +import { REMOTE_FILE_SYSTEM_CHANNEL_NAME } from '../../workbench/services/remote/common/remoteFileSystemProviderClient.js'; +import { ExtensionHostStatusService, IExtensionHostStatusService } from './extensionHostStatusService.js'; +import { IExtensionsScannerService } from '../../platform/extensionManagement/common/extensionsScannerService.js'; +import { ExtensionsScannerService } from './extensionsScannerService.js'; +import { IExtensionsProfileScannerService } from '../../platform/extensionManagement/common/extensionsProfileScannerService.js'; +import { IUserDataProfilesService } from '../../platform/userDataProfile/common/userDataProfile.js'; +import { NullPolicyService } from '../../platform/policy/common/policy.js'; +import { LoggerService } from '../../platform/log/node/loggerService.js'; +import { ServerUserDataProfilesService } from '../../platform/userDataProfile/node/userDataProfile.js'; +import { ExtensionsProfileScannerService } from '../../platform/extensionManagement/node/extensionsProfileScannerService.js'; +import { LogService } from '../../platform/log/common/logService.js'; +import { LoggerChannel } from '../../platform/log/common/logIpc.js'; +import { localize } from '../../nls.js'; +import { RemoteExtensionsScannerChannel, RemoteExtensionsScannerService } from './remoteExtensionsScanner.js'; +import { RemoteExtensionsScannerChannelName } from '../../platform/remote/common/remoteExtensionsScanner.js'; +import { RemoteUserDataProfilesServiceChannel } from '../../platform/userDataProfile/common/userDataProfileIpc.js'; +import { IServerLifetimeService, ServerLifetimeService } from './serverLifetimeService.js'; +import { CSSDevelopmentService, ICSSDevelopmentService } from '../../platform/cssDev/node/cssDevService.js'; + +export async function setupServerServices(connectionToken: ServerConnectionToken, args: ServerParsedArgs, REMOTE_DATA_FOLDER: string, disposables: DisposableStore) { + const services = new ServiceCollection(); + const socketServer = new SocketServer(); + + // Whiteboard: upstream installs these through ErrorTelemetry, which is cut with telemetry. + installErrorListeners(); + + const productService: IProductService = { _serviceBrand: undefined, ...product }; + services.set(IProductService, productService); + + const environmentService = new ServerEnvironmentService(args, productService); + services.set(IEnvironmentService, environmentService); + services.set(INativeEnvironmentService, environmentService); + + const loggerService = new LoggerService(getLogLevel(environmentService), environmentService.logsHome); + services.set(ILoggerService, loggerService); + socketServer.registerChannel('logger', new LoggerChannel(loggerService, (ctx: RemoteAgentConnectionContext) => getUriTransformer(ctx.remoteAuthority))); + + const logger = loggerService.createLogger('remoteagent', { name: localize('remoteExtensionLog', "Server") }); + const logService = disposables.add(new LogService(logger, [new ServerLogger(getLogLevel(environmentService))])); + services.set(ILogService, logService); + setTimeout(() => cleanupOlderLogs(environmentService.logsHome.with({ scheme: Schemas.file }).fsPath).then(null, err => logService.error(err)), 10000); + disposables.add(logService.onDidChangeLogLevel(logLevel => log(logService, logLevel, `Log level changed to ${LogLevelToString(logService.getLevel())}`))); + + logService.trace(`Remote configuration data at ${REMOTE_DATA_FOLDER}`); + logService.trace('process arguments:', environmentService.args); + if (Array.isArray(productService.serverGreeting)) { + logService.info(`\n\n${productService.serverGreeting.join('\n')}\n\n`); + } + + // ExtensionHost Debug broadcast service + socketServer.registerChannel(ExtensionHostDebugBroadcastChannel.ChannelName, new ExtensionHostDebugBroadcastChannel()); + + // Files + const fileService = disposables.add(new FileService(logService)); + services.set(IFileService, fileService); + fileService.registerProvider(Schemas.file, disposables.add(new DiskFileSystemProvider(logService))); + + // URI Identity + const uriIdentityService = new UriIdentityService(fileService); + services.set(IUriIdentityService, uriIdentityService); + + // Configuration + const configurationService = new ConfigurationService(environmentService.machineSettingsResource, fileService, new NullPolicyService(), logService); + services.set(IConfigurationService, configurationService); + + // User Data Profiles + const userDataProfilesService = new ServerUserDataProfilesService(uriIdentityService, environmentService, fileService, logService); + services.set(IUserDataProfilesService, userDataProfilesService); + socketServer.registerChannel('userDataProfiles', new RemoteUserDataProfilesServiceChannel(userDataProfilesService, (ctx: RemoteAgentConnectionContext) => getUriTransformer(ctx.remoteAuthority))); + + // Dev Only: CSS service (for ESM) + services.set(ICSSDevelopmentService, new SyncDescriptor(CSSDevelopmentService, undefined, true)); + + // Initialize + // Whiteboard: the machine ids are cut; only telemetry used them. + await Promise.all([ + configurationService.initialize(), + userDataProfilesService.init() + ]); + + const extensionHostStatusService = new ExtensionHostStatusService(); + services.set(IExtensionHostStatusService, extensionHostStatusService); + + // Whiteboard: requests, telemetry, the gallery, extension installs and MCP are cut. + services.set(ITelemetryService, NullTelemetryService); + services.set(IExtensionsProfileScannerService, new SyncDescriptor(ExtensionsProfileScannerService)); + services.set(IExtensionsScannerService, new SyncDescriptor(ExtensionsScannerService)); + + const instantiationService: IInstantiationService = new InstantiationService(services); + // Whiteboard: language packs and the pty host (terminals) are cut. + + const serverLifetimeService = instantiationService.createInstance(ServerLifetimeService, { + enableAutoShutdown: !!args['enable-remote-auto-shutdown'], + shutdownWithoutDelay: !!args['remote-auto-shutdown-without-delay'], + }); + services.set(IServerLifetimeService, serverLifetimeService); + + // Whiteboard: the agent host and MCP are cut. + + instantiationService.invokeFunction(accessor => { + const extensionsScannerService = accessor.get(IExtensionsScannerService); + const remoteExtensionEnvironmentChannel = new RemoteAgentEnvironmentChannel(connectionToken, environmentService, userDataProfilesService, extensionHostStatusService, logService); + socketServer.registerChannel('remoteextensionsenvironment', remoteExtensionEnvironmentChannel); + + // Whiteboard: the client always opens this channel; telemetry is cut. + socketServer.registerChannel('telemetry', new NullTelemetryChannel()); + + const remoteExtensionsScanner = new RemoteExtensionsScannerService(userDataProfilesService, extensionsScannerService, logService); + socketServer.registerChannel(RemoteExtensionsScannerChannelName, new RemoteExtensionsScannerChannel(remoteExtensionsScanner, (ctx: RemoteAgentConnectionContext) => getUriTransformer(ctx.remoteAuthority))); + + const remoteFileSystemChannel = disposables.add(new RemoteAgentFileSystemProviderChannel(logService, environmentService, configurationService)); + socketServer.registerChannel(REMOTE_FILE_SYSTEM_CHANNEL_NAME, remoteFileSystemChannel); + }); + + return { socketServer, instantiationService }; +} + +// Whiteboard: the listeners of upstream's node `ErrorTelemetry.installErrorListeners`. +function installErrorListeners(): void { + // Print a console message when rejection isn't handled within N seconds. For details: + // see https://nodejs.org/api/process.html#process_event_unhandledrejection + // and https://nodejs.org/api/process.html#process_event_rejectionhandled + const unhandledPromises: Promise[] = []; + process.on('unhandledRejection', (reason: unknown, promise: Promise) => { + unhandledPromises.push(promise); + setTimeout(() => { + const idx = unhandledPromises.indexOf(promise); + if (idx >= 0) { + promise.catch(e => { + unhandledPromises.splice(idx, 1); + if (!isCancellationError(e)) { + console.warn(`rejected promise not handled within 1 second: ${e}`); + if (e.stack) { + console.warn(`stack trace: ${e.stack}`); + } + if (reason) { + onUnexpectedError(reason); + } + } + }); + } + }, 1000); + }); + + process.on('rejectionHandled', (promise: Promise) => { + const idx = unhandledPromises.indexOf(promise); + if (idx >= 0) { + unhandledPromises.splice(idx, 1); + } + }); + + // Print a console message when an exception isn't handled. + process.on('uncaughtException', (err: Error | NodeJS.ErrnoException) => { + if (isSigPipeError(err)) { + return; + } + + onUnexpectedError(err); + }); +} + +class NullTelemetryChannel implements IServerChannel { + async call(): Promise { return undefined as T; } + listen(): Event { return Event.None; } +} + +const _uriTransformerCache: { [remoteAuthority: string]: IURITransformer } = Object.create(null); + +function getUriTransformer(remoteAuthority: string): IURITransformer { + if (!_uriTransformerCache[remoteAuthority]) { + _uriTransformerCache[remoteAuthority] = createURITransformer(remoteAuthority); + } + return _uriTransformerCache[remoteAuthority]; +} + +export class SocketServer extends IPCServer { + + private _onDidConnectEmitter: Emitter; + + constructor() { + const emitter = new Emitter(); + super(emitter.event); + this._onDidConnectEmitter = emitter; + } + + public acceptConnection(protocol: IMessagePassingProtocol, onDidClientDisconnect: Event): void { + this._onDidConnectEmitter.fire({ protocol, onDidClientDisconnect }); + } +} + +class ServerLogger extends AbstractLogger { + private useColors: boolean; + + constructor(logLevel: LogLevel = DEFAULT_LOG_LEVEL) { + super(); + this.setLevel(logLevel); + this.useColors = Boolean(process.stdout.isTTY); + } + + trace(message: string, ...args: unknown[]): void { + if (this.canLog(LogLevel.Trace)) { + if (this.useColors) { + console.log(`\x1b[90m[${now()}]\x1b[0m`, message, ...args); + } else { + console.log(`[${now()}]`, message, ...args); + } + } + } + + debug(message: string, ...args: unknown[]): void { + if (this.canLog(LogLevel.Debug)) { + if (this.useColors) { + console.log(`\x1b[90m[${now()}]\x1b[0m`, message, ...args); + } else { + console.log(`[${now()}]`, message, ...args); + } + } + } + + info(message: string, ...args: unknown[]): void { + if (this.canLog(LogLevel.Info)) { + if (this.useColors) { + console.log(`\x1b[90m[${now()}]\x1b[0m`, message, ...args); + } else { + console.log(`[${now()}]`, message, ...args); + } + } + } + + warn(message: string | Error, ...args: unknown[]): void { + if (this.canLog(LogLevel.Warning)) { + if (this.useColors) { + console.warn(`\x1b[93m[${now()}]\x1b[0m`, message, ...args); + } else { + console.warn(`[${now()}]`, message, ...args); + } + } + } + + error(message: string, ...args: unknown[]): void { + if (this.canLog(LogLevel.Error)) { + if (this.useColors) { + console.error(`\x1b[91m[${now()}]\x1b[0m`, message, ...args); + } else { + console.error(`[${now()}]`, message, ...args); + } + } + } + + flush(): void { + // noop + } +} + +function now(): string { + const date = new Date(); + return `${twodigits(date.getHours())}:${twodigits(date.getMinutes())}:${twodigits(date.getSeconds())}`; +} + +function twodigits(n: number): string { + if (n < 10) { + return `0${n}`; + } + return String(n); +} + +/** + * Cleans up older logs, while keeping the 10 most recent ones. + */ +async function cleanupOlderLogs(logsPath: string): Promise { + const currentLog = path.basename(logsPath); + const logsRoot = path.dirname(logsPath); + + if (!await Promises.exists(logsRoot)) { + return; // Logs root doesn't exist yet, nothing to clean up + } + + const children = await Promises.readdir(logsRoot); + const allSessions = children.filter(name => /^\d{8}T\d{6}$/.test(name)); + const oldSessions = allSessions.sort().filter((d) => d !== currentLog); + const toDelete = oldSessions.slice(0, Math.max(0, oldSessions.length - 9)); + + await Promise.all(toDelete.map(name => Promises.rm(path.join(logsRoot, name)))); +} diff --git a/apps/review-desktop/code-oss/src/vs/workbench/electron-browser/desktop.main.ts b/apps/review-desktop/code-oss/src/vs/workbench/electron-browser/desktop.main.ts index 03ed0459d..7aaaba718 100644 --- a/apps/review-desktop/code-oss/src/vs/workbench/electron-browser/desktop.main.ts +++ b/apps/review-desktop/code-oss/src/vs/workbench/electron-browser/desktop.main.ts @@ -366,7 +366,7 @@ export class DesktopMain extends Disposable { return toWorkspaceIdentifier(this.configuration.backupPath, environmentService.isExtensionDevelopment); } - private async createWorkspaceService( + protected async createWorkspaceService( workspace: IAnyWorkspaceIdentifier, environmentService: INativeWorkbenchEnvironmentService, userDataProfileService: IUserDataProfileService, diff --git a/apps/review-desktop/package.json b/apps/review-desktop/package.json index 37690dc7a..e478b35db 100644 --- a/apps/review-desktop/package.json +++ b/apps/review-desktop/package.json @@ -19,7 +19,7 @@ "test:e2e:lsp": "node scripts/lsp-e2e.mjs", "test:e2e:telemetry": "node scripts/e2e/run.mjs --journey telemetry-contract --runtime \"${REVIEW_E2E_RUNTIME:?stage a runtime, see scripts/e2e/TESTING.md}\"", "pretest": "pnpm run protocol:sync", - "test": "node --test \"scripts/**/*.test.mjs\" && TSX_TSCONFIG_PATH=tsconfig.test.json node --import tsx --test \"code-oss/src/vs/review/**/*.test.ts\"", + "test": "node --test \"scripts/**/*.test.mjs\" && TSX_TSCONFIG_PATH=tsconfig.test.json node --import tsx --test \"code-oss/src/vs/review/**/*.test.ts\" && TSX_TSCONFIG_PATH=tsconfig.test.json node --import tsx --test --test-force-exit \"code-oss/src/vs/server/**/*.test.ts\"", "pretypecheck": "pnpm run protocol:sync", "typecheck": "npm --prefix code-oss run typecheck-client", "app:package:windows": "bash scripts/package-windows.sh" diff --git a/apps/review-desktop/scripts/build-remote-runtime.mjs b/apps/review-desktop/scripts/build-remote-runtime.mjs new file mode 100644 index 000000000..9a1386a3f --- /dev/null +++ b/apps/review-desktop/scripts/build-remote-runtime.mjs @@ -0,0 +1,224 @@ +#!/usr/bin/env node +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import { createRequire, isBuiltin } from "node:module"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; + +const appDirectory = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "..", +); + +const codeOss = path.join(appDirectory, "code-oss"); + +const monorepoRoot = path.resolve(appDirectory, "..", ".."); + +export const DEFAULT_REMOTE_RUNTIME = path.join( + codeOss, + ".build/remote-runtime", +); + +export const REMOTE_RUNTIME_ENTRIES = [ + "server-main", + "bootstrap-fork", + "vs/workbench/api/node/extensionHostProcess", + "vs/platform/files/node/watcher/watcherMain", +]; + +export const OPTIONAL_NATIVE_PACKAGES = [ + "@parcel/watcher", + "@vscode/deviceid", + "@vscode/native-watchdog", + "@vscode/spdlog", + "@vscode/sqlite3", + "@vscode/windows-ca-certs", + "@vscode/windows-process-tree", + "@vscode/windows-registry", + "electron", + "kerberos", + "native-keymap", + "node-pty", + "vsda", +]; + +export const REMOTE_BUILTIN_EXTENSIONS = [ + "typescript-language-features", + "json-language-features", + "css-language-features", + "html-language-features", +]; + +async function buildBuiltinExtensions(out) { + const buildRequire = createRequire(path.join(codeOss, "build/package.json")); + const vsce = buildRequire("@vscode/vsce"); + const extensions = path.join(codeOss, "extensions"); + + for (const name of REMOTE_BUILTIN_EXTENSIONS) { + const source = path.join(extensions, name); + const destination = path.join(out, "extensions", name); + + execFileSync(process.execPath, ["esbuild.mts"], { + cwd: source, + stdio: ["ignore", "ignore", "inherit"], + }); + + const files = await vsce.listFiles({ + cwd: source, + packageManager: vsce.PackageManager.None, + }); + + for (const file of files.filter((f) => !f.endsWith(".map"))) { + fs.cpSync(path.join(source, file), path.join(destination, file)); + } + + const manifestPath = path.join(destination, "package.json"); + const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")); + + delete manifest.scripts; + delete manifest.dependencies; + delete manifest.devDependencies; + manifest.main &&= manifest.main.replace("/out/", "/dist/"); + fs.writeFileSync(manifestPath, JSON.stringify(manifest)); + } + + fs.cpSync( + path.join(extensions, "node_modules/typescript"), + path.join(out, "extensions/node_modules/typescript"), + { recursive: true }, + ); +} + +function desktopCommit() { + const fromEnv = process.env.BUILD_SOURCEVERSION?.trim(); + + if (fromEnv && /^[0-9a-f]{40}$/i.test(fromEnv)) return fromEnv; + + return execFileSync("git", ["-C", monorepoRoot, "rev-parse", "HEAD"], { + encoding: "utf8", + }).trim(); +} + +function packageName(specifier) { + const parts = specifier.split("/"); + + return specifier.startsWith("@") ? parts.slice(0, 2).join("/") : parts[0]; +} + +export async function buildRemoteRuntime({ + out = DEFAULT_REMOTE_RUNTIME, + commit = desktopCommit(), +} = {}) { + const buildRequire = createRequire(path.join(codeOss, "build/package.json")); + const esbuild = buildRequire("esbuild"); + + fs.rmSync(out, { recursive: true, force: true }); + fs.mkdirSync(path.join(out, "extensions"), { recursive: true }); + + const product = JSON.parse( + fs.readFileSync(path.join(codeOss, "product.json"), "utf8"), + ); + + fs.writeFileSync( + path.join(out, "product.json"), + `${JSON.stringify({ ...product, commit }, null, "\t")}\n`, + ); + + const { name, version } = JSON.parse( + fs.readFileSync(path.join(codeOss, "package.json"), "utf8"), + ); + + fs.writeFileSync( + path.join(out, "package.json"), + `${JSON.stringify({ name, version, type: "module" }, null, "\t")}\n`, + ); + + const tslib = fs.readFileSync( + path.join(codeOss, "node_modules/tslib/tslib.es6.js"), + "utf8", + ); + + const banner = [ + 'import { createRequire as __wbCreateRequire } from "node:module";', + 'import { dirname as __wbDirname } from "node:path";', + 'import { fileURLToPath as __wbFileURLToPath } from "node:url";', + "const require = __wbCreateRequire(import.meta.url);", + "const __filename = __wbFileURLToPath(import.meta.url);", + "const __dirname = __wbDirname(__filename);", + tslib, + ].join("\n"); + + const external = OPTIONAL_NATIVE_PACKAGES.flatMap((pkg) => [ + pkg, + `${pkg}/*`, + ]); + + const results = await Promise.all( + REMOTE_RUNTIME_ENTRIES.map((entry) => + esbuild.build({ + absWorkingDir: codeOss, + entryPoints: [path.join(codeOss, "src", `${entry}.ts`)], + outfile: path.join(out, "out", `${entry}.js`), + bundle: true, + format: "esm", + platform: "node", + target: ["es2024"], + external, + minify: true, + treeShaking: true, + metafile: true, + banner: { js: banner }, + logLevel: "warning", + logOverride: { "unsupported-require-call": "silent" }, + tsconfigRaw: { + compilerOptions: { + experimentalDecorators: true, + useDefineForClassFields: false, + }, + }, + }), + ), + ); + + const unexpected = new Set(); + + for (const result of results) { + for (const output of Object.values(result.metafile.outputs)) { + for (const { path: specifier, external: isExternal } of output.imports) { + if (!isExternal || isBuiltin(specifier)) continue; + + if (!OPTIONAL_NATIVE_PACKAGES.includes(packageName(specifier))) { + unexpected.add(specifier); + } + } + } + } + + if (unexpected.size > 0) { + throw new Error( + `remote runtime left imports unbundled: ${[...unexpected].join(", ")}`, + ); + } + + await buildBuiltinExtensions(out); + + return { out, commit }; +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + const { values } = parseArgs({ + options: { out: { type: "string" }, commit: { type: "string" } }, + }); + + const started = Date.now(); + + const { out, commit } = await buildRemoteRuntime({ + out: values.out ? path.resolve(values.out) : undefined, + commit: values.commit, + }); + + console.log( + `remote runtime at ${out} (commit ${commit}) in ${Date.now() - started} ms`, + ); +} diff --git a/apps/review-desktop/scripts/build-remote-runtime.test.mjs b/apps/review-desktop/scripts/build-remote-runtime.test.mjs new file mode 100644 index 000000000..761dcfbb3 --- /dev/null +++ b/apps/review-desktop/scripts/build-remote-runtime.test.mjs @@ -0,0 +1,92 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawn } from "node:child_process"; +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +import { + REMOTE_BUILTIN_EXTENSIONS, + REMOTE_RUNTIME_ENTRIES, +} from "./build-remote-runtime.mjs"; + +const script = fileURLToPath( + new URL("./build-remote-runtime.mjs", import.meta.url), +); + +test("builds a runtime that starts on this Node and reports the Desktop's commit", async (t) => { + const root = mkdtempSync(path.join(os.tmpdir(), "wb-remote-runtime-")); + const commit = "89abcdef0123456789abcdef0123456789abcdef"; + let server; + t.after(() => { + server?.kill(); + rmSync(root, { recursive: true, force: true }); + }); + + const runtime = path.join(root, "remote-runtime"); + execFileSync(process.execPath, [script, "--out", runtime], { + env: { ...process.env, BUILD_SOURCEVERSION: commit }, + stdio: "pipe", + }); + + for (const entry of REMOTE_RUNTIME_ENTRIES) { + assert.ok(existsSync(path.join(runtime, "out", `${entry}.js`)), entry); + } + + for (const name of REMOTE_BUILTIN_EXTENSIONS) { + const extension = path.join(runtime, "extensions", name); + + const { main } = JSON.parse( + readFileSync(path.join(extension, "package.json"), "utf8"), + ); + + assert.ok(existsSync(path.join(extension, `${main}.js`)), `${name} ${main}`); + assert.ok(!existsSync(path.join(extension, "node_modules")), name); + } + + assert.ok( + existsSync( + path.join(runtime, "extensions/node_modules/typescript/lib/tsserver.js"), + ), + ); + + const tokenFile = path.join(root, "token"); + writeFileSync(tokenFile, "token", { mode: 0o600 }); + server = spawn( + process.execPath, + [ + path.join(runtime, "out/server-main.js"), + "--host", + "127.0.0.1", + "--port", + "0", + "--connection-token-file", + tokenFile, + "--server-data-dir", + path.join(root, "data"), + ], + { stdio: ["ignore", "pipe", "pipe"] }, + ); + let output = ""; + server.stderr.on("data", (chunk) => (output += chunk)); + + const port = await new Promise((resolve, reject) => { + server.once("exit", () => reject(new Error(output))); + server.stdout.on("data", (chunk) => { + output += chunk; + const match = /Extension host agent listening on (\d+)/.exec(output); + + if (match) resolve(Number(match[1])); + }); + }); + + const response = await fetch(`http://127.0.0.1:${port}/version`); + assert.equal(await response.text(), commit); +}); diff --git a/apps/review-desktop/scripts/curated-extensions.manifest.mjs b/apps/review-desktop/scripts/curated-extensions.manifest.mjs index 4eb92d34e..84e41ae98 100644 --- a/apps/review-desktop/scripts/curated-extensions.manifest.mjs +++ b/apps/review-desktop/scripts/curated-extensions.manifest.mjs @@ -114,6 +114,12 @@ export const curatedExtensions = Object.freeze([ "317cb128e8caf2495b955ef6612d828fef809187ac445242116ad8e2e32382ff", size: 16313907, }, + "linux-arm64": { + url: "https://open-vsx.org/api/rust-lang/rust-analyzer/linux-arm64/0.4.2990/file/rust-lang.rust-analyzer-0.4.2990@linux-arm64.vsix", + sha256: + "f8b326bc52f5bd9af94979fea09224697c431ef0e76f0239cb599a86df1e32b7", + size: 15931320, + }, }, executables: ["server/rust-analyzer"], stripExtensionPack: false, @@ -192,6 +198,12 @@ export const curatedExtensions = Object.freeze([ "78bc006683cc998e9fd1a6f2760d8cb3da63096464a217bbd192ecfb490a5516", size: 78144854, }, + "linux-arm64": { + url: "https://open-vsx.org/api/muhammad-sammy/csharp/linux-arm64/2.145.21-g154a82fd27/file/muhammad-sammy.csharp-2.145.21-g154a82fd27@linux-arm64.vsix", + sha256: + "74a50cd78ca1eb45d87eb922b76e6ef3b4d843b4fc8b7b8187a7d2a9583ab532", + size: 75204300, + }, }, executables: [], stripExtensionPack: false, @@ -230,6 +242,7 @@ export const curatedExtensions = Object.freeze([ universal: { sha256: "232aeafb01f069824fdd92d3e628c1c442bbcfa1d3cc945ff97076340bb2b4a6", + size: 6826731, }, }, executables: [], @@ -266,6 +279,12 @@ export const curatedExtensions = Object.freeze([ "linux-x64": { sha256: "d64fc3104f07c4d47c3122a0fa9f2da3e593937c8b506b5f952b4283d877d212", + size: 12848973, + }, + "linux-arm64": { + sha256: + "c8874795cb19bbe3bd3f3a28398a6cbb63fab1bd917b622aa8aaa31efc9f775d", + size: 12328752, }, }, executables: ["bundled/libs/bin/ty"], @@ -297,6 +316,12 @@ export const curatedExtensions = Object.freeze([ "linux-x64": { sha256: "3ed6bc6d6dc9a70cff97698d498844b756110b5c66964689dad5839845f06556", + size: 12225917, + }, + "linux-arm64": { + sha256: + "7a6e4eea699617bb0ff318d0301a52df8d6f36c6a8f9dbc977ba695cef30425e", + size: 11760344, }, }, executables: ["bundled/libs/bin/ruff"], @@ -338,6 +363,20 @@ export const supportedTargets = Object.freeze([ "win32-x64", ]); +export const remoteTargets = Object.freeze(["linux-x64", "linux-arm64"]); + +export const remoteExtensionIds = Object.freeze([ + "ms-python.python", + "astral-sh.ty", + "charliermarsh.ruff", + "golang.go", + "rust-lang.rust-analyzer", + "swiftlang.swift-vscode", + "llvm-vs-code-extensions.lldb-dap", + "muhammad-sammy.csharp", + "ms-dotnettools.vscode-dotnet-runtime", +]); + /** Group tokens accepted by DEV_REVIEW_EXTENSIONS, in display order. */ export const curatedGroups = Object.freeze([ "rust", diff --git a/apps/review-desktop/scripts/curated-extensions.mjs b/apps/review-desktop/scripts/curated-extensions.mjs index 1fbadd457..0bc6bf2a9 100644 --- a/apps/review-desktop/scripts/curated-extensions.mjs +++ b/apps/review-desktop/scripts/curated-extensions.mjs @@ -13,12 +13,10 @@ // node scripts/curated-extensions.mjs --clean # remove materialized dirs // node scripts/curated-extensions.mjs --copy-to # stage into a package -import { createHash } from "node:crypto"; import fs from "node:fs"; import { createRequire } from "node:module"; import os from "node:os"; import path from "node:path"; -import { pipeline } from "node:stream/promises"; import { fileURLToPath } from "node:url"; import { @@ -26,9 +24,20 @@ import { curatedGroups, openVsxUrl, parseGroupSelection, + remoteExtensionIds, + remoteTargets, supportedTargets, targetKeyFor, } from "./curated-extensions.manifest.mjs"; +import { + downloadFile, + downloadPinned, + sha256File, +} from "../../../packages/review/src/pinned-download.ts"; +import { + extractVsix, + sanitizeVsixManifest, +} from "../../../packages/review/src/vsix.ts"; const APP_DIR = path.dirname(path.dirname(fileURLToPath(import.meta.url))); @@ -40,8 +49,6 @@ const CACHE_DIR = path.join(CHECKOUT, ".build", "curated-extensions", "cache"); const STAMP_FILE = ".curated.json"; -const codeOssRequire = createRequire(path.join(CHECKOUT, "package.json")); - /** Maps process.platform/arch onto the manifest's target names. */ export function detectTarget() { const platform = { darwin: "darwin", linux: "linux", win32: "win32" }[ @@ -136,10 +143,6 @@ export function selectExtensions(target, groups) { return selected; } -function sha256Of(file) { - return createHash("sha256").update(fs.readFileSync(file)).digest("hex"); -} - function vsixUrlFor(extension, targetKey) { const pinnedUrl = extension.targets[targetKey].url; @@ -164,51 +167,14 @@ function cachePathFor(extension, targetKey) { ); } -/** Downloads to a temp file and renames, so a killed run never leaves a torn cache entry. */ -async function download(url, destination) { - fs.mkdirSync(path.dirname(destination), { recursive: true }); - // Open VSX answers with a 302 to its storage host; fetch follows by default. - const response = await fetch(url, { redirect: "follow" }); - - if (!response.ok) { - throw new Error( - `GET ${url} failed with ${response.status} ${response.statusText}`, - ); - } - - const partial = `${destination}.part`; - fs.writeFileSync(partial, Buffer.from(await response.arrayBuffer())); - fs.renameSync(partial, destination); -} - -async function ensureVsix( - extension, - targetKey, - expectedSha, - { allowDownload }, -) { +async function ensureVsix(extension, targetKey) { const cached = cachePathFor(extension, targetKey); + const { sha256, size } = extension.targets[targetKey]; - if (!fs.existsSync(cached)) { - if (!allowDownload) { - throw new Error( - `${extension.id}: ${cached} is missing and downloads are disabled`, - ); - } - - await download(vsixUrlFor(extension, targetKey), cached); - } - - const actual = sha256Of(cached); - - if (actual !== expectedSha) { - // A mismatch means the pin is stale or the download was tampered with. - // Drop the file so the next run refetches instead of failing forever. - fs.rmSync(cached, { force: true }); - throw new Error( - `${extension.id} checksum mismatch for ${targetKey}\n expected ${expectedSha}\n actual ${actual}`, - ); - } + await downloadPinned( + { url: vsixUrlFor(extension, targetKey), sha256, size }, + cached, + ); return cached; } @@ -232,30 +198,6 @@ function stampMatches(stamp, extension, targetKey, sha256) { ); } -/** - * VSIX payloads declare `dependencies` they ship prebundled and `scripts` that - * only make sense in their own repo. Both confuse tooling that walks - * extensions/*/package.json (vsce's npm file listing in particular), so drop - * them. `extensionPack` is dropped for extensions whose pack members Review - * deliberately does not ship. - */ -function sanitizeManifest(directory, extension) { - const manifestPath = path.join(directory, "package.json"); - const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")); - delete manifest.scripts; - delete manifest.dependencies; - delete manifest.devDependencies; - - if (extension.stripExtensionPack) { - delete manifest.extensionPack; - } - - const engine = manifest.engines?.vscode; - fs.writeFileSync(manifestPath, `${JSON.stringify(manifest, undefined, 2)}\n`); - - return { engine, id: `${manifest.publisher}.${manifest.name}` }; -} - function prunePayload(directory, extension) { for (const relative of extension.prunePaths ?? []) { fs.rmSync(path.join(directory, relative), { recursive: true, force: true }); @@ -280,96 +222,7 @@ function ensureExecutables(directory, extension, targetKey) { } } -/** Extract the extension payload with Code OSS's pinned ZIP reader. */ -export async function extractVsixPayload(vsix, destination) { - const yauzl = codeOssRequire("yauzl"); - - await new Promise((resolve, reject) => { - yauzl.open(vsix, { lazyEntries: true }, (openError, zipFile) => { - if (openError) { - reject(openError); - - return; - } - - const pending = new Set(); - let failure; - - const fail = (error) => { - if (failure) { - return; - } - - failure = error; - zipFile.close(); - reject(error); - }; - - zipFile.on("error", fail); - zipFile.on("entry", (entry) => { - const name = entry.fileName.replaceAll("\\", "/"); - - if (name.endsWith("/") || !name.startsWith("extension/")) { - zipFile.readEntry(); - - return; - } - - const relative = name.slice("extension/".length); - const parts = relative.split("/"); - - if ( - parts.some((part) => part === ".." || part === "." || part === "") || - path.posix.isAbsolute(relative) || - path.win32.isAbsolute(relative) || - parts.some((part) => part.includes(":")) - ) { - fail(new Error(`VSIX contains an unsafe extension path: ${name}`)); - - return; - } - - const mode = (entry.externalFileAttributes >>> 16) & 0xffff; - - if ((mode & 0o170000) === 0o120000) { - fail(new Error(`VSIX contains an unexpected symlink: ${name}`)); - - return; - } - - const output = path.join(destination, ...parts); - fs.mkdirSync(path.dirname(output), { recursive: true }); - zipFile.openReadStream(entry, (streamError, stream) => { - if (streamError) { - fail(streamError); - - return; - } - - const task = pipeline(stream, fs.createWriteStream(output)) - .then(() => { - if (process.platform !== "win32" && (mode & 0o111) !== 0) { - fs.chmodSync(output, mode & 0o777); - } - }) - .catch(fail) - .finally(() => pending.delete(task)); - - pending.add(task); - zipFile.readEntry(); - }); - }); - zipFile.on("end", () => { - Promise.all(pending).then(() => { - if (!failure) resolve(); - }, fail); - }); - zipFile.readEntry(); - }); - }); -} - -async function extractVsix(vsix, extension, targetKey, sha256) { +async function materializeVsix(vsix, extension, targetKey, sha256) { const destination = path.join(EXTENSIONS_DIR, extension.id); const staging = `${destination}.staging`; fs.rmSync(staging, { recursive: true, force: true }); @@ -378,21 +231,15 @@ async function extractVsix(vsix, extension, targetKey, sha256) { try { const payload = path.join(staging, "extension"); fs.mkdirSync(payload, { recursive: true }); - await extractVsixPayload(vsix, payload); + await extractVsix(vsix, payload); if (!fs.existsSync(payload)) { throw new Error(`${extension.id}: VSIX has no extension/ payload`); } - const { engine, id } = sanitizeManifest(payload, extension); + const engine = await sanitizeVsixManifest(payload, extension); prunePayload(payload, extension); - if (id.toLowerCase() !== extension.id.toLowerCase()) { - throw new Error( - `${extension.id}: VSIX declares a different identifier (${id})`, - ); - } - ensureExecutables(payload, extension, targetKey); fs.writeFileSync( path.join(payload, STAMP_FILE), @@ -562,16 +409,21 @@ async function printHashes(target) { for (const extension of curatedExtensions) { const targetKeys = extension.targets.universal ? ["universal"] - : supportedTargets; + : [ + ...new Set([ + ...supportedTargets, + ...(remoteExtensionIds.includes(extension.id) ? remoteTargets : []), + ]), + ]; for (const targetKey of targetKeys) { const cached = cachePathFor(extension, targetKey); - if (!fs.existsSync(cached)) { - await download(vsixUrlFor(extension, targetKey), cached); - } + const sha256 = fs.existsSync(cached) + ? await sha256File(cached) + : await downloadFile(vsixUrlFor(extension, targetKey), cached); - console.log(`${extension.id} ${targetKey} ${sha256Of(cached)}`); + console.log(`${extension.id} ${targetKey} ${sha256}`); } } @@ -651,11 +503,9 @@ async function main() { throw new Error(`${extension.id} is not materialized for ${target}`); } - const vsix = await ensureVsix(extension, targetKey, sha256, { - allowDownload: !options.check, - }); + const vsix = await ensureVsix(extension, targetKey); - await extractVsix(vsix, extension, targetKey, sha256); + await materializeVsix(vsix, extension, targetKey, sha256); verifyEngine(destination, extension); console.log( `materialized ${extension.id}@${extension.version} (${targetKey})`, diff --git a/apps/review-desktop/scripts/curated-extensions.test.mjs b/apps/review-desktop/scripts/curated-extensions.test.mjs index 41173c604..3f2732185 100644 --- a/apps/review-desktop/scripts/curated-extensions.test.mjs +++ b/apps/review-desktop/scripts/curated-extensions.test.mjs @@ -26,12 +26,14 @@ import { openVsxUrl, optionalExtensions, parseGroupSelection, + remoteExtensionIds, + remoteTargets, supportedTargets, targetKeyFor, } from "./curated-extensions.manifest.mjs"; +import { extractVsix } from "../../../packages/review/src/vsix.ts"; import { copyCuratedExtensions, - extractVsixPayload, verifyCuratedExtensions, } from "./curated-extensions.mjs"; @@ -94,9 +96,13 @@ test("pins every curated extension to a checksum for every supported target", () if (extension.targets.universal) { assert.deepEqual(targetKeys, ["universal"], `${extension.id} targets`); } else { + const expected = remoteExtensionIds.includes(extension.id) + ? new Set([...supportedTargets, ...remoteTargets]) + : new Set(supportedTargets); + assert.deepEqual( targetKeys.sort(), - [...supportedTargets].sort(), + [...expected].sort(), `${extension.id} must pin every supported target`, ); } @@ -211,7 +217,7 @@ test("extracts nested Windows executables from a VSIX archive", async () => { try { await complete; writeFileSync(archive, Buffer.concat(chunks)); - await extractVsixPayload(archive, destination); + await extractVsix(archive, destination); assert.equal( readFileSync( diff --git a/apps/review-desktop/scripts/e2e/TESTING.md b/apps/review-desktop/scripts/e2e/TESTING.md index 06d2fa530..73b2e95a3 100644 --- a/apps/review-desktop/scripts/e2e/TESTING.md +++ b/apps/review-desktop/scripts/e2e/TESTING.md @@ -43,8 +43,11 @@ prints a JSON summary on stdout, one entry per journey, `ok | failed | skipped`. ## Phases Phase 1 runs offline, after a one-time network fetch of the curated VSIX cache -that `lsp-python` triggers. Phase 2 (`lsp-go`, `lsp-rust`) downloads toolchains -and runs only with `REVIEW_E2E_NETWORK=1`. In development mode each journey +that `lsp-python` triggers. Phase 2 (`lsp-go`, `lsp-rust`, `remote-host`, `remote-lsp`, `remote-install`) +downloads toolchains or a container image and runs only with +`REVIEW_E2E_NETWORK=1` or when named with `--journey`. A journey that exports +`manual = true` (`remote-lsp-rust`, `remote-lsp-swift`, `remote-lsp-csharp`, +whose container images are large) runs only when named. In development mode each journey re-materializes its extension group through `run.sh`, so this checkout's `code-oss/extensions` holds the last journey's selection afterwards; `node scripts/curated-extensions.mjs --only=all` restores it. @@ -106,3 +109,208 @@ request fails; `install` gives the route back only while `npm` runs, then checks again. `--delay-ms` delays both directions with `netem`. Both run their network commands from a throwaway container, so the remote itself never holds `NET_ADMIN`. + +## The remote-host journey + +`remote-host` runs the review server on another machine: a container from +`remote.mjs` with `sshd` on a loopback port and the package from this checkout. +It needs Docker; without it the journey is skipped with +`skip: remote-host needs Docker for its SSH server`. The Desktop gets +`DEV_FAST_REVIEW_SSH_CONFIG`, so its `ssh` uses the run's configuration and +key, and the journey adds `wb-test-a` in Settings as a user would. It checks, +on the DOM and on the page's requests: the host goes `online`; Home lists the +container's review as `wb-test-a: wbrepo`; the document, a code peek, the Diff +view and the structural diff load; an edit and a `session_create` on the +remote reach the window; a killed `ssh` master shows "Connection lost" and +recovers without a reload; a stopped server is `offline` within 15 s while a +laptop review still opens; another package version is `incompatible` with the +install command in Settings; and removing the host takes its reviews out of +Home. Throughout, no Desktop route fails, and no request from the window goes +anywhere but the local server or carries the remote's token. + +It runs in development mode only: a packaged build ignores +`DEV_FAST_REVIEW_SSH_CONFIG`, so with `--app` the journey is skipped. + +To run it against a host you prepared yourself, such as an AWS instance with +the package installed by hand, set `REVIEW_E2E_REMOTE_HOST` to the host's name +in the `WB_TEST_RUN` run. The journey then skips the Docker check, `up` and +`install`, and step 8, which would replace the package. It never removes that +run: it quits the Desktop and ends the Desktop's `ssh`, and you remove the run. + +```sh +export WB_TEST_RUN=aws-$$ +$R aws-up b --arch arm64 # prints wb-test-b; install the package there by hand +REVIEW_E2E_REMOTE_HOST=b node apps/review-desktop/scripts/e2e/run.mjs --runtime "$REVIEW_E2E_RUNTIME" --journey remote-host +$R down --all; $R verify-clean +``` + +Otherwise the journey removes its run with `down --all` when it ends. A runner killed +before that leaves the run behind, so name the run and trap it: + +```sh +R="node apps/review-desktop/scripts/e2e/remote/remote.mjs" +export WB_TEST_RUN=e2e-$$ +trap '$R down --all; $R verify-clean' EXIT +node apps/review-desktop/scripts/e2e/run.mjs --runtime "$REVIEW_E2E_RUNTIME" --journey remote-host +``` + +## The remote-lsp journey + +`remote-lsp` checks language features for remote reviews: a laptop and two +containers (`wb-test-a`, `wb-test-b`) in one window. `a.ts` exports `answer` +as `1`, `42` and `99` at the same path, `/tmp/wbt./proj`, on each +machine, and a hover in each review's Diff tab must show that machine's +number. In order, it checks: + +1. both hosts report language features available; +2. four reviews are open at once; +3. hovers and go to definition answer from their own machine; +4. a Python hover from ty works on A; +5. the laptop's own providers are unchanged; +6. A and the laptop keep answering while B is frozen and its ssh master killed + for longer than its reconnection grace, and B answers again after, without + a reload; +7. B answers again after its VS Code server is killed; +8. B with another commit reads normally, has no hovers, and Settings says why; +9. after two reloads each remote has one extension host, and none after the + close; +10. times and memory, failing a first hover over 10 s or a warm one over + 500 ms. + +Before running it, build the Desktop and the remote runtime from this +checkout, and stage the runtime as above: + +```sh +REVIEW_DESKTOP_DEV_FAST=1 DEV_REVIEW_EXTENSIONS=none pnpm desktop:build +node apps/review-desktop/scripts/build-remote-runtime.mjs +R="node apps/review-desktop/scripts/e2e/remote/remote.mjs" +export WB_TEST_RUN=e2e-$$ DEV_FAST_REVIEW_DESKTOP_BACKGROUND=1 +trap '$R down --all; $R verify-clean' EXIT +node apps/review-desktop/scripts/e2e/run.mjs --runtime "$REVIEW_E2E_RUNTIME" --journey remote-lsp +``` + +Like `remote-host`, it needs Docker and runs in development mode only. The +containers download the language extensions from Open VSX. For the +commit check, the journey writes `code-oss/product.overrides.json` with the +remote runtime's commit and removes it when the run ends; it refuses to run +over a different one. Each check is printed on stderr as it passes. + +Two development-only variables, set in the environment of `whiteboard remote +attach` on a remote, change only that remote's VS Code server: + +- `DEV_FAST_REVIEW_REMOTE_SHUTDOWN_WITHOUT_DELAY=1` makes it exit as soon as the last window leaves, not 5 minutes later. +- `DEV_FAST_REVIEW_REMOTE_RECONNECTION_GRACE_SECONDS=<1–9999>` replaces its 10-minute reconnection grace; the journey gives B 20 s for step 6. + +Against two hosts you prepared, such as AWS instances, set +`REVIEW_E2E_REMOTE_HOSTS` to their two names in the `WB_TEST_RUN` run. Install +Node 24 and the package on each by hand, from `$R pack --out `. The +journey then skips `up` and `install` and the package swap (step 8). It +freezes B with `kill -STOP` on its servers, and it leaves the run for you to +remove: + +```sh +export WB_TEST_RUN=aws-$$ +$R aws-up a --arch x64; $R aws-up b --arch arm64 +$R pack --out /tmp/wb.tgz # then, on each host: Node 24, `sudo npm install -g` the tarball +REVIEW_E2E_REMOTE_HOSTS=a,b node apps/review-desktop/scripts/e2e/run.mjs --runtime "$REVIEW_E2E_RUNTIME" --journey remote-lsp +$R down --all; $R verify-clean +``` + +## The remote-lsp-rust, -swift and -csharp journeys + +Each checks one optional language group on a container built from the +toolchain's official image (`remote.mjs up --toolchain rust|swift|dotnet`), +with the toolchain on the login shell's `PATH` only for Rust and .NET: + +1. the toolchain is found by a login shell; +2. the group is turned on in the Desktop's extension picker; +3. a review of the fixture from `fixtures/lsp/` on the remote; +4. the host added in Settings: language features available, the row lists the + group as installed, and the remote installed that group and no other + optional group (Rust: the server binary runs); +5. Swift only: a second host, `wb-test-d`, without Swift, attaches, and its + Settings row names the missing `swift`; Swift and C# stop here: they do + not answer hovers yet (`docs/remote-hosts.md`); +6. a pointer hover shows the type, within 120 s of the Diff click; +7. go to definition opens the remote's file, read-only; +8. the extension host's `PATH` holds the toolchain, no debugger runs, and the + remote's memory; +9. no extension host is left after the window closes. + +Build and stage as for `remote-lsp`, then run one by name. Set +`REVIEW_E2E_REMOTE_PLATFORM=linux/amd64` for an x64 container. On failure the +journey prints the remote's processes and extension logs. + +```sh +R="node apps/review-desktop/scripts/e2e/remote/remote.mjs" +export WB_TEST_RUN=e2e-$$ DEV_FAST_REVIEW_DESKTOP_BACKGROUND=1 +trap '$R down --all; $R verify-clean' EXIT +node apps/review-desktop/scripts/e2e/run.mjs --runtime "$REVIEW_E2E_RUNTIME" --journey remote-lsp-rust +``` + +The base images stay in Docker after `down`; remove them with `docker rmi` +when done (`rust:1.98-bookworm`, `swift:6.4-noble`, +`mcr.microsoft.com/dotnet/sdk:10.0-noble`). + +## The remote-install journey + +`remote-install` proves that Desktop installs itself on a remote: it brings up +containers with nothing of Whiteboard on them, adds each one in Settings, +answers the install question in the window, and checks each host over SSH. +Like `remote-host`, it needs Docker, runs in development mode only, and +removes its run with `down --all` when it ends. + +| Container | `up` flags | What it proves | +|---|---|---| +| `fresh` | `--platform linux/amd64 --node none` | The first install, x64 target: the question, each step in Settings, `online`, and a review written there with `~/.local/bin/whiteboard` listed in Home. Later: the next version, Connect for an agent, and removal with "Also remove Whiteboard". | +| `node` | `--node 24` | The host's own Node runs Whiteboard; no `~/.dev/whiteboard-remote/node`. Later: a server the user started at the old version makes the newer Desktop `incompatible`, and keeps running. | +| `sealed` | `--sealed --node none` | No route out: Node and the package are uploaded, npm goes through Desktop's relay. | +| `old` | `--image debian:11 --node none` | glibc 2.31: `unsupported`, no question, nothing written in the home. | +| `arm` | `--platform linux/arm64 --node none` | The ARM64 target. | +| `fresh2` | `--platform linux/amd64 --node none` | The ssh master killed during the package step: the reconnect finishes the install, with no `.part` left. | +| `deny` | `--node none` | "Don't install": `not-installed` with the npm command, nothing written, and no second question after Retry. | + +On an Apple-silicon Mac the containers without `--platform` are ARM64 too, +and `fresh` and `fresh2` run under emulation; elsewhere, swap the two +`--platform` values if the emulated one is too slow. + +For "the next version" the journey raises the patch version in +`packages/review/package.json`, which the Desktop's server reports and its +development pack carries, restarts the Desktop, and writes the file back when +it ends. Do not run it while you edit that file. Run it under a trap that also +restores the file, in case the runner is killed: + +```sh +R="node apps/review-desktop/scripts/e2e/remote/remote.mjs" +export WB_TEST_RUN=e2e-$$ +trap '$R down --all; $R verify-clean; git checkout -- packages/review/package.json' EXIT +DEV_FAST_REVIEW_DESKTOP_BACKGROUND=1 node apps/review-desktop/scripts/e2e/run.mjs --runtime "$REVIEW_E2E_RUNTIME" --journey remote-install +``` + +The connect step installs Claude Code from npm on `fresh`, and needs GitHub +there for its plugin, so the journey needs the network throughout. It +prints the time from **Install** to `online` for each host in its checks. + +### On AWS + +With `REVIEW_E2E_REMOTE_HOSTS`, a comma-separated list of host names in the +`WB_TEST_RUN` run, the journey uses hosts you brought up with `aws-up` and +did not touch: for each one it checks that nothing of Whiteboard is there, +adds it, answers Install, waits for `online`, and lists a review written +there in Home, reporting the time from Install to `online`. It runs no +container step and never removes the run. `aws-up` keeps at most two +instances at a time, so run the sealed host on its own: + +```sh +R="node apps/review-desktop/scripts/e2e/remote/remote.mjs" +export WB_TEST_RUN=aws-$$ +trap '$R down --all; $R verify-clean' EXIT +$R aws-up a --arch x64 +$R aws-up b --arch arm64 +REVIEW_E2E_REMOTE_HOSTS=a,b DEV_FAST_REVIEW_DESKTOP_BACKGROUND=1 \ + node apps/review-desktop/scripts/e2e/run.mjs --runtime "$REVIEW_E2E_RUNTIME" --journey remote-install +$R down a; $R down b +$R aws-up c --sealed +REVIEW_E2E_REMOTE_HOSTS=c DEV_FAST_REVIEW_DESKTOP_BACKGROUND=1 \ + node apps/review-desktop/scripts/e2e/run.mjs --runtime "$REVIEW_E2E_RUNTIME" --journey remote-install +``` diff --git a/apps/review-desktop/scripts/e2e/fixtures/lsp/csharp/Orders.cs b/apps/review-desktop/scripts/e2e/fixtures/lsp/csharp/Orders.cs new file mode 100644 index 000000000..369903fd5 --- /dev/null +++ b/apps/review-desktop/scripts/e2e/fixtures/lsp/csharp/Orders.cs @@ -0,0 +1,8 @@ +namespace Orders; + +public static class Queue +{ + /// Hands a queued order to the storage layer. + public static OrderRecord QueueOrder(string id) => + Storage.SaveOrder(new OrderRecord(id, "queued")); +} diff --git a/apps/review-desktop/scripts/e2e/fixtures/lsp/csharp/Orders.csproj b/apps/review-desktop/scripts/e2e/fixtures/lsp/csharp/Orders.csproj new file mode 100644 index 000000000..0957a12e2 --- /dev/null +++ b/apps/review-desktop/scripts/e2e/fixtures/lsp/csharp/Orders.csproj @@ -0,0 +1,6 @@ + + + net10.0 + enable + + diff --git a/apps/review-desktop/scripts/e2e/fixtures/lsp/csharp/Storage.cs b/apps/review-desktop/scripts/e2e/fixtures/lsp/csharp/Storage.cs new file mode 100644 index 000000000..bc181ebaa --- /dev/null +++ b/apps/review-desktop/scripts/e2e/fixtures/lsp/csharp/Storage.cs @@ -0,0 +1,10 @@ +namespace Orders; + +/// One stored order. +public record OrderRecord(string Id, string Status); + +public static class Storage +{ + /// Persists the order and returns what was stored. + public static OrderRecord SaveOrder(OrderRecord order) => order; +} diff --git a/apps/review-desktop/scripts/e2e/fixtures/lsp/swift/Package.swift b/apps/review-desktop/scripts/e2e/fixtures/lsp/swift/Package.swift new file mode 100644 index 000000000..454ce96c1 --- /dev/null +++ b/apps/review-desktop/scripts/e2e/fixtures/lsp/swift/Package.swift @@ -0,0 +1,7 @@ +// swift-tools-version:5.9 +import PackageDescription + +let package = Package( + name: "Orders", + targets: [.target(name: "Orders")] +) diff --git a/apps/review-desktop/scripts/e2e/fixtures/lsp/swift/Sources/Orders/Orders.swift b/apps/review-desktop/scripts/e2e/fixtures/lsp/swift/Sources/Orders/Orders.swift new file mode 100644 index 000000000..a80695886 --- /dev/null +++ b/apps/review-desktop/scripts/e2e/fixtures/lsp/swift/Sources/Orders/Orders.swift @@ -0,0 +1,5 @@ +/// Hands a queued order to the storage layer. +public func queueOrder(id: String) -> OrderRecord { + let status = "queued" + return saveOrder(OrderRecord(id: id, status: status)) +} diff --git a/apps/review-desktop/scripts/e2e/fixtures/lsp/swift/Sources/Orders/Storage.swift b/apps/review-desktop/scripts/e2e/fixtures/lsp/swift/Sources/Orders/Storage.swift new file mode 100644 index 000000000..63f15796f --- /dev/null +++ b/apps/review-desktop/scripts/e2e/fixtures/lsp/swift/Sources/Orders/Storage.swift @@ -0,0 +1,10 @@ +/// One stored order. +public struct OrderRecord { + public var id: String + public var status: String +} + +/// Persists the order and returns what was stored. +public func saveOrder(_ order: OrderRecord) -> OrderRecord { + order +} diff --git a/apps/review-desktop/scripts/e2e/harness.mjs b/apps/review-desktop/scripts/e2e/harness.mjs index f14bc973a..06c6f9ea2 100644 --- a/apps/review-desktop/scripts/e2e/harness.mjs +++ b/apps/review-desktop/scripts/e2e/harness.mjs @@ -563,13 +563,16 @@ export async function createHarness({ } // Artifacts are best-effort: a Desktop that outlives the runner is worse than a missing screenshot. - if (!success && page) { - await page + // A journey may have moved ctx.page to another window after a relaunch. + const shown = ctx.page ?? page; + + if (!success && shown) { + await shown .screenshot({ path: path.join(root, "failure.png") }) .catch(() => {}); await writeFile( path.join(root, "failure-dom.txt"), - await page + await shown .locator("body") .innerText() .catch(() => ""), diff --git a/apps/review-desktop/scripts/e2e/journeys/remote-ask.mjs b/apps/review-desktop/scripts/e2e/journeys/remote-ask.mjs new file mode 100644 index 000000000..6cf44e2ad --- /dev/null +++ b/apps/review-desktop/scripts/e2e/journeys/remote-ask.mjs @@ -0,0 +1,429 @@ +/** Ask on a review whose checkout is on another machine: a container with a fake OpenCode, reached over SSH and added in Settings as a user would. */ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +import { closeSourceWindow, openHome, openSettings } from "../harness.mjs"; +import { + alias, + assertUriAnswers, + closeDesktop, + createRemoteReview, + masterPid, + onRemote, + prepared, + recordNavigator, + recordRequests, + remote, + remoteToken, + runDir, + sourcePages, +} from "./remote-host.mjs"; +import { removeHost } from "./remote-install.mjs"; + +const exec = promisify(execFile); + +const title = "Remote ask"; + +const question = "what does f.ts do"; + +export const name = "remote-ask"; + +// Phase 2: the container image downloads Ubuntu packages and Node. +export const phase = 2; + +export const options = { + settings: { "review.experimental.remoteHosts.enabled": true }, + env: { DEV_FAST_REVIEW_SSH_CONFIG: `${runDir}/ssh_config` }, +}; + +export async function run(ctx) { + // A packaged build ignores DEV_FAST_REVIEW_SSH_CONFIG, so its ssh would read the user's configuration. + if (ctx.report.mode === "packaged") + throw new Error("skip: remote-ask runs in development mode only"); + + if (prepared === undefined) + try { + await exec("docker", ["info", "--format", "{{.ServerVersion}}"]); + } catch (error) { + throw new Error( + `skip: remote-ask needs Docker for its SSH server (${error.message.split("\n")[0]})`, + ); + } + + try { + await journey(ctx); + } finally { + if (prepared === undefined) + await remote("down", "--all").catch((error) => + console.error(`[remote-ask] down --all: ${error.message}`), + ); + else await closeDesktop(ctx); + } +} + +/** Pauses the container, or stops a prepared host's server and returns its resume. */ +async function freeze() { + if (prepared === undefined) { + await remote("pause", "a"); + + return undefined; + } + + const { serverPid } = JSON.parse( + await onRemote("whiteboard server status --json"), + ); + + await onRemote(`kill -STOP ${serverPid}`); + + return () => onRemote(`kill -CONT ${serverPid}`); +} + +const reviewCanvas = (ctx) => + ctx.page.locator(".review-canvas-root [data-review-api]"); + +/** Ask, docked in the side panel or floating, by the layout. */ +export const askPanel = (ctx) => + ctx.page.locator( + '[role=complementary][aria-label="Ask"], [role=dialog][aria-label="Ask"]', + ); + +/** Home's row for the review titled `title`. */ +export const sessionRow = (ctx, title) => + ctx.page + .locator("main.review-home") + .getByRole("region", { name: "Sessions", exact: true }) + .locator("tbody tr") + .filter({ hasText: title }); + +/** Opens the review titled `title` from Home. */ +export async function openRemoteReview(ctx, title) { + await openHome(ctx); + await sessionRow(ctx, title).waitFor({ timeout: 60000 }); + await sessionRow(ctx, title).getByTitle(title, { exact: true }).click(); + await reviewCanvas(ctx).getByRole("heading", { name: title }).waitFor({ + timeout: 60000, + }); +} + +/** Selects the document's sentence and opens Ask on it with ⌘L. */ +export async function askAboutReviewSentence(ctx) { + // The workbench's usage-data notice sits over the composer's Ask button. + for (const clear of await ctx.page + .locator(".notifications-toasts .codicon-notifications-clear") + .all()) + await clear.click().catch(() => {}); + + const sentence = reviewCanvas(ctx).getByText("One now returns"); + + await sentence.waitFor(); + await sentence.evaluate((element) => { + const range = document.createRange(); + + range.selectNodeContents(element); + document.getSelection().removeAllRanges(); + document.getSelection().addRange(range); + }); + await ctx.page + .getByRole("button", { name: "Ask OpenCode", exact: true }) + .waitFor({ timeout: 30000 }); + await ctx.page.keyboard.press("Meta+l"); + await askPanel(ctx) + .getByRole("combobox", { name: "Question" }) + .waitFor({ timeout: 30000 }); +} + +async function journey(ctx) { + const { until } = ctx; + const timings = {}; + + const hostState = async () => + (await ctx.apiOk("/remote-hosts")).find((host) => host.alias === alias) + ?.state; + + const waitState = (state, label, timeout = 60000) => + until( + async () => (await hostState()) === state, + `${alias} ${label}`, + timeout, + ); + + const canvas = () => reviewCanvas(ctx); + + const panel = () => askPanel(ctx); + + const composer = () => panel().getByRole("combobox", { name: "Question" }); + + const homeRow = () => sessionRow(ctx, title); + + const openReview = () => openRemoteReview(ctx, title); + + const askAboutSentence = () => askAboutReviewSentence(ctx); + + /** Resolves with the ms until the panel shows the answer. */ + async function send(text) { + await composer().fill(text); + + const sent = Date.now(); + + await panel().getByRole("button", { name: "Ask", exact: true }).click(); + await panel().getByText(`You asked: ${text}`).waitFor({ timeout: 20000 }); + + return Date.now() - sent; + } + + const savedConversations = () => + canvas().getByRole("button", { name: "Saved conversations" }); + + async function historyLists(label) { + await until( + async () => { + await savedConversations().first().click(); + + return ( + (await panel() + .getByRole("list") + .filter({ hasText: question }) + .count()) > 0 + ); + }, + label, + 30000, + ); + } + + // 1. A container with sshd, this checkout's package and the fake OpenCode, or a prepared host, holding a review. + if (prepared === undefined) { + await remote("up", "a", "--fake-agent"); + await remote("install", "a"); + } + + const [reviewId] = ( + await onRemote(`bash -s -- '${title}'`, createRemoteReview) + ).split("\n"); + + assert.match(reviewId, /^[0-9a-f-]{36}$/, "the remote review's id"); + + const remoteTokens = new Set([await remoteToken()]); + + ctx.check(`1. ${alias}, with a fake OpenCode, holds a review`); + + // 2. Settings, as a user adds a host. + const section = (await openSettings(ctx)).getByRole("region", { + name: "Remote hosts", + }); + + await section.getByLabel("SSH alias").fill(alias); + + const added = Date.now(); + + await section.getByRole("button", { name: "Add", exact: true }).click(); + await waitState("online", "online"); + timings.addToOnline = Date.now() - added; + ctx.check( + `2. Settings added ${alias}; online ${timings.addToOnline} ms after Add`, + ); + + // 3. The remote review, a sentence selected, ⌘L. + const requests = await recordRequests(ctx.page, /\/ask\/[^/]+\/watch(\?|$)/); + const recorders = [{ requests, origin: new URL(ctx.discovery.url).origin }]; + const navigator = []; + + await recordNavigator(ctx.page, navigator); + + await openReview(); + await askAboutSentence(); + ctx.check("3. ⌘L on a selected sentence of the remote review opened Ask"); + + // 4. The agents are the remote's: with OpenCode gone, the panel names the host; back, it answers again. + await onRemote("mv ~/.opencode ~/.opencode.off"); + await until( + async () => { + await ctx.page.evaluate(() => window.dispatchEvent(new Event("focus"))); + + return ( + (await panel().getByText(`Not installed on ${alias}`).count()) >= 3 + ); + }, + `the panel to read Not installed on ${alias}`, + 30000, + ); + + for (const agent of ["Claude Code", "Codex", "OpenCode"]) + assert.match( + await panel() + .getByRole("listitem") + .filter({ hasText: agent }) + .innerText(), + new RegExp(`Not installed on ${alias}`), + ); + + await onRemote("mv ~/.opencode.off ~/.opencode"); + await until( + async () => { + await ctx.page.evaluate(() => window.dispatchEvent(new Event("focus"))); + + return (await composer().count()) > 0; + }, + "the composer to come back with OpenCode", + 30000, + ); + await panel().getByText("OpenCode", { exact: true }).waitFor(); + + for (const name of ["OpenCode", "Choose an agent"]) + assert.equal( + await panel().getByRole("button", { name, exact: true }).count(), + 0, + ); + ctx.check( + `4. without the remote's OpenCode, Claude Code, Codex and OpenCode read Not installed on ${alias}; with it back, the header names OpenCode alone, without a menu`, + ); + + // 5. The question goes to the remote's agent; its answer streams back through the gateway. + timings.firstAnswer = await send(question); + ctx.check( + `5. OpenCode on ${alias} answered "You asked: …" ${timings.firstAnswer} ms after Ask`, + ); + + // 6. The answer's f.ts is a link: it opens the host's file in a Source window. + const reference = panel() + .getByRole("link") + .filter({ hasText: /^f\.ts$/ }); + + const { serverId } = (await ctx.apiOk("/remote-hosts")).find( + (host) => host.alias === alias, + ); + + const authority = `whiteboard+${serverId.toLowerCase()}`; + + await reference.waitFor(); + await reference.click(); + + const source = await until( + () => sourcePages(ctx)[0], + "a Source window for the answer's f.ts", + 60000, + ); + + await until( + async () => + ( + await source.locator(".tabs-container .tab.active").innerText() + ).includes("f.ts"), + "f.ts in the Source window", + 60000, + ); + await closeSourceWindow(source); + ctx.check("6. the answer's f.ts link opened f.ts in a Source window"); + + // 7. Ask spoke only to the local server, and no remote token reached the page. + const asks = audit(recorders, remoteTokens); + + assert.ok( + asks.some((r) => r.received?.toString().includes("You asked:")), + "no watch stream carried the answer", + ); + ctx.check( + `7. ${asks.length} Ask requests, all to the local server, none with the remote token; the answer came on its watch stream`, + ); + + // 8. The ssh master dies and comes back: the thread is still saved. + const master = await masterPid(); + + assert.ok(master, "no ssh master for the host"); + process.kill(master, "SIGKILL"); + await until( + async () => + (await hostState()) === "online" && (await masterPid()) !== master, + `${alias} online on a new master`, + 60000, + ); + await panel().getByRole("button", { name: "Close Ask" }).click(); + await historyLists("the thread in Ask history after the reattach"); + ctx.check("8. after a new ssh master, Ask history lists the thread"); + + // 9. A Desktop restart: the thread is still listed. + await ctx.quitAndRelaunchDesktop(); + recorders.push({ + requests: await recordRequests(ctx.page, /^$/), + origin: new URL(ctx.discovery.url).origin, + }); + await waitState("online", "online after the relaunch"); + remoteTokens.add(await remoteToken()); + await openReview(); + await historyLists("the thread in Ask history after the relaunch"); + await panel().getByRole("button", { name: "Close Ask" }).click(); + ctx.check("9. after a Desktop restart, Ask history lists the thread"); + + // 10. The host goes away with a conversation open: the panel says so, and the host reads offline. + await askAboutSentence(); + await send("and then"); + const thaw = await freeze(); + + const paused = Date.now(); + + await until( + async () => + (await panel() + .getByRole("alert") + .filter({ hasText: "lost its connection" }) + .count()) > 0, + "the panel's lost-connection alert", + 20000, + ); + timings.streamEnded = Date.now() - paused; + await waitState("offline", "offline after the pause", 20000); + timings.offline = Date.now() - paused; + assert.ok( + timings.streamEnded <= 15000, + `alert after ${timings.streamEnded} ms`, + ); + assert.ok(timings.offline <= 15000, `offline after ${timings.offline} ms`); + await thaw?.(); + ctx.check( + `10. ${prepared === undefined ? "docker pause" : "server SIGSTOP"}: the panel said it lost OpenCode after ${timings.streamEnded} ms; ${alias} offline after ${timings.offline} ms`, + ); + + // 11. Removing the host takes the review out of Home. + await removeHost(ctx, alias); + await openHome(ctx); + await until( + async () => (await homeRow().count()) === 0, + "the remote review to leave Home", + ); + const allAsks = audit(recorders, remoteTokens); + + const answers = assertUriAnswers(navigator, authority); + + ctx.check( + `11. removing the host took the review out of Home; across both launches ${allAsks.length} Ask requests, all to the local server, none with a remote token, and ${answers} navigator answer(s), each with URIs only`, + ); +} + +/** Asserts each launch's Ask requests stayed local and none carried a remote token. */ +function audit(recorders, remoteTokens) { + const asks = []; + + for (const { requests, origin } of recorders) + for (const r of requests.values()) { + if (!r.url) continue; + assert.ok( + ![r.url, ...Object.values(r.headers ?? {})].some((value) => + [...remoteTokens].some((token) => String(value).includes(token)), + ), + "a request carried a remote token", + ); + + if (!/\/ask(\/|\?|$)/.test(new URL(r.url).pathname)) continue; + assert.equal( + new URL(r.url).origin, + origin, + "an Ask request past the local server", + ); + asks.push(r); + } + + assert.ok(asks.length > 0, "no Ask request was recorded"); + + return asks; +} diff --git a/apps/review-desktop/scripts/e2e/journeys/remote-host.mjs b/apps/review-desktop/scripts/e2e/journeys/remote-host.mjs new file mode 100644 index 000000000..9a35f595c --- /dev/null +++ b/apps/review-desktop/scripts/e2e/journeys/remote-host.mjs @@ -0,0 +1,734 @@ +/** A review server on another machine, reached over SSH: a Docker container from remote/remote.mjs, added in Settings as a user would. */ +import assert from "node:assert/strict"; +import { execFile, spawn } from "node:child_process"; +import path from "node:path"; +import { promisify } from "node:util"; + +import { createReview, openHome, openSettings } from "../harness.mjs"; + +const exec = promisify(execFile); + +const remoteScript = path.join(import.meta.dirname, "../remote/remote.mjs"); + +const runId = process.env.WB_TEST_RUN ?? `e2e${Date.now().toString(36)}`; + +export const runDir = `/tmp/wbt.${runId}`; + +export const prepared = process.env.REVIEW_E2E_REMOTE_HOST; + +export const alias = `wb-test-${prepared ?? "a"}`; + +const title = "Remote order"; + +export const name = "remote-host"; + +export const phase = 2; + +export const options = { + settings: { + "review.experimental.remoteHosts.enabled": true, + "review.experimental.structuralDiff.enabled": true, + }, + env: { DEV_FAST_REVIEW_SSH_CONFIG: `${runDir}/ssh_config` }, +}; + +export const createRemoteReview = String.raw` +set -e +field() { node -pe "JSON.parse(require('fs').readFileSync(0, 'utf8')).$1"; } +rm -rf ~/wbrepo +git init -q -b main ~/wbrepo +cd ~/wbrepo +git config user.email e2e@example.invalid +git config user.name e2e +printf 'export function one() {\n return 1;\n}\n' > f.ts +git add f.ts +git commit -qm one +printf 'export function one() {\n return 2;\n}\n\nexport function two() {\n return one() + 1;\n}\n' > f.ts +git commit -qam two +id=$(whiteboard api session_create "{\"title\":\"$1\",\"open\":false,\"target\":{\"kind\":\"commits\",\"repositoryPath\":\"$HOME/wbrepo\",\"base\":\"HEAD~1\",\"head\":\"HEAD\"}}" | field sessionId) +whiteboard api session_edit "{\"sessionId\":\"$id\",\"edit\":{\"type\":\"insert\",\"content\":{\"type\":\"markdown\",\"markdown\":\"One now returns **two**.\"}}}" >/dev/null +whiteboard api session_edit "{\"sessionId\":\"$id\",\"edit\":{\"type\":\"insert\",\"content\":{\"type\":\"code_peek\",\"source\":\"head/f.ts#L5-L7\"}}}" >/dev/null +echo "$id" +pwd +`; + +export async function remote(...args) { + return ( + await exec(process.execPath, [remoteScript, ...args], { + env: { ...process.env, WB_TEST_RUN: runId }, + maxBuffer: 16 * 1024 * 1024, + }) + ).stdout.trim(); +} + +export function onRemote(command, input = "") { + return new Promise((resolve, reject) => { + const child = spawn( + "ssh", + ["-F", `${runDir}/ssh_config`, "-o", "BatchMode=yes", alias, command], + { stdio: ["pipe", "pipe", "pipe"] }, + ); + + let stdout = ""; + let stderr = ""; + + child.stdout.on("data", (chunk) => (stdout += chunk)); + child.stderr.on("data", (chunk) => (stderr += chunk)); + child.once("error", reject); + child.once("close", (code) => + code === 0 + ? resolve(stdout.trim()) + : reject(new Error(`${command} on ${alias}: ${code}: ${stderr}`)), + ); + child.stdin.end(input); + }); +} + +const remoteApi = (tool, input) => + onRemote(`whiteboard api ${tool} -`, JSON.stringify(input)); + +export async function remoteToken() { + const out = (await onRemote("whiteboard remote attach --json")).split("\n"); + + let token; + + try { + token = JSON.parse(out[out.indexOf("WHITEBOARD-REMOTE-BEGIN") + 1]).token; + } catch { + throw new Error("whiteboard remote attach printed no attach line"); + } + + assert.ok( + /^[\w-]+$/.test(String(token ?? "")), + "remote attach gave no token", + ); + + return token; +} + +export async function recordRequests(page, streamed) { + const cdp = await page.context().newCDPSession(page); + const requests = new Map(); + const entry = (id) => requests.get(id) ?? requests.set(id, { id }).get(id); + + cdp.on("Network.requestWillBeSent", ({ requestId, request }) => + Object.assign(entry(requestId), { + url: request.url, + method: request.method, + headers: { ...entry(requestId).headers, ...request.headers }, + }), + ); + cdp.on("Network.requestWillBeSentExtraInfo", ({ requestId, headers }) => + Object.assign(entry(requestId), { + headers: { ...entry(requestId).headers, ...headers }, + }), + ); + cdp.on("Network.responseReceived", ({ requestId, response }) => { + Object.assign(entry(requestId), { status: response.status }); + + if (streamed.test(response.url)) + cdp + .send("Network.streamResourceContent", { requestId }) + .then(({ bufferedData }) => { + entry(requestId).received = Buffer.concat([ + Buffer.from(bufferedData, "base64"), + entry(requestId).received ?? Buffer.alloc(0), + ]); + }) + .catch(() => {}); + }); + cdp.on("Network.dataReceived", ({ requestId, data }) => { + if (data) + entry(requestId).received = Buffer.concat([ + entry(requestId).received ?? Buffer.alloc(0), + Buffer.from(data, "base64"), + ]); + }); + cdp.on("Network.loadingFailed", ({ requestId, errorText, canceled }) => + Object.assign(entry(requestId), { failed: errorText, canceled }), + ); + await cdp.send("Network.enable"); + + return requests; +} + +/** Every other workbench window than `ctx.page`: the Source windows. */ +export function sourcePages(ctx) { + return ctx.browser + .contexts() + .flatMap((context) => context.pages()) + .filter((page) => page !== ctx.page && !page.isClosed()); +} + +/** Pushes every `navigator` answer `page` receives, with its body, to `answers`. */ +export async function recordNavigator(page, answers) { + const cdp = await page.context().newCDPSession(page); + const urls = new Map(); + + cdp.on("Network.responseReceived", ({ requestId, response }) => { + if (/\/reviews-api\/[^/]+\/navigator(\?|$)/.test(response.url)) + urls.set(requestId, { url: response.url, status: response.status }); + }); + cdp.on("Network.loadingFinished", ({ requestId }) => { + const answer = urls.get(requestId); + + if (answer) + cdp + .send("Network.getResponseBody", { requestId }) + .then(({ body }) => answers.push({ ...answer, body })) + .catch((error) => answers.push({ ...answer, error: error.message })); + }); + await cdp.send("Network.enable"); +} + +/** Asserts each 200 `navigator` answer named URIs on `authority` and no host path; returns how many there were. */ +export function assertUriAnswers(answers, authority) { + const bodies = answers.flatMap((answer) => + answer.status === 200 ? [JSON.parse(answer.body)] : [], + ); + + assert.ok(bodies.length > 0, "no navigator answer was recorded"); + + for (const body of bodies) { + assert.deepEqual( + Object.keys(body).filter((key) => /Path$/.test(key)), + [], + JSON.stringify(body), + ); + assert.equal(body.remoteAuthority, authority); + assert.ok(body.workspaceUri.startsWith(`vscode-remote://${authority}/`)); + assert.ok( + body.emptySide === true || + body.fileUri.startsWith(`vscode-remote://${authority}/`), + JSON.stringify(body), + ); + } + + return bodies.length; +} + +const DESKTOP_ROUTE = + /^\/(app|control|crash-reports|diffr-config|install|preferences|remote-hosts|telemetry|tutorial)(\/|$)|^\/reviews-api\/[^/]+\/telemetry\//; + +export async function run(ctx) { + const { page, until } = ctx; + + // A packaged build ignores DEV_FAST_REVIEW_SSH_CONFIG, so its ssh would read the user's configuration. + if (ctx.report.mode === "packaged") + throw new Error("skip: remote-host runs in development mode only"); + + if (prepared === undefined) + try { + await exec("docker", ["info", "--format", "{{.ServerVersion}}"]); + } catch (error) { + throw new Error( + `skip: remote-host needs Docker for its SSH server (${error.message.split("\n")[0]})`, + ); + } + + try { + await journey(ctx, page, until); + } finally { + if (prepared === undefined) + await remote("down", "--all").catch((error) => + console.error(`[remote-host] down --all: ${error.message}`), + ); + else await closeDesktop(ctx); + } +} + +export async function closeDesktop(ctx) { + const session = await ctx.browser.newBrowserCDPSession().catch(() => null); + + await Promise.race([ + session?.send("Browser.close").catch(() => {}), + new Promise((resolve) => setTimeout(resolve, 2000)), + ]); + + for (let i = 0; i < 40 && (await desktopSsh()).length; i++) + await new Promise((resolve) => setTimeout(resolve, 250)); + + for (const pid of await desktopSsh()) process.kill(pid, "SIGTERM"); +} + +async function journey(ctx, page, until) { + const requests = await recordRequests(page, /\/structural-diff(\?|$)/); + const remoteTokens = new Set(); + const timeOrigin = await page.evaluate(() => performance.timeOrigin); + const timings = {}; + + const hostState = async () => + (await ctx.apiOk("/remote-hosts")).find((host) => host.alias === alias); + + const waitState = (state, label, timeout) => + until( + async () => (await hostState())?.state === state, + `${alias} ${label}`, + timeout, + ); + + if (prepared === undefined) { + await remote("up", "a"); + await remote("install", "a"); + } + + const [reviewId, repoPath] = ( + await onRemote(`bash -s -- '${title}'`, createRemoteReview) + ).split("\n"); + + assert.match(reviewId, /^[0-9a-f-]{36}$/, "the remote review's id"); + remoteTokens.add(await remoteToken()); + ctx.check( + `${alias}${prepared ? " (prepared)" : ", a container with sshd and this checkout's package,"} holds a review`, + ); + + // 1. Settings, as a user adds a host. + let settings = await openSettings(ctx); + let section = settings.getByRole("region", { name: "Remote hosts" }); + + await until( + async () => + ( + await section + .locator("#remote-host-suggestions option") + .evaluateAll((options) => options.map((o) => o.value)) + ).includes(alias), + `${alias} among the suggestions from the test ssh_config`, + ); + await section.getByLabel("SSH alias").fill(alias); + + const added = Date.now(); + + await section.getByRole("button", { name: "Add", exact: true }).click(); + await until( + async () => (await hostState())?.state === "online", + `${alias} online`, + 60000, + ); + timings.addToOnline = Date.now() - added; + + const hostRow = () => + section.locator("[data-remote-host]").filter({ hasText: alias }); + + await until( + async () => /\bonline\b/.test(await hostRow().innerText()), + "the Settings row to read online", + ); + ctx.check( + `1. Settings added ${alias}; online ${timings.addToOnline} ms after Add`, + ); + + // 2. Home lists the remote review under the host's label. + await openHome(ctx); + + const rows = page + .locator("main.review-home") + .getByRole("region", { name: "Sessions", exact: true }) + .locator("tbody tr"); + + const row = (text) => rows.filter({ hasText: text }); + + await row(title).waitFor({ timeout: 60000 }); + assert.ok((await row(title).innerText()).includes(`${alias}: wbrepo`)); + assert.equal(await row(title).getAttribute("data-unavailable"), null); + ctx.check(`2. Home lists the remote review as ${alias}: wbrepo`); + + // 3. Open it: the document, its code peek, the Diff view and the structural diff. + const canvas = page.locator(".review-canvas-root [data-review-api]"); + const peek = canvas.locator('[data-review-inline-editor="f.ts"]'); + const opened = Date.now(); + + await row(title).getByTitle(title, { exact: true }).click(); + await canvas + .getByRole("heading", { name: title }) + .waitFor({ timeout: 60000 }); + await canvas.getByText("One now returns").waitFor(); + timings.openReview = Date.now() - opened; + await until( + async () => (await lines(peek)).includes("return one() + 1;"), + "the code peek's lines from the remote", + ); + timings.codePeek = Date.now() - opened; + ctx.check( + `3a. document ${timings.openReview} ms and code peek ${timings.codePeek} ms after the click in Home`, + ); + + const view = (label) => + page.locator(`[aria-label="Session views"] button[aria-label="${label}"]`); + + await view("Diff").click(); + await until( + async () => + (await page.locator(".review-path-label").allInnerTexts()).some((t) => + t.includes("f.ts"), + ) && (await lines(page)).includes("return 2;"), + "the Diff view to show f.ts", + ); + + const streamEvents = (r) => + (r.received?.toString() ?? "") + .split("\n") + .filter((line) => line.startsWith("{")) + .flatMap((line) => { + try { + return [JSON.parse(line)]; + } catch { + return []; + } + }); + + const reads = () => + [...requests.values()].filter( + (r) => + r.url?.includes(`/reviews-api/${reviewId}/structural-diff`) && + r.status === 200, + ); + + const events = await until(() => { + const read = reads().find((r) => + streamEvents(r).some((event) => event.type === "complete"), + ); + + if (read) return streamEvents(read); + + throw new Error( + JSON.stringify(reads().map((r) => streamEvents(r).map((e) => e.type))), + ); + }, "a complete structural diff in what the page received"); + + assert.ok( + events.some((event) => event.type === "file"), + `no file event in the structural diff: ${events.map((e) => e.type)}`, + ); + assert.deepEqual( + reads() + .flatMap(streamEvents) + .filter((event) => event.type === "error"), + [], + "structural diff errors", + ); + + const streamStatus = page.locator(".review-structural-stream-status"); + + await until( + async () => + (await streamStatus.count()) > 0 && + (await streamStatus.evaluateAll((all) => + all.every((e) => e.hidden && e.textContent === ""), + )), + "the structural diff's stream status to be hidden and empty", + ); + ctx.check( + `3b. the Diff view shows f.ts; its structural diff streamed ${events.filter((e) => e.type === "file").length} file event(s), no error, and its status line cleared`, + ); + await view("Whiteboard").click(); + + // 4. An edit on the remote reaches the open tab without a reload. + await remoteApi("session_edit", { + sessionId: reviewId, + edit: { + type: "insert", + content: { type: "markdown", markdown: "Edited on the remote." }, + }, + }); + await canvas.getByText("Edited on the remote.").waitFor({ timeout: 30000 }); + assert.equal(await page.evaluate(() => performance.timeOrigin), timeOrigin); + ctx.check("4. an edit on the remote updates the open tab without a reload"); + + // 5. An agent on the remote opens a new review on this laptop. + const second = "Opened from the remote"; + await onRemote( + `cd ~/wbrepo && whiteboard api session_create -`, + JSON.stringify({ + title: second, + open: true, + target: { + kind: "commits", + repositoryPath: repoPath, + base: "HEAD~1", + head: "HEAD", + }, + }), + ); + + const tab = (text) => + page.locator(".tabs-container .tab").filter({ hasText: text }); + + await tab(second).waitFor({ timeout: 30000 }); + await canvas.getByRole("heading", { name: second }).waitFor(); + + const laptopOnly = canvas.locator( + 'button[aria-label="Share review"], button[aria-label="Shared review"], [aria-label="Session views"] button[aria-label="Trace"]', + ); + + const sourceTree = canvas.locator('button[aria-label="Source tree ↗"]'); + + await until( + async () => + (await laptopOnly.count()) === 0 && (await sourceTree.count()) === 1, + "the pushed tab with Source tree, without Share and Trace", + 10000, + ); + await view("Diff").click(); + await until( + async () => + (await page.locator(".review-path-label").allInnerTexts()).some((t) => + t.includes("f.ts"), + ), + "the pushed tab's Diff view to show f.ts", + ); + // A host with language features opens a review's files in a Source window on it. + await until( + () => + page + .locator(".review-multidiff-open-container") + .evaluateAll((all) => all.length > 0 && all.some((e) => !e.hidden)), + "Open file in the pushed tab's Diff view", + 10000, + ); + await view("Whiteboard").click(); + ctx.check( + "5. session_create with open: true on the remote opened a tab with Open file and Source tree, without Share or Trace", + ); + + // 6. The ssh master dies: the review says so, then recovers in the same page. + await tab(title).click(); + await canvas.getByRole("heading", { name: title }).waitFor(); + await page.evaluate(() => { + const seen = (window.__remoteHostBanner = []); + + new MutationObserver(() => { + for (const status of document.querySelectorAll( + ".review-canvas-root [role=status]", + )) + if ( + status.textContent.startsWith("Connection lost") && + !seen.includes(status.textContent) + ) + seen.push(status.textContent); + }).observe(document.body, { + subtree: true, + childList: true, + characterData: true, + }); + }); + + const master = await masterPid(); + + assert.ok(master, "no ssh master for the host"); + process.kill(master, "SIGKILL"); + await until( + () => page.evaluate(() => window.__remoteHostBanner.length > 0), + "the Connection lost banner", + 30000, + ); + await until( + async () => + (await hostState())?.state === "online" && (await masterPid()) !== master, + `${alias} online on a new master`, + 60000, + ); + await until( + async () => + (await canvas + .locator("[role=status]") + .filter({ hasText: "Connection lost" }) + .count()) === 0, + "the banner to clear", + ); + await remoteApi("session_edit", { + sessionId: reviewId, + edit: { + type: "insert", + content: { type: "markdown", markdown: "After the reconnect." }, + }, + }); + await canvas.getByText("After the reconnect.").waitFor({ timeout: 30000 }); + assert.equal(await page.evaluate(() => performance.timeOrigin), timeOrigin); + ctx.check( + `6. a killed ssh master showed "${await page.evaluate(() => window.__remoteHostBanner[0])}" and the review recovered in the same page`, + ); + + // 7. A hung server: offline within 15 s, and a laptop review still opens at once. + const serverPid = JSON.parse( + await onRemote("whiteboard server status --json"), + ).serverPid; + + const stopped = Date.now(); + + await onRemote(`kill -STOP ${serverPid}`); + await waitState("offline", "offline after SIGSTOP", 20000); + timings.offline = Date.now() - stopped; + assert.ok(timings.offline <= 15000, `offline after ${timings.offline} ms`); + + const laptopOpened = Date.now(); + + await createReview(ctx, { + title: "Laptop order", + blocks: [{ type: "markdown", markdown: "On this laptop." }], + }); + timings.laptopOpen = Date.now() - laptopOpened; + assert.ok( + timings.laptopOpen < 2000, + `a laptop review took ${timings.laptopOpen} ms`, + ); + await onRemote(`kill -CONT ${serverPid}`); + await waitState("online", "online after SIGCONT", 30000); + ctx.check( + `7. SIGSTOP: offline after ${timings.offline} ms; a laptop review opened in ${timings.laptopOpen} ms; SIGCONT: online`, + ); + + async function anotherVersion() { + await remote("install", "a", "--version", "0.0.2-e2e"); + await onRemote("whiteboard server stop"); + await waitState("incompatible", "incompatible", 60000); + remoteTokens.add(await remoteToken()); + const settings = await openSettings(ctx); + + section = settings.getByRole("region", { name: "Remote hosts" }); + await until( + async () => /incompatible/.test(await hostRow().innerText()), + "the Settings row to read incompatible", + ); + assert.match( + await hostRow().locator("code").innerText(), + /^npm install -g @dev\.fast\/whiteboard@\d+\.\d+\.\d+/, + ); + await openHome(ctx); + + for (const text of [title, second]) { + await row(text).waitFor(); + assert.equal(await row(text).getAttribute("data-unavailable"), ""); + assert.match(await row(text).innerText(), /incompatible/); + } + + ctx.check( + "8. another version made the host incompatible, Settings shows the install command, and its reviews stay listed", + ); + + await remote("install", "a"); + await onRemote("whiteboard server stop"); + await waitState("online", "online after the matching install", 60000); + remoteTokens.add(await remoteToken()); + await until( + async () => (await row(title).getAttribute("data-unavailable")) === null, + "the remote review to be available again", + ); + ctx.check( + "8b. installing the matching version and stopping the server brought the host back online", + ); + } + + // 8. Another version on the remote. Only on a host this journey installed: it never replaces a prepared host's package. + if (prepared === undefined) await anotherVersion(); + else { + console.error( + `[remote-host] step 8 skipped: ${alias} was prepared by hand, and the journey does not replace its package`, + ); + ctx.check("8. skipped on a prepared host: no package swap"); + } + + // 9. Removing the host takes its reviews out of Home. + settings = await openSettings(ctx); + section = settings.getByRole("region", { name: "Remote hosts" }); + await section.getByRole("button", { name: `Remove ${alias}` }).click(); + await section + .getByRole("group", { name: `Remove ${alias}` }) + .getByRole("button", { name: "Remove host" }) + .click(); + await until(async () => (await hostRow().count()) === 0, "the row to go"); + await openHome(ctx); + await row("Laptop order").waitFor(); + await until( + async () => (await row(title).count()) + (await row(second).count()) === 0, + "the remote reviews to leave Home", + ); + assert.ok( + (await ctx.apiOk("/reviews-api")).every((review) => !review.host), + "the list still has a remote entry", + ); + await until(async () => !(await masterPid()), "the master to end"); + ctx.check("9. removing the host in Settings took its reviews out of Home"); + + // 10. Throughout: every Desktop route answered, the UI spoke only to the local server, and no remote token reached it. + const seen = [...requests.values()].filter((r) => r.url); + const localServer = new URL(ctx.discovery.url).origin; + + const loopback = seen.filter((r) => + /^https?:\/\/(127\.0\.0\.1|localhost)[:/]/.test(r.url), + ); + + const desktopRoutes = loopback.filter((r) => + DESKTOP_ROUTE.test(new URL(r.url).pathname), + ); + + const failed = desktopRoutes.filter( + (r) => (r.status ?? 0) >= 400 || (r.failed && !r.canceled), + ); + + assert.ok(desktopRoutes.length > 0, "no Desktop route was requested"); + assert.deepEqual( + failed.map((r) => `${r.method} ${r.url} ${r.status ?? r.failed}`), + [], + "failed Desktop routes", + ); + assert.deepEqual( + loopback.flatMap((r) => + new URL(r.url).origin === localServer ? [] : [r.url], + ), + [], + "requests past the local server", + ); + + const header = (r) => + Object.entries(r.headers ?? {}).find( + ([key]) => key.toLowerCase() === "x-review-token", + )?.[1]; + + assert.ok( + loopback.some(header), + "no request carried a token, so the check proves nothing", + ); + assert.equal( + seen.filter((r) => + [r.url, ...Object.values(r.headers ?? {})].some((value) => + [...remoteTokens].some((token) => String(value).includes(token)), + ), + ).length, + 0, + "requests that carried a remote token", + ); + ctx.check( + `10. ${loopback.length} requests to the local server, ${desktopRoutes.length} of them Desktop routes: none failed, none went elsewhere, none carried either remote token`, + ); +} + +async function lines(scope) { + return (await scope.locator(".view-line").allInnerTexts()) + .join("\n") + .replaceAll("\u00a0", " "); +} + +async function desktopSshProcesses() { + const { stdout } = await exec("ps", ["-axo", "pid=,args="]); + + return stdout + .split("\n") + .map((line) => line.trim().match(/^(\d+) (.*)$/)) + .filter( + (match) => + match && + /(^|\/)ssh /.test(match[2]) && + match[2].includes(`-F ${runDir}/ssh_config -S `), + ) + .map((match) => [Number(match[1]), match[2]]); +} + +const desktopSsh = async () => + (await desktopSshProcesses()).map(([pid]) => pid); + +export async function masterPid() { + return (await desktopSshProcesses()).find( + ([, args]) => args.includes(" -M -N ") && args.endsWith(`-- ${alias}`), + )?.[0]; +} diff --git a/apps/review-desktop/scripts/e2e/journeys/remote-install.mjs b/apps/review-desktop/scripts/e2e/journeys/remote-install.mjs new file mode 100644 index 000000000..98ffdec3f --- /dev/null +++ b/apps/review-desktop/scripts/e2e/journeys/remote-install.mjs @@ -0,0 +1,965 @@ +/** Desktop installs itself on remote hosts it reaches over SSH: containers from remote/remote.mjs with nothing installed by hand, added in Settings as a user would. */ +import assert from "node:assert/strict"; +import { execFile, spawn } from "node:child_process"; +import { readFile, readdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { promisify } from "node:util"; + +import { openHome, openSettings, sourcePackage } from "../harness.mjs"; + +const exec = promisify(execFile); + +const remoteScript = path.join(import.meta.dirname, "../remote/remote.mjs"); + +const runId = process.env.WB_TEST_RUN ?? `e2e${Date.now().toString(36)}`; + +const runDir = `/tmp/wbt.${runId}`; + +const prepared = process.env.REVIEW_E2E_REMOTE_HOSTS?.split(",").filter(Boolean); + +const CONTAINERS = { + fresh: ["--platform", "linux/amd64", "--node", "none"], + node: ["--node", "24"], + sealed: ["--sealed", "--node", "none"], + old: ["--image", "debian:11", "--node", "none"], + arm: ["--platform", "linux/arm64", "--node", "none"], + fresh2: ["--platform", "linux/amd64", "--node", "none"], + deny: ["--node", "none"], +}; + +const alias = (name) => `wb-test-${name}`; + +const ROOT = "~/.dev/whiteboard-remote"; + +const WHITEBOARD = "~/.local/bin/whiteboard"; + +const CLAUDE_CODE = "@anthropic-ai/claude-code@2.1.286"; + +const title = "Installed remote order"; + +const MINUTE = 60_000; + +export const name = "remote-install"; + +export const phase = 2; + +export const options = { + settings: { "review.experimental.remoteHosts.enabled": true }, + env: { DEV_FAST_REVIEW_SSH_CONFIG: `${runDir}/ssh_config` }, +}; + +async function remote(args, timeout = 15 * MINUTE) { + return ( + await exec(process.execPath, [remoteScript, ...args], { + env: { ...process.env, WB_TEST_RUN: runId }, + maxBuffer: 16 * 1024 * 1024, + timeout, + killSignal: "SIGKILL", + }) + ).stdout.trim(); +} + +function onRemote(host, command, { input = "", timeout = 2 * MINUTE } = {}) { + return new Promise((resolve, reject) => { + const child = spawn( + "ssh", + ["-F", `${runDir}/ssh_config`, "-o", "BatchMode=yes", host, command], + { stdio: ["pipe", "pipe", "pipe"] }, + ); + + let stdout = ""; + let stderr = ""; + const timer = setTimeout(() => child.kill("SIGKILL"), timeout); + + child.stdout.on("data", (chunk) => (stdout += chunk)); + child.stderr.on("data", (chunk) => (stderr += chunk)); + child.once("error", reject); + child.once("close", (code, signal) => { + clearTimeout(timer); + + if (code === 0) resolve(stdout.trim()); + else + reject( + new Error( + `${command} on ${host}: ${signal ?? code}: ${stderr.slice(-2000)}`, + ), + ); + }); + child.stdin.end(input); + }); +} + +const remoteApi = async (host, tool, input) => + JSON.parse( + await onRemote(host, `${WHITEBOARD} api ${tool} -`, { + input: JSON.stringify(input), + }), + ); + +export async function run(ctx) { + // A packaged build ignores DEV_FAST_REVIEW_SSH_CONFIG, so its ssh would read the user's configuration. + if (ctx.report.mode === "packaged") + throw new Error("skip: remote-install runs in development mode only"); + + if (!prepared) + try { + await exec("docker", ["info", "--format", "{{.ServerVersion}}"], { + timeout: 30_000, + }); + } catch (error) { + throw new Error( + `skip: remote-install needs Docker for its SSH servers (${error.message.split("\n")[0]})`, + ); + } + + const manifestPath = path.join(sourcePackage, "package.json"); + const manifest = await readFile(manifestPath, "utf8"); + + try { + await (prepared ? preparedJourney(ctx) : journey(ctx, manifestPath)); + } finally { + if ((await readFile(manifestPath, "utf8")) !== manifest) + await writeFile(manifestPath, manifest); + + await closeDesktop(ctx); + + if (!prepared) + await remote(["down", "--all"], 5 * MINUTE).catch((error) => + console.error(`[remote-install] down --all: ${error.message}`), + ); + } +} + +async function closeDesktop(ctx) { + const session = await ctx.browser.newBrowserCDPSession().catch(() => null); + + await Promise.race([ + session?.send("Browser.close").catch(() => {}), + new Promise((resolve) => setTimeout(resolve, 2000)), + ]); + + for (let i = 0; i < 40 && (await desktopSsh()).length; i++) + await new Promise((resolve) => setTimeout(resolve, 250)); + + for (const [pid] of await desktopSsh()) process.kill(pid, "SIGTERM"); +} + +async function desktopSsh() { + const { stdout } = await exec("ps", ["-axo", "pid=,args="], { + timeout: 10_000, + }); + + return stdout + .split("\n") + .map((line) => line.trim().match(/^(\d+) (.*)$/)) + .filter( + (match) => + match && + /(^|\/)ssh /.test(match[2]) && + match[2].includes(`-F ${runDir}/ssh_config -S `), + ) + .map((match) => [Number(match[1]), match[2]]); +} + +const masterPid = async (host) => + (await desktopSsh()).find( + ([, args]) => args.includes(" -M -N ") && args.endsWith(`-- ${host}`), + )?.[0]; + +function watcher(ctx) { + const history = new Map(); + let stopped = false; + + const poll = (async () => { + while (!stopped) { + try { + for (const host of await ctx.apiOk("/remote-hosts")) { + const seen = history.get(host.alias) ?? []; + const last = seen.at(-1); + + if ( + last?.state !== host.state || + last?.detail !== host.detail || + last?.declined !== host.declined + ) + seen.push({ ...host, at: Date.now() }); + + history.set(host.alias, seen); + } + } catch { + } + + await new Promise((resolve) => setTimeout(resolve, 200)); + } + })(); + + return { + history: (host) => history.get(host) ?? [], + stop: async () => { + stopped = true; + await poll; + }, + }; +} + +async function watchPage(page) { + await page.evaluate(() => { + const ui = (window.__remoteInstall = { prompts: [], rows: {} }); + let open = false; + + const look = () => { + const title = [ + ...document.querySelectorAll(".quick-input-widget .quick-input-title"), + ].find( + (e) => + e.offsetParent !== null && + e.textContent.startsWith("Install Whiteboard on "), + ); + + if (title && !open) + ui.prompts.push({ title: title.textContent, at: Date.now() }); + + open = Boolean(title); + + for (const row of document.querySelectorAll("[data-remote-host]")) { + const [label, detail] = row.firstElementChild?.children ?? []; + const rows = (ui.rows[label?.textContent] ??= []); + const text = detail?.textContent; + + if (text && rows.at(-1) !== text) rows.push(text); + } + }; + + new MutationObserver(look).observe(document.body, { + subtree: true, + childList: true, + characterData: true, + attributes: true, + attributeFilter: ["style", "class", "aria-hidden"], + }); + }); +} + +async function mainLog(ctx) { + const logs = path.join(ctx.userData, "logs"); + + const files = (await readdir(logs, { recursive: true })).filter((file) => + file.endsWith("main.log"), + ); + + return ( + await Promise.all(files.map((file) => readFile(path.join(logs, file), "utf8"))) + ).join("\n"); +} + +const pageRecord = (ctx) => ctx.page.evaluate(() => window.__remoteInstall); + +async function journey(ctx, manifestPath) { + const { until } = ctx; + const timings = {}; + + for (const [host, flags] of Object.entries(CONTAINERS)) + await remote(["up", host, ...flags]); + ctx.check( + `0. containers up, Whiteboard on none: ${Object.entries(CONTAINERS) + .map(([host, flags]) => `${host} (${flags.join(" ")})`) + .join(", ")}`, + ); + + const watch = watcher(ctx); + + try { + await watchPage(ctx.page); + await steps(ctx, until, watch, timings, manifestPath); + } finally { + await watch.stop(); + } +} + +async function hostsSection(ctx) { + const settings = await openSettings(ctx); + + return settings.getByRole("region", { name: "Remote hosts" }); +} + +const hostRow = (section, host) => + section.locator("[data-remote-host]").filter({ hasText: host }); + +async function addHost(ctx, host) { + const section = await hostsSection(ctx); + + await section.getByLabel("SSH alias").fill(host); + await section.getByRole("button", { name: "Add", exact: true }).click(); + await hostRow(section, host).waitFor({ timeout: 30_000 }); + + return section; +} + +export async function removeHost(ctx, host, { uninstall = false } = {}) { + const section = await hostsSection(ctx); + + await section.getByRole("button", { name: `Remove ${host}` }).click(); + + const confirm = section.getByRole("group", { name: `Remove ${host}` }); + + if (uninstall) + await confirm + .getByRole("checkbox", { name: `Also remove Whiteboard from ${host}` }) + .check(); + + await confirm.getByRole("button", { name: "Remove host" }).click(); + await ctx.until( + async () => (await hostRow(section, host).count()) === 0, + `the ${host} row to go`, + 2 * MINUTE, + ); + + return section; +} + +async function answerPrompt(ctx, host, label) { + const widget = ctx.page.locator(".quick-input-widget"); + const heading = `Install Whiteboard on ${host}?`; + + await ctx.until( + async () => + (await widget.isVisible()) && + (await widget.locator(".quick-input-title").first().textContent()) === + heading, + `the install question for ${host}`, + 2 * MINUTE, + ); + + const text = await widget + .locator(".quick-input-description") + .first() + .textContent(); + + await widget + .locator(".monaco-list-row", { hasText: new RegExp(`^${label}$`) }) + .first() + .click({ timeout: 10_000 }); + + return { text, at: Date.now() }; +} + +async function reached(ctx, watch, host, states, label, timeout) { + return ctx.until( + () => watch.history(host).find((entry) => states.includes(entry.state)), + `${host} ${label}`, + timeout, + ); +} + +const installSteps = (watch, host) => + watch + .history(host) + .filter((entry) => entry.state === "installing") + .map((entry) => entry.detail); + +const STEP_ORDER = [ + "Preparing to install Whiteboard", + "Waiting for another install to finish", + "Installing Node 24", + "Installing the Whiteboard package", + "Checking the install", + "Installed; starting the server", +]; + +function assertStepOrder(details, label) { + const order = details.map((detail) => + STEP_ORDER.findIndex((step) => detail.startsWith(step)), + ); + + assert.ok(!order.includes(-1), `${label}: unknown step in ${details}`); + assert.deepEqual( + order, + [...order].sort((a, b) => a - b), + `${label}: steps out of order: ${details}`, + ); +} + +const promptsFor = async (ctx, host) => + (await pageRecord(ctx)).prompts.filter( + (prompt) => prompt.title === `Install Whiteboard on ${host}?`, + ); + +const marker = async (host, version) => + JSON.parse( + await onRemote( + host, + `cat ${ROOT}/versions/${version}/.whiteboard-install.json`, + ), + ); + +async function install(ctx, watch, host, { timeout = 10 * MINUTE } = {}) { + await addHost(ctx, host); + + const { text, at } = await answerPrompt(ctx, host, "Install"); + + const online = await reached( + ctx, + watch, + host, + ["online", "not-installed", "unsupported", "incompatible", "auth-failed"], + "online after the install", + timeout, + ); + + assert.equal( + online.state, + "online", + `${host}: ${online.state}: ${online.detail}`, + ); + + const steps = installSteps(watch, host); + + assertStepOrder(steps, host); + + return { text, steps, ms: online.at - at }; +} + +async function createRemoteReview(host, reviewTitle) { + const home = await onRemote(host, 'printf %s "$HOME"'); + + await onRemote( + host, + [ + "set -e", + "rm -rf ~/wbrepo", + "git init -q -b main ~/wbrepo", + "cd ~/wbrepo", + "git config user.email e2e@example.invalid", + "git config user.name e2e", + "printf 'export const one = 1;\\n' > f.ts", + "git add f.ts", + "git commit -qm one", + "printf 'export const one = 2;\\n' > f.ts", + "git commit -qam two", + ].join("\n"), + ); + + const created = await remoteApi(host, "session_create", { + title: reviewTitle, + open: false, + target: { + kind: "commits", + repositoryPath: `${home}/wbrepo`, + base: "HEAD~1", + head: "HEAD", + }, + }); + + assert.match(created.sessionId, /^[0-9a-f-]{36}$/, "the remote review's id"); +} + +async function homeRow(ctx, reviewTitle, timeout = MINUTE) { + await openHome(ctx); + + const row = ctx.page + .locator("main.review-home") + .getByRole("region", { name: "Sessions", exact: true }) + .locator("tbody tr") + .filter({ hasText: reviewTitle }); + + await row.waitFor({ timeout }); + + return row; +} + +async function targetOf(host, version) { + const { node } = await marker(host, version); + + return { + machine: await onRemote(host, "uname -m"), + node: await onRemote(host, `'${node}' -p process.arch`), + }; +} + +async function steps(ctx, until, watch, timings, manifestPath) { + const version = JSON.parse(await readFile(manifestPath, "utf8")).version; + + // 1. fresh: the question, its steps in Settings, online, and a review made there with the installed CLI listed in Home. + const fresh = alias("fresh"); + const first = await install(ctx, watch, fresh); + + timings.fresh = first.ms; + assert.match(first.text, /about 200 MB for Node 24/); + assert.doesNotMatch(first.text, /\/home\//, "the question names a path"); + + const settingsSteps = (await pageRecord(ctx)).rows[fresh].filter((text) => + text.startsWith("installing · "), + ); + + assert.ok( + settingsSteps.length >= 2, + `Settings showed ${settingsSteps.length} install step(s): ${settingsSteps}`, + ); + assertStepOrder( + settingsSteps.map((text) => text.slice("installing · ".length)), + "Settings", + ); + assert.ok( + first.steps.some((step) => step.startsWith("Installing Node 24")) && + first.steps.some((step) => + step.startsWith("Installing the Whiteboard package"), + ), + `fresh's steps: ${first.steps}`, + ); + await createRemoteReview(fresh, title); + + const listed = await homeRow(ctx, title); + + assert.ok((await listed.innerText()).includes(`${fresh}: wbrepo`)); + assert.equal(await listed.getAttribute("data-unavailable"), null); + + const freshTarget = await targetOf(fresh, version); + + assert.deepEqual(freshTarget, { machine: "x86_64", node: "x64" }); + ctx.check( + `1. ${fresh} (linux/amd64): the question, ${first.steps.length} steps from the API (${first.steps.join(" → ")}), ${settingsSteps.length} in Settings, online ${first.ms} ms after Install; a review written with ${WHITEBOARD} is in Home as ${fresh}: wbrepo`, + ); + + // 2. node: the host's own Node 24 runs Whiteboard; none is installed under whiteboard-remote/node. + const node = alias("node"); + const second = await install(ctx, watch, node); + + assert.doesNotMatch(second.text, /Node 24/); + assert.equal( + await onRemote(node, `test -e ${ROOT}/node && echo yes || echo no`), + "no", + ); + assert.equal((await marker(node, version)).node, "/usr/local/bin/node"); + assert.ok( + !second.steps.some((step) => step.startsWith("Installing Node")), + `node's steps: ${second.steps}`, + ); + ctx.check( + `2. ${node}: online ${second.ms} ms after Install, with /usr/local/bin/node; no ${ROOT}/node`, + ); + + // 3. sealed: no route out, so Node and the package are uploaded, and npm reaches the registry through Desktop's relay. + const sealed = alias("sealed"); + const third = await install(ctx, watch, sealed); + + timings.sealed = third.ms; + + for (const step of ["Installing Node 24", "Installing the Whiteboard package"]) + assert.ok( + third.steps.includes(`${step} (uploaded from this computer).`), + `sealed's steps: ${third.steps}`, + ); + await assert.rejects( + onRemote(sealed, "curl -sS -m 5 -o /dev/null https://registry.npmjs.org/"), + "the sealed host reaches the registry", + ); + assert.match( + await onRemote( + sealed, + "ls ~/.dev/whiteboard-remote/versions/*/node_modules/@dev.fast/diffr-linux-*/diffr", + ), + /diffr$/, + ); + ctx.check( + `3. ${sealed}: Node and the package uploaded from this computer, npm through the relay; online ${third.ms} ms after Install; the host still has no route out, and diffr came through the relay`, + ); + + // 4. old: glibc 2.31 is refused before anything is written, and no question is asked. + const old = alias("old"); + const newer = `find ~ -newer /tmp/wb-test-added | sort; test -e ${ROOT} && echo installed || true`; + + await onRemote(old, "touch /tmp/wb-test-added"); + const oldSection = await addHost(ctx, old); + + const refused = await reached( + ctx, + watch, + old, + ["unsupported", "online", "not-installed", "installing"], + "unsupported", + 2 * MINUTE, + ); + + assert.equal(refused.state, "unsupported", refused.detail); + assert.match( + refused.detail, + /glibc 2\.31; Whiteboard needs 2\.34 or newer/, + refused.detail, + ); + await until( + async () => + /unsupported · .*glibc 2\.31/.test( + await hostRow(oldSection, old).innerText(), + ), + "the Settings row to read unsupported", + 30_000, + ); + assert.equal(await onRemote(old, newer), "", "old was written to"); + assert.deepEqual(await promptsFor(ctx, old), []); + ctx.check( + `4. ${old} (debian:11): unsupported, "${refused.detail}"; no question; nothing in its home newer than just before it was added`, + ); + + // 5. arm: the other target. + const arm = alias("arm"); + const fifth = await install(ctx, watch, arm); + const armTarget = await targetOf(arm, version); + + assert.deepEqual(armTarget, { machine: "aarch64", node: "arm64" }); + ctx.check( + `5. ${arm} (linux/arm64): online ${fifth.ms} ms after Install, with Node for ${armTarget.node}`, + ); + + // 6. fresh2: the SSH connection dies during the package step; the reconnect finishes the install and leaves no .part. + const fresh2 = alias("fresh2"); + + await addHost(ctx, fresh2); + await answerPrompt(ctx, fresh2, "Install"); + + const packaging = await reached( + ctx, + watch, + fresh2, + ["installing", "online"], + "the package step", + 10 * MINUTE, + ).then(() => + until( + () => + watch + .history(fresh2) + .find((entry) => + entry.detail?.startsWith("Installing the Whiteboard package"), + ), + `${fresh2} installing the package`, + 10 * MINUTE, + ), + ); + + const master = await masterPid(fresh2); + + assert.ok(master, `no ssh master for ${fresh2}`); + process.kill(master, "SIGKILL"); + + const killed = Date.now(); + + const recovered = await until( + () => + watch + .history(fresh2) + .find((entry) => entry.at > killed && entry.state === "online"), + `${fresh2} online after the dropped connection`, + 10 * MINUTE, + ); + + const dropped = (await mainLog(ctx)) + .split("\n") + .find((line) => line.includes(`[Remote hosts] ${fresh2}: unreachable: `)); + + assert.ok(dropped, `no unreachable line for ${fresh2} in main.log`); + assert.notEqual(await masterPid(fresh2), master); + assert.equal( + await onRemote(fresh2, `find ${ROOT} -name '*.part' | head -n 5`), + "", + ); + assert.equal( + await onRemote(fresh2, `ls ${ROOT}/versions; ls -d ${ROOT}/install.lock* 2>/dev/null || true`), + version, + ); + assert.equal((await promptsFor(ctx, fresh2)).length, 1); + ctx.check( + `6. ${fresh2}: ssh master killed during "${packaging.detail}"; main.log: "${dropped.split("[Remote hosts] ")[1]}"; online ${recovered.at - killed} ms after the kill, asked once, no .part and no lock left`, + ); + + // 9. deny: declined, not installed, the manual command; a reconnect does not ask again. + const deny = alias("deny"); + const denySection = await addHost(ctx, deny); + + await answerPrompt(ctx, deny, "Don't install"); + + const declined = await reached( + ctx, + watch, + deny, + ["not-installed", "online", "installing"], + "not-installed", + 2 * MINUTE, + ); + + assert.equal(declined.state, "not-installed", declined.detail); + assert.equal(declined.declined, true); + assert.equal( + declined.installCommand, + `npm install -g @dev.fast/whiteboard@${version}`, + ); + await denySection + .getByRole("button", { name: `Install ${deny}` }) + .waitFor({ timeout: 30_000 }); + assert.equal( + await hostRow(denySection, deny).locator("code").innerText(), + declined.installCommand, + ); + + const nothing = `ls -d ${ROOT} ${WHITEBOARD} 2>/dev/null || true`; + + assert.equal(await onRemote(deny, nothing), ""); + + const connects = async () => + (await mainLog(ctx)).split(`${deny}: connecting, ssh pid`).length - 1; + + const before9 = await connects(); + + await denySection.getByRole("button", { name: `Retry ${deny}` }).click(); + await until( + async () => (await connects()) > before9, + `${deny} to connect again`, + MINUTE, + ); + await until( + async () => + (await ctx.apiOk("/remote-hosts")).find((host) => host.alias === deny) + ?.state === "not-installed", + `${deny} not-installed again`, + MINUTE, + ); + await new Promise((resolve) => setTimeout(resolve, 5000)); + assert.equal((await promptsFor(ctx, deny)).length, 1); + assert.equal(await onRemote(deny, nothing), ""); + ctx.check( + `9. ${deny}: declined → not-installed with "${declined.installCommand}" and Install in Settings; nothing written; Retry reconnected without asking again`, + ); + + for (const host of [sealed, old, arm, fresh2, deny, node]) + await removeHost(ctx, host); + + for (const host of ["sealed", "old", "arm", "fresh2", "deny"]) + await remote(["down", host], 5 * MINUTE); + + await onRemote(node, `${WHITEBOARD} server stop --json`); + await onRemote( + node, + `setsid nohup ${WHITEBOARD} server start --json > /tmp/user-server.log 2>&1 < /dev/null &`, + ); + + const userServer = JSON.parse( + await until( + () => + onRemote(node, `${WHITEBOARD} server status --json`).catch(() => null), + "the user's server on node", + MINUTE, + ), + ); + + assert.equal(userServer.version, version); + + const discovery = JSON.parse( + await onRemote(node, "cat ~/.dev/review-server/server.json"), + ); + + assert.equal(discovery.startedBy, "user"); + + const privateNode = `$(ls -d ${ROOT}/node/v*/bin | tail -n 1)`; + + await onRemote( + fresh, + `PATH="${privateNode}:$PATH" npm install -g --prefix ~/.local --no-audit --no-fund ${CLAUDE_CODE} >/dev/null && mkdir -p ~/.claude && ~/.local/bin/claude --version`, + { timeout: 10 * MINUTE }, + ); + + const freshServer = async () => + JSON.parse(await onRemote(fresh, `${WHITEBOARD} server status --json`)); + + const oldFreshServer = await freshServer(); + + // 7. The next version: Desktop's server reports what packages/review/package.json says, and its pack carries it. + const next = version.replace(/\d+$/, (patch) => String(Number(patch) + 1)); + const manifest = await readFile(manifestPath, "utf8"); + + await writeFile( + manifestPath, + manifest.replace(`"version": "${version}"`, `"version": "${next}"`), + ); + const sshBefore = (await desktopSsh()).map(([pid]) => pid); + const restarted = Date.now(); + + await ctx.restartDesktop(); + await watchPage(ctx.page); + await until( + async () => + !(await desktopSsh()).some(([pid]) => sshBefore.includes(pid)), + "the previous Desktop's ssh to end", + 30_000, + ); + + assert.equal((await ctx.apiOk("/health")).version, next); + + const upgraded = await until( + () => + watch + .history(fresh) + .find( + (entry) => + entry.at > restarted && + ["online", "incompatible", "not-installed"].includes(entry.state), + ), + `${fresh} on ${next}`, + 10 * MINUTE, + ); + + assert.equal(upgraded.state, "online", upgraded.detail); + assert.deepEqual(await promptsFor(ctx, fresh), []); + assert.equal( + await onRemote(fresh, `ls ${ROOT}/versions | sort -V | tr '\\n' ' '`), + `${version} ${next}`, + ); + + const newFreshServer = await freshServer(); + + assert.equal(newFreshServer.version, next); + assert.notEqual(newFreshServer.serverPid, oldFreshServer.serverPid); + assert.match( + await onRemote(fresh, `cat /proc/${newFreshServer.serverPid}/cmdline | tr '\\0' ' '`), + new RegExp(`/versions/${next.replaceAll(".", "\\.")}/`), + ); + await homeRow(ctx, title); + ctx.check( + `7. Desktop ${next} (packages/review/package.json bumped, Desktop restarted, none of its ${sshBefore.length} ssh processes left): ${fresh} installed ${next} beside ${version} without asking, its server restarted on ${next} (pid ${oldFreshServer.serverPid} → ${newFreshServer.serverPid}), and the step 1 review is still listed`, + ); + + // 8. A server the user started on node, at the old version: the newer Desktop leaves it and says so. + const readded = Date.now(); + + await addHost(ctx, node); + + const blocked = await until( + () => + watch + .history(node) + .find( + (entry) => + entry.at > readded && + ["incompatible", "online", "not-installed"].includes(entry.state), + ), + `${node} settled`, + 10 * MINUTE, + ); + + assert.equal(blocked.state, "incompatible", blocked.detail); + assert.match(blocked.detail, /started by a user is running on/); + assert.deepEqual(await promptsFor(ctx, node), []); + await onRemote(node, `kill -0 ${userServer.serverPid}`); + assert.equal( + ( + await onRemote(node, `cat /proc/${userServer.serverPid}/cmdline | tr '\\0' ' '`) + ).includes(`/versions/${version}/`), + true, + ); + ctx.check( + `8. ${node}: "${blocked.detail}"; the user's server (pid ${userServer.serverPid}, ${version}) still runs`, + ); + + // 10. fresh has Claude Code's configuration: Settings offers it, Connect connects it on the remote. + const section = await hostsSection(ctx); + const freshRow = hostRow(section, fresh); + + await until( + async () => + /Agents on wb-test-fresh: Claude Code/.test(await freshRow.innerText()), + "the agent offer for fresh", + 2 * MINUTE, + ); + await freshRow + .getByRole("button", { name: `Connect agents on ${fresh}` }) + .click(); + await until( + async () => + /Claude Code (is connected|was not connected)/.test( + await freshRow.innerText(), + ), + "Connect's result", + 5 * MINUTE, + ); + assert.match( + await freshRow.getByRole("status").innerText(), + /^Claude Code is connected on wb-test-fresh\.$/, + ); + assert.ok( + JSON.parse( + await onRemote(fresh, "cat ~/.claude/plugins/installed_plugins.json"), + ).plugins["whiteboard@devfast"], + "no whiteboard@devfast in installed_plugins.json", + ); + ctx.check( + `10. ${fresh}: Settings offered Claude Code (${CLAUDE_CODE}, ~/.claude present); Connect wrote whiteboard@devfast into ~/.claude/plugins/installed_plugins.json`, + ); + + // 11. Remove fresh with its Whiteboard: the install and the launcher go, the review store stays. + await removeHost(ctx, fresh, { uninstall: true }); + assert.equal( + await section.getByRole("alert").count(), + 0, + await section + .getByRole("alert") + .innerText() + .catch(() => ""), + ); + assert.equal(await onRemote(fresh, nothing), ""); + assert.match( + await onRemote( + fresh, + `cat ~/.dev/review-api.db ~/.dev/review-api.db-wal 2>/dev/null | grep -ac '${title}' || true`, + ), + /^[1-9]/, + ); + ctx.check( + `11. ${fresh} removed with "Also remove Whiteboard": no ${ROOT}, no ${WHITEBOARD}; ~/.dev/review-api.db still holds "${title}"`, + ); + + // 12. + ctx.check( + `12. Install → online: fresh ${timings.fresh} ms, sealed ${timings.sealed} ms`, + ); + console.error(`[remote-install] timings ${JSON.stringify(timings)}`); +} + +async function preparedJourney(ctx) { + const watch = watcher(ctx); + + try { + await watchPage(ctx.page); + + const timings = {}; + + for (const name of prepared) { + const host = alias(name); + + assert.equal( + await onRemote(host, `ls -d ${ROOT} ${WHITEBOARD} 2>/dev/null || true`), + "", + `${host} has seen Whiteboard`, + ); + + const installed = await install(ctx, watch, host, { + timeout: 20 * MINUTE, + }); + + timings[name] = installed.ms; + + const reviewTitle = `${title} on ${name}`; + + await createRemoteReview(host, reviewTitle); + assert.ok( + (await (await homeRow(ctx, reviewTitle)).innerText()).includes( + `${host}: wbrepo`, + ), + ); + ctx.check( + `${host} (${await onRemote(host, "uname -m")}): ${installed.steps.join(" → ")}; online ${installed.ms} ms after Install; its review is in Home`, + ); + } + + console.error(`[remote-install] timings ${JSON.stringify(timings)}`); + } finally { + await watch.stop(); + } +} diff --git a/apps/review-desktop/scripts/e2e/journeys/remote-lsp-csharp.mjs b/apps/review-desktop/scripts/e2e/journeys/remote-lsp-csharp.mjs new file mode 100644 index 000000000..baab25e60 --- /dev/null +++ b/apps/review-desktop/scripts/e2e/journeys/remote-lsp-csharp.mjs @@ -0,0 +1,15 @@ +/** The csharp group on an SSH host with its toolchain; see remote-lsp-languages.mjs. Its container image is large, so it runs only when named. */ +import { + remoteLspOptions, + runRemoteLspJourney, +} from "../remote-lsp-languages.mjs"; + +export const name = "remote-lsp-csharp"; + +export const phase = 2; + +export const manual = true; + +export const options = remoteLspOptions(); + +export const run = (ctx) => runRemoteLspJourney(ctx, "csharp"); diff --git a/apps/review-desktop/scripts/e2e/journeys/remote-lsp-rust.mjs b/apps/review-desktop/scripts/e2e/journeys/remote-lsp-rust.mjs new file mode 100644 index 000000000..1d83f3515 --- /dev/null +++ b/apps/review-desktop/scripts/e2e/journeys/remote-lsp-rust.mjs @@ -0,0 +1,15 @@ +/** The rust group on an SSH host with its toolchain; see remote-lsp-languages.mjs. Its container image is large, so it runs only when named. */ +import { + remoteLspOptions, + runRemoteLspJourney, +} from "../remote-lsp-languages.mjs"; + +export const name = "remote-lsp-rust"; + +export const phase = 2; + +export const manual = true; + +export const options = remoteLspOptions(); + +export const run = (ctx) => runRemoteLspJourney(ctx, "rust"); diff --git a/apps/review-desktop/scripts/e2e/journeys/remote-lsp-swift.mjs b/apps/review-desktop/scripts/e2e/journeys/remote-lsp-swift.mjs new file mode 100644 index 000000000..01a956312 --- /dev/null +++ b/apps/review-desktop/scripts/e2e/journeys/remote-lsp-swift.mjs @@ -0,0 +1,15 @@ +/** The swift group on an SSH host with its toolchain; see remote-lsp-languages.mjs. Its container image is large, so it runs only when named. */ +import { + remoteLspOptions, + runRemoteLspJourney, +} from "../remote-lsp-languages.mjs"; + +export const name = "remote-lsp-swift"; + +export const phase = 2; + +export const manual = true; + +export const options = remoteLspOptions(); + +export const run = (ctx) => runRemoteLspJourney(ctx, "swift"); diff --git a/apps/review-desktop/scripts/e2e/journeys/remote-lsp.mjs b/apps/review-desktop/scripts/e2e/journeys/remote-lsp.mjs new file mode 100644 index 000000000..a75b8538a --- /dev/null +++ b/apps/review-desktop/scripts/e2e/journeys/remote-lsp.mjs @@ -0,0 +1,1221 @@ +/** Language features for reviews on two SSH hosts and the laptop, in one window: containers from remote/remote.mjs, or two hosts prepared by hand. */ +import assert from "node:assert/strict"; +import { execFile, spawn } from "node:child_process"; +import { existsSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { cp, mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { promisify } from "node:util"; + +import { DEFAULT_REMOTE_RUNTIME } from "../../build-remote-runtime.mjs"; +import { + appRoot, + closeSourceWindow, + createReview, + openHome, + openSettings, + sleep, + sourceWindowFor, +} from "../harness.mjs"; + +const exec = promisify(execFile); + +const remoteScript = path.join(import.meta.dirname, "../remote/remote.mjs"); + +const runId = process.env.WB_TEST_RUN ?? `e2e${Date.now().toString(36)}`; + +export const runDir = `/tmp/wbt.${runId}`; + +const prepared = process.env.REVIEW_E2E_REMOTE_HOSTS?.split(","); + +const [nameA, nameB] = prepared ?? ["a", "b"]; + +const [aliasA, aliasB] = [`wb-test-${nameA}`, `wb-test-${nameB}`]; + +const proj = `${runDir}/proj`; + +const pyProj = `${runDir}/py`; + +const GRACE_SECONDS = 20; + +const overridesPath = path.join(appRoot, "code-oss/product.overrides.json"); + +export const name = "remote-lsp"; + +export const phase = 2; + +export const options = { + settings: { "review.experimental.remoteHosts.enabled": true }, + env: { DEV_FAST_REVIEW_SSH_CONFIG: `${runDir}/ssh_config` }, + beforeLaunch: stampDesktopCommit, +}; + +async function stampDesktopCommit(ctx) { + if (ctx.report.mode === "packaged") return; + + const { commit } = JSON.parse( + await readFile(path.join(DEFAULT_REMOTE_RUNTIME, "product.json"), "utf8"), + ); + + const stamp = `${JSON.stringify({ commit })}\n`; + + if (existsSync(overridesPath)) { + assert.equal( + readFileSync(overridesPath, "utf8"), + stamp, + `${overridesPath} exists with other content; move it away first`, + ); + + return; + } + + writeFileSync(overridesPath, stamp); + process.once("exit", () => rmSync(overridesPath, { force: true })); +} + +const typescriptFixture = String.raw` +set -e +field() { node -pe "JSON.parse(require('fs').readFileSync(0, 'utf8')).$1"; } +rm -rf "$1" +git init -q -b main "$1" +cd "$1" +git config user.email e2e@example.invalid +git config user.name e2e +printf '{"compilerOptions":{"strict":true,"target":"ES2022","module":"commonjs"}}\n' > tsconfig.json +printf 'export const answer = %s;\n' "$2" > a.ts +printf 'import { answer } from "./a";\n\nexport const total = 0;\n' > b.ts +git add . +git commit -qm one +printf 'import { answer } from "./a";\n\nexport const total = answer + 1;\n' > b.ts +git commit -qam two +whiteboard api session_create "{\"title\":\"$3\",\"open\":false,\"target\":{\"kind\":\"commits\",\"repositoryPath\":\"$1\",\"base\":\"HEAD~1\",\"head\":\"HEAD\"}}" | field sessionId +`; + +const pythonFixture = String.raw` +set -e +field() { node -pe "JSON.parse(require('fs').readFileSync(0, 'utf8')).$1"; } +rm -rf "$1" +git init -q -b main "$1" +cd "$1" +git config user.email e2e@example.invalid +git config user.name e2e +printf 'def f(x: int) -> int:\n return x\n' > m.py +git add . +git commit -qm one +printf 'def f(x: int) -> int:\n return x\n\n\nvalue = f(1)\n' > m.py +git commit -qam two +whiteboard api session_create "{\"title\":\"$2\",\"open\":false,\"target\":{\"kind\":\"commits\",\"repositoryPath\":\"$1\",\"base\":\"HEAD~1\",\"head\":\"HEAD\"}}" | field sessionId +`; + +const processTree = String.raw` +cat ~/.dev/whiteboard-remote/server/server.json +echo +ps -eo pid=,ppid=,rss=,args= +`; + +export function bounded(promise, ms, label) { + let timer; + + return Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error(`${label} did not finish within ${ms} ms`)), + ms, + ); + }), + ]).finally(() => clearTimeout(timer)); +} + +export async function remote(...args) { + return ( + await exec(process.execPath, [remoteScript, ...args], { + env: { ...process.env, WB_TEST_RUN: runId }, + maxBuffer: 16 * 1024 * 1024, + timeout: 900000, + }) + ).stdout.trim(); +} + +export function onRemote(alias, command, input = "", timeout = 300000) { + return new Promise((resolve, reject) => { + const child = spawn( + "ssh", + ["-F", `${runDir}/ssh_config`, "-o", "BatchMode=yes", alias, command], + { stdio: ["pipe", "pipe", "pipe"], timeout }, + ); + + let stdout = ""; + let stderr = ""; + + child.stdout.on("data", (chunk) => (stdout += chunk)); + child.stderr.on("data", (chunk) => (stderr += chunk)); + child.once("error", reject); + child.once("close", (code) => + code === 0 + ? resolve(stdout.trim()) + : reject( + new Error( + `${command.split("\n")[0]} on ${alias}: ${code}: ${stderr}`, + ), + ), + ); + child.stdin.end(input); + }); +} + +function check(ctx, text) { + console.error(`[remote-lsp] ${new Date().toISOString()} ${text}`); + ctx.check(text); +} + +async function attach(alias, env = "") { + const out = (await onRemote(alias, `${env} whiteboard remote attach --json`)) + .split("\n") + .map((line) => line.trim()); + + let parsed; + + try { + parsed = JSON.parse(out[out.indexOf("WHITEBOARD-REMOTE-BEGIN") + 1]); + } catch { + throw new Error( + `whiteboard remote attach on ${alias} printed no attach line`, + ); + } + + assert.ok( + parsed.languageServer, + `${alias} has no VS Code server: ${parsed.languageServerDetail}`, + ); + + return parsed.serverId; +} + +export async function extensionHosts(alias) { + return Number(await onRemote(alias, `pgrep -fc '[e]xtensionHost' || true`)); +} + +export async function remoteMemory(alias) { + const [serverJson, , ...lines] = (await onRemote(alias, processTree)).split( + "\n", + ); + + const root = JSON.parse(serverJson).pid; + + const processes = lines + .map((line) => line.trim().match(/^(\d+)\s+(\d+)\s+(\d+)\s+(.*)$/)) + .filter(Boolean) + .map(([, pid, ppid, rss, args]) => ({ + pid: Number(pid), + ppid: Number(ppid), + rss: Number(rss), + args, + })); + + const tree = processes.filter((p) => p.pid === root); + + for (let i = 0; i < tree.length; i++) + tree.push(...processes.filter((p) => p.ppid === tree[i].pid)); + + const mb = (list) => + Math.round(list.reduce((sum, p) => sum + p.rss, 0) / 1024); + + const hosts = tree.filter((p) => p.args.includes("--type=extensionHost")); + + return { + server: mb(tree.slice(0, 1)), + extensionHost: mb(hosts), + languageServers: mb(tree.slice(1).filter((p) => !hosts.includes(p))), + total: mb(tree), + processes: tree.length, + }; +} + +async function freeze(alias, name, frozen) { + if (prepared === undefined) return remote(frozen ? "pause" : "resume", name); + + await onRemote( + alias, + `pkill -${frozen ? "STOP" : "CONT"} -f '[v]scode-server/out/server-main.js|[w]hiteboard.*server'`, + ); +} + +export async function run(ctx) { + if (ctx.report.mode === "packaged") + throw new Error("skip: remote-lsp runs in development mode only"); + + if (prepared === undefined) + try { + await exec("docker", ["info", "--format", "{{.ServerVersion}}"]); + } catch (error) { + throw new Error( + `skip: remote-lsp needs Docker for its SSH servers (${error.message.split("\n")[0]})`, + ); + } + else + assert.equal(prepared.length, 2, "REVIEW_E2E_REMOTE_HOSTS names two hosts"); + + try { + await journey(ctx); + } finally { + rmSync(overridesPath, { force: true }); + + if (prepared === undefined) + await remote("down", "--all").catch((error) => + console.error(`[remote-lsp] down --all: ${error.message}`), + ); + else { + await closeDesktop(ctx); + await rm(proj, { recursive: true, force: true }); + await rm(`${runDir}/runtime-other`, { recursive: true, force: true }); + } + } +} + +async function journey(ctx) { + const { page, until } = ctx; + const timings = {}; + const memory = {}; + + if (prepared === undefined) { + for (const name of [nameA, nameB]) await remote("up", name); + + for (const name of [nameA, nameB]) await remote("install", name); + } + + let started = Date.now(); + + await Promise.all( + [aliasA, aliasB].map((alias) => + onRemote(alias, "whiteboard remote extensions ensure --json"), + ), + ); + timings.extensionsDownload = Date.now() - started; + + const serverB = await attach( + aliasB, + `DEV_FAST_REVIEW_REMOTE_RECONNECTION_GRACE_SECONDS=${GRACE_SECONDS}`, + ); + + const authorityB = `whiteboard+${serverB.toLowerCase()}`; + + const serverA = await attach(aliasA); + + const title = { + laptop: "Answer on the laptop", + a: `Answer on ${aliasA}`, + b: `Answer on ${aliasB}`, + py: `Python on ${aliasA}`, + }; + + const reviewA = await onRemote( + aliasA, + `bash -s -- '${proj}' 42 '${title.a}'`, + typescriptFixture, + ); + + const reviewB = await onRemote( + aliasB, + `bash -s -- '${proj}' 99 '${title.b}'`, + typescriptFixture, + ); + + const reviewPy = await onRemote( + aliasA, + `bash -s -- '${pyProj}' '${title.py}'`, + pythonFixture, + ); + + for (const id of [reviewA, reviewB, reviewPy]) + assert.match(id, /^[0-9a-f-]{36}$/, "a remote review's id"); + + const commits = await laptopFixture(); + + check( + ctx, + `${aliasA} and ${aliasB}${prepared ? " (prepared)" : ", containers with this checkout's package,"} hold reviews of ${proj} with answer = 42 and 99, and ${aliasA} a Python review; their extensions downloaded in ${timings.extensionsDownload} ms`, + ); + + const hostState = async (alias) => + (await ctx.apiOk("/remote-hosts")).find((host) => host.alias === alias); + + // 1. Add both hosts in Settings; both report language features. + let settings = await openSettings(ctx); + let section = settings.getByRole("region", { name: "Remote hosts" }); + const added = {}; + + const hostRow = (alias) => + section.locator("[data-remote-host]").filter({ hasText: alias }); + + for (const alias of [aliasA, aliasB]) { + await section.getByLabel("SSH alias").fill(alias); + added[alias] = Date.now(); + await section.getByRole("button", { name: "Add", exact: true }).click(); + await hostRow(alias).waitFor(); + } + + await Promise.all( + [aliasA, aliasB].map(async (alias) => { + await until( + async () => { + const state = await hostState(alias); + + return state?.state === "online" && state.languageFeatures === true; + }, + `${alias} online with language features`, + 120000, + ); + timings[`available ${alias}`] = Date.now() - added[alias]; + await until( + async () => + (await hostRow(alias).innerText()).includes( + "Language features: available", + ), + `the Settings row of ${alias} to say language features are available`, + ); + }), + ); + + check( + ctx, + `1. Settings added ${aliasA} and ${aliasB}; both online with "Language features: available" ${timings[`available ${aliasA}`]} and ${timings[`available ${aliasB}`]} ms after Add`, + ); + + // 2. A review on each machine, open at the same time. + const laptop = await createReview(ctx, { + title: title.laptop, + repoPath: proj, + base: commits.base, + head: commits.head, + blocks: [{ type: "markdown", markdown: "The laptop's answer." }], + }); + + await openHome(ctx); + + const rows = page + .locator("main.review-home") + .getByRole("region", { name: "Sessions", exact: true }) + .locator("tbody tr"); + + for (const text of [title.a, title.py, title.b]) { + const row = rows.filter({ hasText: text }); + + await row.waitFor({ timeout: 60000 }); + await row.getByTitle(text, { exact: true }).click(); + await visibleCanvas(page) + .getByRole("heading", { name: text }) + .waitFor({ timeout: 60000 }); + await openHome(ctx); + } + + for (const text of Object.values(title)) await tab(page, text).waitFor(); + check( + ctx, + "2. the laptop's review, two on A and one on B are open in tabs at once", + ); + + const timeOrigin = await page.evaluate(() => performance.timeOrigin); + + const machines = [ + { key: "laptop", alias: "the laptop", review: laptop.reviewId, answer: 1 }, + { key: "a", alias: aliasA, review: reviewA, answer: 42, serverId: serverA }, + { key: "b", alias: aliasB, review: reviewB, answer: 99, serverId: serverB }, + ]; + + const answer = (n) => new RegExp(`\\banswer: ${n}\\b`); + + const hover = (m) => + windowCall(ctx, "query", m.review, "b.ts", 3, "answer", m.uri); + + const answers = async (m) => { + const deadline = Date.now() + 20000; + let result; + + do { + result = await hover(m); + + if (answer(m.answer).test(result.text)) return result; + await sleep(500); + } while (Date.now() < deadline); + + throw new Error(`${m.alias} stopped answering: ${JSON.stringify(result)}`); + }; + + // 3. Hover `answer` in each; go to definition lands in that machine's a.ts. + const warm = {}; + let countsBefore; + + for (const m of machines) { + if (m.key === "a") { + countsBefore = await windowCall(ctx, "counts", laptop.reviewId, "b.ts"); + assert.deepEqual( + [await extensionHosts(aliasA), await extensionHosts(aliasB)], + [0, 0], + "an extension host on a remote before its review's Diff view", + ); + } + + const diffShown = Date.now(); + + await showDiff(ctx, title[m.key], "b.ts"); + + const point = await windowCall(ctx, "point", m.review, "b.ts", 3, "answer"); + + m.uri = point.uri; + const shown = await pointerHover(page, point, answer(m.answer), 60000); + + assert.ok( + shown.ms !== null, + `no "answer: ${m.answer}" hover on ${m.alias}: ${JSON.stringify(shown.seen)}`, + ); + timings[`first hover ${m.key}`] = shown.ms; + timings[`diff to hover ${m.key}`] = Date.now() - diffShown; + + if (m.key !== "laptop") + timings[`connect to first hover ${m.key}`] = Date.now() - added[m.alias]; + + warm[m.key] = []; + + for (let i = 0; i < 3; i++) { + const result = await hover(m); + + assert.match( + result.text, + answer(m.answer), + `warm hover on ${m.alias}: ${JSON.stringify(result)}`, + ); + warm[m.key].push(result.ms); + } + + const opened = await windowCall(ctx, "goto", m.review, "b.ts", 3, "answer"); + + if (m.key === "laptop") { + const source = await sourceWindowFor(ctx, "a.ts"); + + assert.match(await lines(source), /export const answer = 1;/); + await closeSourceWindow(source); + } else { + assert.match( + opened.active?.resource ?? "", + new RegExp( + `^vscode-remote://whiteboard%2B${m.serverId.toLowerCase()}/.*/a\\.ts$`, + ), + JSON.stringify(opened), + ); + assert.match(opened.active.text, new RegExp(`answer = ${m.answer};`)); + assert.ok(opened.active.readonly, "the host's a.ts is not read-only"); + assert.ok(opened.active.label.startsWith(`${m.alias}: `)); + await windowCall(ctx, "closeModal"); + } + + check( + ctx, + `3${m.key}. ${m.alias}: the hover read "${shown.text.match(answer(m.answer))[0]}" ${shown.ms} ms after the pointer arrived; go to definition opened ${m.key === "laptop" ? "a Source window on" : "read-only"} its a.ts`, + ); + } + + // 4. A Python review on A: ty answers with a type. + await showDiff(ctx, title.py, "m.py"); + + const pyPoint = await windowCall(ctx, "point", reviewPy, "m.py", 5, "value"); + const pyHover = await pointerHover(page, pyPoint, /\bint\b/, 120000); + + assert.ok( + pyHover.ms !== null, + `no type in the Python hover: ${JSON.stringify(pyHover.seen)}`, + ); + timings["first hover python"] = pyHover.ms; + check( + ctx, + `4. ${aliasA}'s Python review: the hover on value read "${pyHover.text.slice(0, 80)}" after ${pyHover.ms} ms`, + ); + + // 5. The laptop's own providers for its file are the same after the remotes connected. + await showDiff(ctx, title.laptop, "b.ts"); + await answers(machines[0]); + + const countsAfter = await windowCall(ctx, "counts", laptop.reviewId, "b.ts"); + + assert.deepEqual(countsAfter, countsBefore); + assert.ok(countsBefore.file, "no laptop model for b.ts"); + check( + ctx, + `5. the laptop's b.ts has ${countsBefore.file.hover} hover and ${countsBefore.file.definition} definition provider(s) in the window, and its review model ${countsBefore.review.hover}/${countsBefore.review.definition}, before and after both remotes connected`, + ); + + for (const alias of [aliasA, aliasB]) + memory[alias] = await remoteMemory(alias); + + const logLines = (pattern) => + windowLog(ctx).then((log) => log.match(pattern) ?? []); + + const failedB = new RegExp( + `\\[Remote language\\] ${authorityB.replace("+", "\\+")}: .* failed; connecting again`, + "g", + ); + + // 6. B's tunnel ends for longer than its reconnection lasts; A and the laptop keep answering; B comes back without a reload. + const [a, b] = [machines[1], machines[2]]; + await showDiff(ctx, title.b, "b.ts"); + + const failuresBefore = (await logLines(failedB)).length; + const master = await masterPid(aliasB); + + assert.ok(master, `no ssh master for ${aliasB}`); + + const down = Date.now(); + + await freeze(aliasB, nameB, true); + process.kill(master, "SIGKILL"); + + const whileDown = []; + + try { + while (Date.now() - down < (GRACE_SECONDS + 25) * 1000) { + const [fromA, fromLaptop, fromB] = await Promise.all([ + answers(a), + answers(machines[0]), + hover(b), + ]); + + assert.doesNotMatch(fromB.text, answer(99)); + assert.ok(fromB.ms <= 5500, `B's hover took ${fromB.ms} ms while down`); + whileDown.push([fromA.ms, fromLaptop.ms, fromB.ms]); + await sleep(3000); + } + } finally { + await freeze(aliasB, nameB, false); + } + + assert.ok( + (await logLines(failedB)).length > failuresBefore, + "B's session did not fail for good while its tunnel was down", + ); + + const restored = Date.now(); + + await until( + async () => answer(99).test((await hover(b)).text), + `${aliasB} to answer again`, + 240000, + ); + timings.tunnelBack = Date.now() - restored; + assert.equal(await page.evaluate(() => performance.timeOrigin), timeOrigin); + check( + ctx, + `6. ${aliasB} frozen and its ssh master killed for ${GRACE_SECONDS + 25} s (its grace is ${GRACE_SECONDS} s; its session failed for good): ${whileDown.length} rounds of A 42 and the laptop 1 answered, B empty within ${Math.max(...whileDown.map((r) => r[2]))} ms; B answered 99 ${timings.tunnelBack} ms after it came back, no reload`, + ); + + // 7. B's VS Code server restarts; A keeps answering; B answers again without a reload. + const serverPid = () => + onRemote(aliasB, "pgrep -f '[v]scode-server/out/server-main.js' || true"); + + const oldServer = await serverPid(); + + assert.ok(oldServer, `no VS Code server on ${aliasB}`); + await onRemote(aliasB, `kill ${oldServer}`); + + const killed = Date.now(); + + await answers(a); + await until( + async () => answer(99).test((await hover(b)).text), + `${aliasB} to answer from a new server`, + 240000, + ); + timings.serverBack = Date.now() - killed; + + const newServer = await serverPid(); + + assert.ok(newServer && newServer !== oldServer, "B's server did not restart"); + await answers(a); + assert.equal(await page.evaluate(() => performance.timeOrigin), timeOrigin); + check( + ctx, + `7. ${aliasB}'s VS Code server killed (pid ${oldServer}); A kept answering; B answered 99 from pid ${newServer} ${timings.serverBack} ms later, no reload`, + ); + + // 8. B runs another commit: its reviews read normally, hovers are absent, and Settings says why. + if (prepared === undefined) { + const other = `${runDir}/runtime-other`; + + const { commit } = JSON.parse( + await readFile(path.join(DEFAULT_REMOTE_RUNTIME, "product.json"), "utf8"), + ); + + await cp(DEFAULT_REMOTE_RUNTIME, other, { recursive: true }); + await writeFile( + path.join(other, "product.json"), + JSON.stringify({ + ...JSON.parse(await readFile(path.join(other, "product.json"), "utf8")), + commit: commit === "f".repeat(40) ? "e".repeat(40) : "f".repeat(40), + }), + ); + await remote("install", nameB, "--runtime", other); + await onRemote(aliasB, "whiteboard server stop"); + + const why = `language features need the same Whiteboard version on ${aliasB}`; + + await until( + async () => { + const state = await hostState(aliasB); + + return ( + state?.state === "online" && + state.languageFeatures === false && + state.languageFeaturesDetail?.startsWith(why) + ); + }, + `${aliasB} online without language features`, + 120000, + ); + settings = await openSettings(ctx); + section = settings.getByRole("region", { name: "Remote hosts" }); + await until( + async () => + (await hostRow(aliasB).innerText()).includes( + `Language features: unavailable — ${why}`, + ), + "the Settings row to say why", + ); + + const detail = (await hostRow(aliasB).innerText()).match( + /Language features: unavailable — .*/, + )[0]; + + await showDiff(ctx, title.b, "b.ts"); + assert.match(await lines(visibleCanvas(page)), /answer \+ 1/); + + const absent = await hover(b); + + assert.equal(absent.text, "", "a hover from a host with another commit"); + + const point = await windowCall(ctx, "point", reviewB, "b.ts", 3, "answer"); + const shown = await pointerHover(page, point, /answer/, 6000); + + assert.equal( + shown.ms, + null, + `a hover widget: ${JSON.stringify(shown.seen)}`, + ); + check( + ctx, + `8. ${aliasB} with another commit: its review and Diff read normally, the hover is empty (${absent.ms} ms) and no widget shows; Settings says "${detail}"`, + ); + + await remote("install", nameB); + await onRemote(aliasB, "whiteboard server stop"); + await until( + async () => (await hostState(aliasB))?.languageFeatures === true, + `${aliasB} available again`, + 120000, + ); + } else check(ctx, "8. skipped on prepared hosts: no package swap"); + + // 9. Two reloads leave one extension host per remote; closing the window leaves none. + const perReload = []; + + let origin = timeOrigin; + + for (let reload = 1; reload <= 2; reload++) { + await windowCall(ctx, "reload"); + origin = await until(async () => { + const now = await page + .evaluate(() => performance.timeOrigin) + .catch(() => origin); + + return ( + now !== origin && + (await page.locator(".monaco-workbench").count()) > 0 && + now + ); + }, `the window to reload (${reload})`); + + for (const m of [a, b]) + await until( + async () => (await windowCall(ctx, "host", m.serverId)).authority, + `${m.alias} to connect after reload ${reload}`, + 180000, + ); + + const counts = []; + + for (const alias of [aliasA, aliasB]) { + await until( + async () => (await extensionHosts(alias)) === 1, + `one extension host on ${alias} after reload ${reload}`, + 30000, + ); + counts.push(await extensionHosts(alias)); + } + + perReload.push(counts); + } + + const session = await ctx.browser.newBrowserCDPSession(); + + await Promise.race([ + session.send("Browser.close").catch(() => {}), + sleep(2000), + ]); + + const closed = Date.now(); + + for (const alias of [aliasA, aliasB]) + await waitFor( + async () => (await extensionHosts(alias)) === 0, + `no extension host on ${alias} after the close`, + 300000, + ); + timings.closeToNone = Date.now() - closed; + check( + ctx, + `9. after each of two reloads, extension hosts on A and B: ${JSON.stringify(perReload)}; ${timings.closeToNone} ms after the window closed, none`, + ); + + // 10. Timings and memory. + const firstHovers = ["a", "b"].map((key) => timings[`diff to hover ${key}`]); + const warmest = Math.max(...Object.values(warm).flat()); + + assert.ok( + Math.max(...firstHovers) <= 10000, + `a first hover took ${firstHovers} ms`, + ); + assert.ok( + warmest <= 500, + `a warm hover took ${warmest} ms: ${JSON.stringify(warm)}`, + ); + check( + ctx, + `10. first hover from the Diff click (host connect, activation, hover): laptop ${timings["diff to hover laptop"]}, A ${firstHovers[0]}, B ${firstHovers[1]} ms; pointer to text: laptop ${timings["first hover laptop"]}, A ${timings["first hover a"]}, B ${timings["first hover b"]}, Python on A ${timings["first hover python"]} ms; Add to first hover: A ${timings["connect to first hover a"]}, B ${timings["connect to first hover b"]} ms; warm hovers ${JSON.stringify(warm)} ms; memory in MB ${JSON.stringify(memory)}`, + ); +} + +async function laptopFixture() { + const git = async (...args) => + (await exec("git", args, { cwd: proj })).stdout.trim(); + + await rm(proj, { recursive: true, force: true }); + await mkdir(proj, { recursive: true }); + await git("init", "-q", "-b", "main"); + await git("config", "user.name", "Review E2E"); + await git("config", "user.email", "review-e2e@example.invalid"); + await writeFile( + `${proj}/tsconfig.json`, + '{"compilerOptions":{"strict":true,"target":"ES2022","module":"commonjs"}}\n', + ); + await writeFile(`${proj}/a.ts`, "export const answer = 1;\n"); + await writeFile( + `${proj}/b.ts`, + 'import { answer } from "./a";\n\nexport const total = 0;\n', + ); + await git("add", "."); + await git("commit", "-qm", "one"); + + const base = await git("rev-parse", "HEAD"); + + await writeFile( + `${proj}/b.ts`, + 'import { answer } from "./a";\n\nexport const total = answer + 1;\n', + ); + await git("commit", "-qam", "two"); + + return { base, head: await git("rev-parse", "HEAD") }; +} + +export const visibleCanvas = (page) => + page + .locator(".review-canvas-root [data-review-api]") + .filter({ visible: true }); + +export const tab = (page, text) => + page.locator(".tabs-container .tab").filter({ hasText: text }); + +export async function showDiff(ctx, title, file) { + await tab(ctx.page, title).click(); + await ctx.page + .locator('[aria-label="Session views"] button[aria-label="Diff"]') + .filter({ visible: true }) + .click(); + await ctx.until( + async () => + ( + await ctx.page + .locator(".review-path-label") + .filter({ visible: true }) + .allInnerTexts() + ).some((text) => text.includes(file)), + `the Diff view of ${title} to show ${file}`, + ); +} + +export async function lines(scope) { + return (await scope.locator(".view-line").allInnerTexts()) + .join("\n") + .replaceAll(" ", " "); +} + +export async function pointerHover(page, point, pattern, timeout) { + const started = Date.now(); + const seen = []; + let moved = 0; + + while (Date.now() - started < timeout) { + const text = await bounded( + page.evaluate(() => + [...document.querySelectorAll(".monaco-hover")] + .flatMap((e) => + e.offsetParent !== null && !e.classList.contains("hidden") + ? [e.innerText.replace(/\s+/g, " ").trim()] + : [], + ) + .filter(Boolean) + .join(" | "), + ), + 30000, + "reading the hover", + ); + + if (text && seen.at(-1) !== text) seen.push(text); + + if (pattern.test(text)) return { ms: Date.now() - started, text, seen }; + + if (Date.now() - moved > 3000) { + await page.mouse.move(5, 5); + await sleep(100); + await page.mouse.move(point.x - 1, point.y); + await page.mouse.move(point.x, point.y); + moved = Date.now(); + } + + await sleep(50); + } + + return { ms: null, text: "", seen }; +} + +async function windowLog(ctx) { + const root = path.join(ctx.userData, "logs"); + + const files = (await readdir(root, { recursive: true })).filter((file) => + file.endsWith(".log"), + ); + + return ( + await Promise.all( + files.map((file) => + readFile(path.join(root, file), "utf8").catch(() => ""), + ), + ) + ).join("\n"); +} + +export async function waitFor(check, label, timeout) { + const deadline = Date.now() + timeout; + + while (Date.now() < deadline) { + if (await check().catch(() => false)) return; + await sleep(1000); + } + + throw new Error(`Timed out waiting for ${label}`); +} + +export async function windowCall(ctx, command, ...args) { + const cdp = await bounded( + ctx.page.context().newCDPSession(ctx.page), + 30000, + "a CDP session", + ); + + const stage = { at: "start" }; + + try { + return await bounded( + inWindowOver(cdp, command, args, stage), + 60000, + `the window's ${command}`, + ).catch((error) => { + throw new Error(`${error.message} (at ${stage.at})`); + }); + } finally { + await cdp.detach().catch(() => {}); + } +} + +async function inWindowOver(cdp, command, args, stage) { + stage.at = "focus emulation"; + await cdp.send("Emulation.setFocusEmulationEnabled", { enabled: true }); + stage.at = "import"; + + const prototype = await cdp.send("Runtime.evaluate", { + expression: + 'import(globalThis._VSCODE_FILE_ROOT + "vs/review/services/remote/reviewRemoteHosts.js").then((m) => m.ReviewRemoteHostsService.prototype)', + awaitPromise: true, + }); + + stage.at = "queryObjects"; + + const { objects } = await cdp.send("Runtime.queryObjects", { + prototypeObjectId: prototype.result.objectId, + }); + + stage.at = "call"; + + const result = await cdp.send("Runtime.callFunctionOn", { + objectId: objects.objectId, + functionDeclaration: inWindow.toString(), + arguments: [{ value: command }, { value: args }], + awaitPromise: true, + returnByValue: true, + }); + + if (result.exceptionDetails) + throw new Error( + result.exceptionDetails.exception?.description ?? + result.exceptionDetails.text, + ); + + if (result.result.value?.error) throw new Error(result.result.value.error); + + return result.result.value; +} + +async function inWindow(command, args) { + const service = [...this].reverse().find((s) => !s.closing); + + if (!service) return { error: "no live ReviewRemoteHostsService" }; + + const get = async (file, id) => { + const module = await import(globalThis._VSCODE_FILE_ROOT + file); + + return service.instantiationService.invokeFunction((a) => + a.get(module[id]), + ); + }; + + const started = performance.now(); + const ms = () => Math.round(performance.now() - started); + + if (command === "host") { + const host = await service.host(args[0]); + + return { authority: host?.authority ?? null, ms: ms() }; + } + + if (command === "reload") { + const { INativeHostService } = await import( + `${globalThis._VSCODE_FILE_ROOT}vs/platform/native/common/native.js` + ); + + // Never awaited: the service is an IPC proxy, whose `then` is a remote call that never answers. + // After this call answers, as the reload ends the page it runs in. + setTimeout( + () => + service.instantiationService.invokeFunction((a) => + a.get(INativeHostService).reload(), + ), + 100, + ); + + return {}; + } + + if (command === "closeModal") { + const groups = await get( + "vs/workbench/services/editor/common/editorGroupsService.js", + "IEditorGroupsService", + ); + + return { closed: (await groups.activeModalEditorPart?.close()) ?? null }; + } + + const languages = await get( + "vs/editor/common/services/languageFeatures.js", + "ILanguageFeaturesService", + ); + + const models = ( + await get("vs/editor/common/services/model.js", "IModelService") + ).getModels(); + + const [reviewId, file, line, word, uri] = args; + + const isReviewHead = (uri) => + uri?.scheme === "review-api-source" && + uri.authority === reviewId && + uri.path === `/${file}` && + new URLSearchParams(uri.query).get("side") === "head"; + + if (command === "counts") { + const count = (model) => + model && { + hover: languages.hoverProvider.all(model).length, + definition: languages.definitionProvider.all(model).length, + }; + + return { + file: count( + models.find( + (m) => m.uri.scheme === "file" && m.uri.path.endsWith(`/${file}`), + ), + ), + review: count(models.find((m) => isReviewHead(m.uri))), + }; + } + + const { CancellationToken } = await import( + `${globalThis._VSCODE_FILE_ROOT}vs/base/common/cancellation.js` + ); + + if (command === "query") { + const { URI } = await import( + `${globalThis._VSCODE_FILE_ROOT}vs/base/common/uri.js` + ); + + const reference = + !models.some((m) => isReviewHead(m.uri)) && + (await get( + "vs/editor/common/services/resolverService.js", + "ITextModelService", + ).then((resolver) => resolver.createModelReference(URI.parse(uri)))); + + const model = reference + ? reference.object.textEditorModel + : models.find((m) => isReviewHead(m.uri)); + + const queried = performance.now(); + const column = model.getLineContent(line).indexOf(word) + 2; + + const { getHoversPromise } = await import( + `${globalThis._VSCODE_FILE_ROOT}vs/editor/contrib/hover/browser/getHover.js` + ); + + const hovers = await getHoversPromise( + languages.hoverProvider, + model, + { lineNumber: line, column }, + CancellationToken.None, + ); + + if (reference) reference.dispose(); + + return { + ms: Math.round(performance.now() - queried), + text: hovers.flatMap((h) => h.contents.map((c) => c.value)).join(" | "), + }; + } + + const editor = ( + await get( + "vs/editor/browser/services/codeEditorService.js", + "ICodeEditorService", + ) + ) + .listCodeEditors() + .find( + (e) => + isReviewHead(e.getModel()?.uri) && + e.getDomNode()?.isConnected && + e.getDomNode().offsetParent !== null, + ); + + if (!editor) + return { error: `no visible head editor for ${reviewId}/${file}` }; + + const model = editor.getModel(); + + const position = { + lineNumber: line, + column: model.getLineContent(line).indexOf(word) + 2, + }; + + if (command === "point") { + editor.getDomNode().scrollIntoView({ block: "center" }); + editor.revealLineInCenter(line); + await new Promise((resolve) => setTimeout(resolve, 300)); + + const at = editor.getScrolledVisiblePosition(position); + const rect = editor.getDomNode().getBoundingClientRect(); + + return { + uri: model.uri.toString(), + x: Math.round(rect.left + at.left), + y: Math.round(rect.top + at.top + at.height / 2), + }; + } + + if (command === "goto") { + const [editors, labels, commands, { SymbolNavigationAnchor }] = + await Promise.all([ + get( + "vs/workbench/services/editor/common/editorService.js", + "IEditorService", + ), + get("vs/platform/label/common/label.js", "ILabelService"), + get("vs/platform/commands/common/commands.js", "ICommandService"), + import( + `${globalThis._VSCODE_FILE_ROOT}vs/editor/contrib/gotoSymbol/browser/goToCommands.js` + ), + ]); + + editor.setPosition(position); + editor.focus(); + await commands.executeCommand( + "editor.action.revealDefinition", + new SymbolNavigationAnchor(model, position), + ); + await new Promise((resolve) => setTimeout(resolve, 1000)); + + const active = editors.activeEditor; + + return { + ms: ms(), + active: active && { + resource: active.resource?.toString() ?? null, + label: active.resource ? labels.getUriLabel(active.resource) : null, + readonly: Boolean(active.isReadonly()), + text: editors.activeTextEditorControl?.getModel?.()?.getValue?.() ?? "", + }, + }; + } + + return { error: `unknown ${command}` }; +} + +async function closeDesktop(ctx) { + const session = await ctx.browser?.newBrowserCDPSession().catch(() => null); + + await Promise.race([ + session?.send("Browser.close").catch(() => {}), + sleep(2000), + ]); + + for (let i = 0; i < 40 && (await desktopSsh()).length; i++) await sleep(250); + + for (const pid of await desktopSsh()) process.kill(pid, "SIGTERM"); +} + +async function desktopSshProcesses() { + const { stdout } = await exec("ps", ["-axo", "pid=,args="]); + + return stdout + .split("\n") + .map((line) => line.trim().match(/^(\d+) (.*)$/)) + .filter( + (match) => + match && + /(^|\/)ssh /.test(match[2]) && + match[2].includes(`-F ${runDir}/ssh_config -S `), + ) + .map((match) => [Number(match[1]), match[2]]); +} + +const desktopSsh = async () => + (await desktopSshProcesses()).map(([pid]) => pid); + +async function masterPid(alias) { + return (await desktopSshProcesses()).find( + ([, args]) => args.includes(" -M -N ") && args.endsWith(`-- ${alias}`), + )?.[0]; +} diff --git a/apps/review-desktop/scripts/e2e/journeys/remote-source-window.mjs b/apps/review-desktop/scripts/e2e/journeys/remote-source-window.mjs new file mode 100644 index 000000000..406279bcd --- /dev/null +++ b/apps/review-desktop/scripts/e2e/journeys/remote-source-window.mjs @@ -0,0 +1,754 @@ +/** A remote review's Source window, as a user meets it: opened from the review and from Ask, offline and back, restored at start, closed when its host goes. */ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +import { openSettings } from "../harness.mjs"; +import { + askAboutReviewSentence, + askPanel, + openRemoteReview, +} from "./remote-ask.mjs"; +import { + alias, + assertUriAnswers, + closeDesktop, + onRemote, + prepared, + recordNavigator, + remote, + runDir, + sourcePages, +} from "./remote-host.mjs"; +import { removeHost } from "./remote-install.mjs"; +import { lines, pointerHover } from "./remote-lsp.mjs"; +import { + fileText, + focusFile, + inSource, + pointAt, + quickRows, + typeInto, + windowState, +} from "../remote/source-window-helpers.mjs"; + +const exec = promisify(execFile); + +const title = "Remote source"; + +const second = "Second checkout"; + +export const name = "remote-source-window"; + +// Phase 2: the container image downloads Ubuntu packages and Node. +export const phase = 2; + +export const options = { + settings: { "review.experimental.remoteHosts.enabled": true }, + env: { DEV_FAST_REVIEW_SSH_CONFIG: `${runDir}/ssh_config` }, +}; + +/** Runs on the remote: two repositories, each with a review of its second commit (f.ts changed, added.ts added). Prints the reviews' ids. */ +const fixture = String.raw` +set -e +field() { node -pe "JSON.parse(require('fs').readFileSync(0, 'utf8')).$1"; } +make() { + rm -rf ~/$1 + git init -q -b main ~/$1 + cd ~/$1 + git config user.email e2e@example.invalid + git config user.name e2e + mkdir docs + printf '%s\n' "$3" > docs/needle.md + printf 'export function one() {\n return 1;\n}\n' > f.ts + git add . + git commit -qm one + printf 'export function one() {\n return 2;\n}\n\nexport function two() {\n const unused = 3;\n return one() + 1;\n}\n' > f.ts + printf 'export const added = "wbsource-added";\n' > added.ts + git add . + git commit -qm two + id=$(whiteboard api session_create "{\"title\":\"$2\",\"open\":false,\"target\":{\"kind\":\"commits\",\"repositoryPath\":\"$HOME/$1\",\"base\":\"HEAD~1\",\"head\":\"HEAD\"}}" | field sessionId) + whiteboard api session_edit "{\"sessionId\":\"$id\",\"edit\":{\"type\":\"insert\",\"content\":{\"type\":\"markdown\",\"markdown\":\"One now returns **two**.\"}}}" >/dev/null + echo "$id" +} +make wbsource "$1" wbsource-needle +make wbsource2 "$2" wbsource-other +`; + +export async function run(ctx) { + // A packaged build ignores DEV_FAST_REVIEW_SSH_CONFIG, so its ssh would read the user's configuration. + if (ctx.report.mode === "packaged") + throw new Error("skip: remote-source-window runs in development mode only"); + + if (prepared === undefined) + try { + await exec("docker", ["info", "--format", "{{.ServerVersion}}"]); + } catch (error) { + throw new Error( + `skip: remote-source-window needs Docker for its SSH server (${error.message.split("\n")[0]})`, + ); + } + + try { + await journey(ctx); + } finally { + if (prepared === undefined) + await remote("down", "--all").catch((error) => + console.error(`[remote-source-window] down --all: ${error.message}`), + ); + else await closeDesktop(ctx); + } +} + +function check(ctx, text) { + console.error(`[remote-source-window] ${new Date().toISOString()} ${text}`); + ctx.check(text); +} + +/** + * Freezes the host: the container, or on a prepared host its user's Node + * and Whiteboard processes (the review server, the VS Code server and its + * extension hosts, the attach the Desktop's ssh runs). + * Returns the function that thaws it. + */ +async function freeze() { + if (prepared === undefined) { + await remote("pause", "a"); + + return () => remote("resume", "a"); + } + + const pids = await onRemote(String.raw` + s=$(whiteboard server status --json | node -pe "JSON.parse(require('fs').readFileSync(0, 'utf8')).serverPid") + p=$(pgrep -u "$(id -u)" -f 'node|whiteboard' | grep -vx "$$" | tr '\n' ' ') + kill -STOP $s $p 2>/dev/null + echo $s $p + `); + + return () => onRemote(`kill -CONT ${pids}`); +} + +/** The host's remote extension hosts, as `[pid, rss KiB]`. */ +async function extensionHosts() { + return ( + await onRemote( + "ps -eo pid=,rss=,args= | grep -- '--type=extensionHost' | grep -v grep || true", + ) + ) + .split("\n") + .filter(Boolean) + .map((line) => line.trim().split(/\s+/).slice(0, 2).map(Number)); +} + +const mib = (kib) => Math.round(kib / 1024); + +/** The Source window's host entry in its status bar. */ +const hostEntry = (page) => + page + .locator('[id="review.sourceWindow.host"]') + .innerText({ timeout: 1000 }) + .then((text) => text.trim()) + .catch(() => ""); + +async function journey(ctx) { + const { until } = ctx; + const timings = {}; + const navigator = []; + + const hostState = async () => + (await ctx.apiOk("/remote-hosts")).find((host) => host.alias === alias); + + const openReview = (text) => openRemoteReview(ctx, text); + + /** The review's Diff view, then Open file on f.ts; resolves with the Source window that shows it. */ + async function openFile(text) { + await openReview(text); + await ctx.page + .locator('[aria-label="Session views"] button[aria-label="Diff"]') + .filter({ visible: true }) + .click(); + await until( + async () => + ( + await ctx.page + .locator(".review-path-label") + .filter({ visible: true }) + .allInnerTexts() + ).some((t) => t.includes("f.ts")), + "the Diff view to show f.ts", + ); + + // The button in the header whose path label reads f.ts. + const index = await until( + () => + ctx.page + .evaluate(() => + [...document.querySelectorAll(".review-multidiff-open")].findIndex( + (button) => { + for (let e = button; e; e = e.parentElement) { + const label = e.querySelector(".review-path-label"); + + if (label) return label.innerText.trim() === "f.ts"; + } + + return false; + }, + ), + ) + .then((i) => (i >= 0 ? i : null)), + "f.ts's Open file button", + ); + + const before = new Set(sourcePages(ctx)); + + await ctx.page.locator(".review-multidiff-open").nth(index).click(); + + const source = await until( + () => sourcePages(ctx).find((page) => !before.has(page)), + `a new Source window for ${text}`, + 60000, + ); + + await until( + async () => (await lines(source)).includes("return one() + 1;"), + "f.ts's text from the host in the Source window", + 120000, + ); + + return source; + } + + // 1. A host holding two reviews, added in Settings. + if (prepared === undefined) { + await remote("up", "a", "--fake-agent"); + await remote("install", "a"); + } + + await onRemote("whiteboard remote extensions ensure --json"); + + const [reviewId, secondId] = ( + await onRemote(`bash -s -- '${title}' '${second}'`, fixture) + ).split("\n"); + + assert.match(reviewId, /^[0-9a-f-]{36}$/, "the remote review's id"); + assert.match(secondId, /^[0-9a-f-]{36}$/, "the second review's id"); + await recordNavigator(ctx.page, navigator); + + const section = (await openSettings(ctx)).getByRole("region", { + name: "Remote hosts", + }); + + await section.getByLabel("SSH alias").fill(alias); + + const added = Date.now(); + + await section.getByRole("button", { name: "Add", exact: true }).click(); + await until( + async () => { + const host = await hostState(); + + return host?.state === "online" && host.languageFeatures === true; + }, + `${alias} online with language features`, + 180000, + ); + timings.online = Date.now() - added; + check( + ctx, + `1. ${alias}${prepared ? " (prepared)" : ""} holds two reviews; Settings added it, online with language features ${timings.online} ms after Add`, + ); + + // 2. Open file opens a Source window bound to the host, titled for the review. + const opened = Date.now(); + const source = await openFile(title); + + timings.openFile = Date.now() - opened; + + let state = await inSource(source, windowState); + const { authority } = state; + const serverId = authority.replace(/^whiteboard\+/, ""); + + assert.match(authority, /^whiteboard\+[0-9a-f-]+$/, JSON.stringify(state)); + const { serverId: hostServerId } = await hostState(); + + assert.ok(hostServerId, "the host reports no server id"); + assert.equal(serverId, hostServerId.toLowerCase()); + assert.equal(state.title, `${title} — Source — Whiteboard`); + assert.equal(await source.title(), state.title); + assert.match( + state.folder, + new RegExp(`^vscode-remote://${authority.replace("+", "%2B")}/`), + ); + assert.equal(sourcePages(ctx).length, 1); + check( + ctx, + `2. Open file opened one Source window on ${authority} in ${timings.openFile} ms, titled "${state.title}"`, + ); + + // 3. The explorer lists the checkout. + await source.keyboard.press("ControlOrMeta+Shift+KeyE"); + + const explorer = await until(async () => { + const rows = await source + .locator(".explorer-folders-view .monaco-list-row") + .allInnerTexts() + .catch(() => []); + + return ["docs", "added.ts", "f.ts"].every((name) => + rows.some((row) => row.trim() === name), + ) + ? rows + : null; + }, "the explorer to list docs, added.ts and f.ts"); + + check( + ctx, + `3. the explorer lists the checkout: ${explorer.map((r) => r.trim()).join(", ")}`, + ); + + // 4. Read-only: typing changes nothing and the notice shows. + await inSource(source, focusFile, state.active.resource); + + const typed = await typeInto(source, ctx); + + state = await inSource(source, windowState); + assert.ok(state.active.readonly, "f.ts is not read-only"); + assert.ok(typed.message, "no read-only notice"); + check( + ctx, + `4. f.ts is read-only: typing changed nothing and showed "${typed.message}"`, + ); + + // 5. Quick open answers from the host: f.ts, and needle.md, which no editor has open. + const quick = source.locator(".quick-input-widget input"); + + for (const query of ["f.ts", "needle"]) { + await source.keyboard.press("ControlOrMeta+KeyP"); + await quick.waitFor({ state: "visible" }); + await quick.fill(query); + await until( + async () => + (await quickRows(source)).some((t) => + t.includes(query === "needle" ? "needle.md" : "f.ts"), + ), + `quick open to find ${query}`, + 60000, + ); + await source.keyboard.press("Escape"); + await quick.waitFor({ state: "hidden" }); + } + + check(ctx, "5. quick open found f.ts, and needle.md, which was not open"); + + // 6. Text search. + await source.keyboard.press("ControlOrMeta+Shift+KeyF"); + await until( + () => + source.evaluate( + () => !!document.activeElement?.closest(".search-view .search-widget"), + ), + "focus in the search view's input", + ); + await source.keyboard.press("ControlOrMeta+KeyA"); + await source.keyboard.type("wbsource-needle"); + + const results = await until(async () => { + const text = await source + .locator(".search-view") + .innerText() + .catch(() => ""); + + return text.includes("needle.md") && text; + }, "needle.md in the search results"); + + check( + ctx, + `6. text search found wbsource-needle: "${results.replace(/\s+/g, " ").slice(0, 60)}…"`, + ); + + // 7. A TypeScript hover from the host's language server. + await inSource(source, focusFile, state.active.resource); + + const hover = await pointerHover( + source, + await inSource(source, pointAt, 7, "one"), + /function one\(\): number/, + 90000, + ); + + assert.ok(hover.ms !== null, `no hover: ${JSON.stringify(hover.seen)}`); + check( + ctx, + `7. the hover on one read "function one(): number" after ${hover.ms} ms`, + ); + + // 8. A code action from the host's TypeScript runs from the window. + const action = await until( + () => inSource(source, codeAction, state.active.resource), + "TypeScript's Remove unused declaration code action", + 60000, + ); + + assert.equal(action.result, true, JSON.stringify(action)); + assert.ok(action.diagnosticBefore, "no unused-declaration diagnostic"); + assert.equal(action.diagnosticAfter, false, JSON.stringify(action)); + check( + ctx, + `8. TypeScript's "${action.title}" (${action.command}) answered ${action.result} from the host and cleared its diagnostic`, + ); + + // 9. An added file's diff opens in the same window, its empty side the laptop's. + await inSource(ctx.page, openDiff, reviewId, "added.ts"); + + const diff = await until( + () => inSource(source, activeDiff), + "added.ts's diff in the Source window", + 60000, + ); + + assert.equal(diff.original.scheme, "vscode-userdata", JSON.stringify(diff)); + assert.equal(diff.original.text, ""); + assert.match( + diff.modified.uri, + new RegExp( + `^vscode-remote://${authority.replace("+", "%2B")}/.*/added\\.ts$`, + ), + ); + assert.match(diff.modified.text, /wbsource-added/); + assert.deepEqual(sourcePages(ctx), [source]); + check( + ctx, + `9. added.ts's diff opened in the same window: its original side is ${diff.original.uri} (empty), its modified side the host's`, + ); + + // 10. An Ask file link opens the same window at its line. + await inSource(source, focusFile, state.active.resource); + await inSource(source, cursorLine, 1); + await ctx.page + .locator('[aria-label="Session views"] button[aria-label="Whiteboard"]') + .filter({ visible: true }) + .click(); + + const link = await askLink(ctx, "where is `./f.ts:7`"); + + await link.click(); + await until( + async () => { + const at = await inSource(source, cursorLine); + + return at.resource?.endsWith("/f.ts") && at.line === 7; + }, + "f.ts at line 7 in the Source window", + 60000, + ); + assert.deepEqual(sourcePages(ctx), [source]); + check( + ctx, + `10. the Ask answer's ./f.ts:7 link opened f.ts at line 7 in the same Source window`, + ); + + // Extension-host RSS with one Source window, then two; checked at the end. + const rss = { one: await extensionHosts() }; + const other = await openFile(second); + + assert.equal(sourcePages(ctx).length, 2); + await until( + async () => (await extensionHosts()).length > rss.one.length, + "the second window's extension host", + 60000, + ); + rss.two = await extensionHosts(); + await other.keyboard.press("ControlOrMeta+Shift+KeyW"); + await until(() => other.isClosed(), "the second Source window to close"); + + // 11–12. The host freezes: the window's status bar says so, then names the host again. + const offlineText = `${alias} — offline, reconnecting…`; + + let thaw = await freeze(); + const paused = Date.now(); + + await until( + async () => (await hostEntry(source)) === offlineText, + "the Source window's status bar to read offline", + 30000, + ); + timings.offline = Date.now() - paused; + assert.ok(timings.offline <= 20000, `offline after ${timings.offline} ms`); + assert.equal((await inSource(source, windowStatus)).dialogs, 0); + check( + ctx, + `11. paused: the Source window's status bar read "${offlineText}" after ${timings.offline} ms, with no dialog`, + ); + + await thaw(); + + const resumed = Date.now(); + + await until( + async () => + (await hostEntry(source)) === alias && + (await inSource(source, readsCheckout).catch(() => false)), + "the Source window to reconnect", + 60000, + ); + timings.reconnect = Date.now() - resumed; + assert.ok( + timings.reconnect <= 30000, + `reconnected after ${timings.reconnect} ms`, + ); + check( + ctx, + `12. resumed: the status bar read "${alias}" again and the checkout answered after ${timings.reconnect} ms`, + ); + + // 13–14. A restart with the host paused: the restored window is offline, without a dialog, then reconnects. + thaw = await freeze(); + await ctx.quitAndRelaunchDesktop(); + + const relaunched = Date.now(); + + const restored = await until( + async () => { + for (const page of ctx.browser.contexts().flatMap((c) => c.pages())) + if ((await page.title().catch(() => "")) === state.title) return page; + + return null; + }, + "the restored Source window", + 60000, + ); + + // The harness takes the first workbench page, which can be the restored Source window. + ctx.page = ctx.browser + .contexts() + .flatMap((c) => c.pages()) + .find((page) => page !== restored && page.url().includes("workbench")); + await ctx.watchPage(ctx.page); + await recordNavigator(ctx.page, navigator); + + await until( + async () => (await hostEntry(restored)) === offlineText, + "the restored window's status bar to read offline", + 120000, + ); + timings.restoredOffline = Date.now() - relaunched; + assert.equal( + (await inSource(restored, windowStatus)).dialogs, + 0, + "a dialog in the restored window", + ); + check( + ctx, + `13. after a restart with ${alias} paused, the restored Source window's status bar read "${offlineText}" ${timings.restoredOffline} ms after the relaunch, with no dialog`, + ); + + await thaw(); + + const unpaused = Date.now(); + + await until( + async () => + (await hostEntry(restored).catch(() => "")) === alias && + (await lines(restored).catch(() => "")).includes("return one() + 1;") && + (await inSource(restored, readsCheckout).catch(() => false)), + "the restored Source window to reconnect", + 90000, + ); + timings.restoredReconnect = Date.now() - unpaused; + assert.ok( + timings.restoredReconnect <= 30000, + `reconnected after ${timings.restoredReconnect} ms`, + ); + check( + ctx, + `14. resumed: the restored window's status bar read "${alias}" and it showed f.ts from the host ${timings.restoredReconnect} ms later`, + ); + + // 15. Removing the host closes its window. + await removeHost(ctx, alias); + await until(() => restored.isClosed(), "the Source window to close", 30000); + assert.deepEqual(sourcePages(ctx), []); + check(ctx, `15. removing ${alias} in Settings closed its Source window`); + + // 16. Every navigator answer the UI received named URIs, never host paths. + const answers = assertUriAnswers(navigator, authority); + + assert.ok(answers >= 4, `${answers} navigator answers`); + + check( + ctx, + `16. ${answers} navigator answers reached the UI, each with workspaceUri and fileUri (or emptySide) on ${authority}, none with workspacePath or filePath`, + ); + + // 17. The host's extension hosts with one Source window, then two. + const total = (hosts) => mib(hosts.reduce((sum, [, kib]) => sum + kib, 0)); + const extra = rss.two.filter(([pid]) => !rss.one.some(([p]) => p === pid)); + + check( + ctx, + `17. extension hosts on ${alias}: the review window and one Source window ${rss.one.length} processes, ${rss.one.map(([, kib]) => `${mib(kib)} MiB`).join(" + ")} = ${total(rss.one)} MiB; two Source windows ${rss.two.length}, ${total(rss.two)} MiB (the second window's ${extra.map(([, kib]) => `${mib(kib)} MiB`).join(", ")})`, + ); +} + +// The functions below run in a window. + +/** TypeScript's quick fix for the unused declaration on line 6, run as a click would run it. */ +async function codeAction({ get, imp }, resource) { + const [commands, markers] = await Promise.all([ + get("vs/platform/commands/common/commands.js", "ICommandService"), + get("vs/platform/markers/common/markers.js", "IMarkerService"), + ]); + + const { URI } = await imp("vs/base/common/uri.js"); + const uri = URI.parse(resource); + + const unused = () => + markers + .read({ resource: uri }) + .find((m) => m.startLineNumber === 6 && /never read/.test(m.message)); + + const diagnostic = unused(); + + if (!diagnostic) return null; + + const actions = await commands.executeCommand( + "_executeCodeActionProvider", + uri, + { + startLineNumber: diagnostic.startLineNumber, + startColumn: diagnostic.startColumn, + endLineNumber: diagnostic.endLineNumber, + endColumn: diagnostic.endColumn, + }, + ); + + const action = actions?.find((a) => + /^Remove unused declaration/.test(a.title), + ); + + if (!action?.command) return null; + + const result = await commands.executeCommand( + action.command.id, + ...(action.command.arguments ?? []), + ); + + for (let frame = 0; unused() && frame < 120; frame++) + await new Promise((resolve) => requestAnimationFrame(resolve)); + + return { + title: action.title, + command: action.command.id, + result, + diagnosticBefore: diagnostic.message, + diagnosticAfter: !!unused(), + }; +} + +/** The canvas bridge's openDiff verb for `path`, on the review's current view. */ +async function openDiff({ get, imp }, reviewId, path) { + const { resolveReviewSourceView } = await imp( + "vs/review/common/reviewProtocol.js", + ); + + const source = await get( + "vs/review/services/reviewApiSourceService.js", + "IReviewApiSourceService", + ); + + const snapshot = await source.read(reviewId, "", { full: "true" }); + + await source.openDiff(resolveReviewSourceView(snapshot), path); + + return true; +} + +async function activeDiff({ get }) { + const editors = await get( + "vs/workbench/services/editor/common/editorService.js", + "IEditorService", + ); + + const control = editors.activeTextEditorControl; + const model = control?.getModel?.(); + + if (!model?.original || !model?.modified) return null; + + if (!model.modified.uri.path.endsWith("/added.ts")) return null; + + return { + original: { + scheme: model.original.uri.scheme, + uri: model.original.uri.toString(), + text: model.original.getValue(), + }, + modified: { + uri: model.modified.uri.toString(), + text: model.modified.getValue(), + }, + }; +} + +/** Moves the cursor to `line` when given; returns the active file and line. */ +async function cursorLine({ get }, line) { + const editor = ( + await get( + "vs/editor/browser/services/codeEditorService.js", + "ICodeEditorService", + ) + ).getActiveCodeEditor(); + + if (line) editor?.setPosition({ lineNumber: line, column: 1 }); + + return { + resource: editor?.getModel()?.uri.toString(), + line: editor?.getPosition()?.lineNumber, + }; +} + +async function readsCheckout({ get }) { + const [files, workspace] = await Promise.all([ + get("vs/platform/files/common/files.js", "IFileService"), + get( + "vs/platform/workspace/common/workspace.js", + "IWorkspaceContextService", + ), + ]); + + const folder = workspace.getWorkspace().folders[0].uri; + const stat = await files.resolve(folder); + + return stat.children?.some((child) => child.name === "f.ts") ?? false; +} + +/** Open dialogs, native or not, and the window's text. */ +async function windowStatus({ get }) { + const dialogs = await get( + "vs/platform/dialogs/common/dialogs.js", + "IDialogService", + ); + + return { + dialogs: dialogs.model.dialogs.length, + body: document.body.innerText, + }; +} + +/** Opens Ask on the review's sentence, asks `question`, and returns the answer's link for its file. */ +async function askLink(ctx, question) { + const panel = askPanel(ctx); + + await askAboutReviewSentence(ctx); + + const composer = panel.getByRole("combobox", { name: "Question" }); + + await composer.waitFor({ timeout: 30000 }); + await composer.fill(question); + await panel.getByRole("button", { name: "Ask", exact: true }).click(); + + const link = panel.getByRole("link").filter({ hasText: /^\.\/f\.ts:7$/ }); + + await link.waitFor({ timeout: 30000 }); + + return link; +} diff --git a/apps/review-desktop/scripts/e2e/remote-lsp-languages.mjs b/apps/review-desktop/scripts/e2e/remote-lsp-languages.mjs new file mode 100644 index 000000000..88aa114e9 --- /dev/null +++ b/apps/review-desktop/scripts/e2e/remote-lsp-languages.mjs @@ -0,0 +1,561 @@ +/** One optional language group on an SSH host with its toolchain: the group turned on in the Desktop, a remote review and, for a group that answers hovers, a hover and go to definition from the remote. */ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { promisify } from "node:util"; + +import { + installExtensionGroup, + openHome, + openSettings, + sleep, +} from "./harness.mjs"; +import { + extensionHosts, + onRemote, + pointerHover, + remote, + remoteMemory, + runDir, + showDiff, + tab, + visibleCanvas, + waitFor, + windowCall, +} from "./journeys/remote-lsp.mjs"; + +const exec = promisify(execFile); + +const fixtures = path.join(import.meta.dirname, "fixtures/lsp"); + +const FIRST_HOVER_LIMIT_MS = 120000; + +const NAME = "c"; + +const ALIAS = `wb-test-${NAME}`; + +export const LANGUAGES = { + rust: { + toolchain: "rust", + toolDir: "/usr/local/cargo/bin", + label: "Rust (rust-analyzer)", + extensionId: "rust-lang.rust-analyzer", + members: ["rust-lang.rust-analyzer"], + fixture: "rust", + file: "src/lib.rs", + before: [ + "storage::save_order(OrderRecord { id, status })", + "OrderRecord { id, status }", + ], + symbol: "save_order", + hoverText: /fn save_order\(order: OrderRecord\) -> OrderRecord/, + definitionFile: "src/storage.rs", + definitionText: /pub fn save_order\(order: OrderRecord\) -> OrderRecord/, + installedCheck: String.raw` +d=$(ls -d ~/.dev/whiteboard-remote/extensions/rust-lang.rust-analyzer-*/) +"$d/server/rust-analyzer" --version +`, + debuggers: [], + hovers: true, + }, + swift: { + toolchain: "swift", + toolDir: "/usr/bin", + label: "Swift", + extensionId: "swiftlang.swift-vscode", + members: ["swiftlang.swift-vscode", "llvm-vs-code-extensions.lldb-dap"], + fixture: "swift", + file: "Sources/Orders/Orders.swift", + before: [ + "return saveOrder(OrderRecord(id: id, status: status))", + "return OrderRecord(id: id, status: status)", + ], + symbol: "saveOrder", + hoverText: /func saveOrder\(_ order: OrderRecord\) -> OrderRecord/, + definitionFile: "Sources/Orders/Storage.swift", + definitionText: /public func saveOrder\(_ order: OrderRecord\)/, + debuggers: ["lldb-dap"], + withoutToolchain: "d", + hovers: false, + }, + csharp: { + toolchain: "dotnet", + toolDir: "/usr/share/dotnet", + label: "C#", + extensionId: "muhammad-sammy.csharp", + members: ["muhammad-sammy.csharp", "ms-dotnettools.vscode-dotnet-runtime"], + fixture: "csharp", + file: "Orders.cs", + before: [ + 'Storage.SaveOrder(new OrderRecord(id, "queued"))', + 'new OrderRecord(id, "queued")', + ], + symbol: "SaveOrder", + hoverText: /OrderRecord Storage\.SaveOrder\(OrderRecord order\)/, + definitionFile: "Storage.cs", + definitionText: /public static OrderRecord SaveOrder\(OrderRecord order\)/, + debuggers: ["netcoredbg", "vsdbg"], + hovers: false, + }, +}; + +export const remoteLspOptions = () => ({ + extensions: "none", + settings: { "review.experimental.remoteHosts.enabled": true }, + env: { DEV_FAST_REVIEW_SSH_CONFIG: `${runDir}/ssh_config` }, +}); + +const repository = String.raw` +set -e +field() { node -pe "JSON.parse(require('fs').readFileSync(0, 'utf8')).$1"; } +cd "$1" +git init -q -b main +git config user.email e2e@example.invalid +git config user.name e2e +cp "$2" "$2.after" +node -e ' +const fs = require("fs"); +const [file, from, to] = process.argv.slice(1); +const text = fs.readFileSync(file, "utf8"); +const [a, b] = [from, to].map((v) => Buffer.from(v, "base64").toString()); +if (!text.includes(a)) process.exit(3); +fs.writeFileSync(file, text.replace(a, b)); +' "$2" "$3" "$4" +git add -A ':!*.after' +git commit -qm one +mv "$2.after" "$2" +git commit -qam two +whiteboard api session_create "{\"title\":\"$5\",\"open\":false,\"target\":{\"kind\":\"commits\",\"repositoryPath\":\"$1\",\"base\":\"HEAD~1\",\"head\":\"HEAD\"}}" | field sessionId +`; + +function check(ctx, id, text) { + console.error(`[remote-lsp-${id}] ${new Date().toISOString()} ${text}`); + ctx.check(text); +} + +const pathOf = async (alias, pid) => + ( + await onRemote( + alias, + `tr '\\0' '\\n' < /proc/${pid}/environ | grep '^PATH=' || true`, + ) + ).slice("PATH=".length); + +export async function runRemoteLspJourney(ctx, id) { + if (ctx.report.mode === "packaged") + throw new Error(`skip: remote-lsp-${id} runs in development mode only`); + + try { + await exec("docker", ["info", "--format", "{{.ServerVersion}}"]); + } catch (error) { + throw new Error( + `skip: remote-lsp-${id} needs Docker for its SSH server (${error.message.split("\n")[0]})`, + ); + } + + try { + await journey(ctx, id, LANGUAGES[id]); + } catch (error) { + await onRemote( + ALIAS, + 'ps -eo pid,rss,args | cut -c1-240; cd ~/.dev/whiteboard-remote/server && find data/logs -name \'*.log\' | while read f; do echo "== $f"; tail -n 60 "$f"; done', + "", + 60000, + ).then( + (logs) => console.error(`[remote-lsp-${id}] remote logs:\n${logs}`), + () => {}, + ); + throw error; + } finally { + const session = await ctx.browser?.newBrowserCDPSession().catch(() => null); + + await Promise.race([ + session?.send("Browser.close").catch(() => {}), + sleep(2000), + ]); + await remote("down", "--all").catch((error) => + console.error(`[remote-lsp-${id}] down --all: ${error.message}`), + ); + } +} + +async function journey(ctx, id, language) { + const { page, until } = ctx; + const [name, alias] = [NAME, ALIAS]; + const platform = process.env.REVIEW_E2E_REMOTE_PLATFORM; + const timings = {}; + + // 1. A container from the toolchain's official image, with this checkout's package. + let started = Date.now(); + + await remote( + "up", + name, + "--toolchain", + language.toolchain, + ...(platform ? ["--platform", platform] : []), + ); + await remote("install", name); + timings.host = Date.now() - started; + + const tool = language.toolchain === "rust" ? "cargo" : language.toolchain; + + // sshd gives a command the system PATH; only a login shell adds the user's. + const [plain, login, arch] = await Promise.all([ + onRemote(alias, `command -v ${tool} || echo none`), + onRemote(alias, `bash -lc 'command -v ${tool}'`), + onRemote(alias, "uname -m"), + ]); + + check( + ctx, + id, + `1. ${alias} (${arch}) from the ${language.toolchain} image in ${timings.host} ms: ${tool} is ${login} for a login shell, ${plain} for a plain one`, + ); + + // 2. The group turned on in the Desktop: the reader consents, and its extensions download. + started = Date.now(); + await installExtensionGroup(ctx, { + label: language.label, + extensionId: language.extensionId, + }); + check( + ctx, + id, + `2. "${language.label}" turned on in the Desktop in ${Date.now() - started} ms`, + ); + + // 3. A review of the fixture on the remote. + const proj = `${runDir}/proj-${id}`; + const title = `${language.label} on ${alias}`; + + const tar = ( + await exec( + "tar", + [ + "--no-xattrs", + "-C", + path.join(fixtures, language.fixture), + "-cf", + "-", + ".", + ], + { encoding: "buffer", env: { ...process.env, COPYFILE_DISABLE: "1" } }, + ) + ).stdout; + + await onRemote( + alias, + `mkdir -p '${proj}' && base64 -d | tar -xf - -C '${proj}'`, + tar.toString("base64"), + ); + + const encode = (text) => Buffer.from(text).toString("base64"); + + const reviewId = await onRemote( + alias, + `bash -s -- '${proj}' '${language.file}' '${encode(language.before[0])}' '${encode(language.before[1])}' '${title}'`, + repository, + ); + + assert.match(reviewId, /^[0-9a-f-]{36}$/, "the remote review's id"); + check( + ctx, + id, + `3. ${alias} holds "${title}", a review of ${proj} whose second commit changes ${language.file}`, + ); + + // 4. The host added in Settings: the Desktop's group reaches the remote, which installs it. + const settings = await openSettings(ctx); + const section = settings.getByRole("region", { name: "Remote hosts" }); + + const hostRow = (host) => + section.locator("[data-remote-host]").filter({ hasText: host }); + + const addHost = async (host) => { + await section.getByLabel("SSH alias").fill(host); + await section.getByRole("button", { name: "Add", exact: true }).click(); + await hostRow(host).waitFor(); + }; + + const hostState = async (host) => + (await ctx.apiOk("/remote-hosts")).find((state) => state.alias === host); + + started = Date.now(); + await addHost(alias); + + const state = await until( + async () => { + const current = await hostState(alias); + + return ( + current?.state === "online" && + current.languageFeatures === true && + current + ); + }, + `${alias} online with language features`, + 300000, + ); + + timings.available = Date.now() - started; + assert.deepEqual(state.languageGroups, [{ group: id, installed: true }]); + await until(async () => { + const text = await hostRow(alias).innerText(); + + return ( + text.includes("Language features: available") && + text.includes(`${id}: installed`) + ); + }, `the Settings row of ${alias} to list ${id} as installed`); + + const listed = JSON.parse( + await onRemote( + alias, + "cat ~/.dev/whiteboard-remote/extensions/extensions.json", + ), + ).map((entry) => entry.identifier.id); + + const others = Object.values(LANGUAGES).flatMap((other) => + other === language ? [] : other.members, + ); + + for (const member of language.members) + assert.ok(listed.includes(member), `${member} on ${alias}: ${listed}`); + assert.deepEqual( + listed.filter((installed) => others.includes(installed)), + [], + "another group on the remote", + ); + + const installed = language.installedCheck + ? (await onRemote(alias, language.installedCheck)).replace(/\s+/g, " ") + : ""; + + if (language.installedCheck) + assert.match(installed, /^rust-analyzer /, "rust-analyzer --version"); + check( + ctx, + id, + `4. ${alias} added in Settings: online with language features after ${timings.available} ms, the row says "${id}: installed"; the remote lists ${language.members.join(", ")} and no other optional group${installed ? `; ${installed}` : ""}`, + ); + + // 5. A second host without the toolchain attaches, and Settings names what is missing. + if (language.withoutToolchain) { + const bare = `wb-test-${language.withoutToolchain}`; + + await remote( + "up", + language.withoutToolchain, + ...(platform ? ["--platform", platform] : []), + ); + await remote("install", language.withoutToolchain); + await addHost(bare); + + const missing = `${tool} was not found on the login shell's PATH`; + + const bareState = await until( + async () => { + const current = await hostState(bare); + + return ( + current?.state === "online" && + current.languageGroups?.length && + current + ); + }, + `${bare} online with its language groups`, + 300000, + ); + + assert.deepEqual(bareState.languageGroups, [ + { group: id, installed: true, detail: missing }, + ]); + await until( + async () => + (await hostRow(bare).innerText()).includes( + `${id}: installed — ${missing}`, + ), + `the Settings row of ${bare} to name what is missing`, + ); + check( + ctx, + id, + `5. ${bare}, with no ${tool}: online, and its Settings row says "${id}: installed — ${missing}"`, + ); + } + + // Swift and C# do not answer hovers yet (docs/remote-hosts.md). + if (!language.hovers) return; + + // 6. The review's Diff view connects the host: a hover shows a type from the remote. + await openHome(ctx); + + const row = page + .locator("main.review-home") + .getByRole("region", { name: "Sessions", exact: true }) + .locator("tbody tr") + .filter({ hasText: title }); + + await row.waitFor({ timeout: 60000 }); + await row.getByTitle(title, { exact: true }).click(); + await visibleCanvas(page) + .getByRole("heading", { name: title }) + .waitFor({ timeout: 60000 }); + await tab(page, title).waitFor(); + + const fileName = path.basename(language.file); + + const content = await readFile( + path.join(fixtures, language.fixture, language.file), + "utf8", + ); + + const line = + content.split("\n").findIndex((text) => text.includes(language.before[0])) + + 1; + + const diffShown = Date.now(); + + await showDiff(ctx, title, fileName); + + const point = await windowCall( + ctx, + "point", + reviewId, + language.file, + line, + language.symbol, + ); + + const shown = await pointerHover( + page, + point, + language.hoverText, + FIRST_HOVER_LIMIT_MS, + ); + + assert.ok( + shown.ms !== null, + `no ${language.hoverText} hover on ${alias}: ${JSON.stringify(shown.seen)}`, + ); + timings.firstHover = Date.now() - diffShown; + assert.ok( + timings.firstHover <= FIRST_HOVER_LIMIT_MS, + `the first hover took ${timings.firstHover} ms`, + ); + + const warm = []; + + for (let i = 0; i < 3; i++) { + const result = await windowCall( + ctx, + "query", + reviewId, + language.file, + line, + language.symbol, + point.uri, + ); + + assert.match( + result.text, + language.hoverText, + `warm hover: ${JSON.stringify(result)}`, + ); + warm.push(result.ms); + } + + check( + ctx, + id, + `6. the hover on ${language.symbol} read "${shown.text.match(language.hoverText)[0]}" ${timings.firstHover} ms after the Diff click (${shown.ms} ms after the pointer arrived); warm ${warm.join(" / ")} ms`, + ); + + // 7. Go to definition opens the remote's own file, read-only. + const opened = await windowCall( + ctx, + "goto", + reviewId, + language.file, + line, + language.symbol, + ); + + const serverId = (await hostState(alias)).serverId; + + assert.match( + opened.active?.resource ?? "", + new RegExp( + `^vscode-remote://whiteboard%2B${serverId.toLowerCase()}/.*/${language.definitionFile.replaceAll(".", "\\.")}$`, + ), + JSON.stringify(opened), + ); + assert.match(opened.active.text, language.definitionText); + assert.ok( + opened.active.readonly, + `the host's ${language.definitionFile} is not read-only`, + ); + assert.ok(opened.active.label.startsWith(`${alias}: `), opened.active.label); + await windowCall(ctx, "closeModal"); + check( + ctx, + id, + `7. go to definition opened ${opened.active.label}, read-only, holding the definition`, + ); + + // 8. The extension host has the login shell's PATH, which the server was not started with; no debugger runs. + const server = JSON.parse( + await onRemote(alias, "cat ~/.dev/whiteboard-remote/server/server.json"), + ).pid; + + const host = ( + await onRemote(alias, "pgrep -f '[t]ype=extensionHost' | head -1") + ).trim(); + + const [serverPath, hostPath] = [ + await pathOf(alias, server), + await pathOf(alias, host), + ]; + + assert.ok( + hostPath.split(":").includes(language.toolDir), + `the extension host's PATH: ${hostPath}`, + ); + + const debuggers = language.debuggers.length + ? await onRemote( + alias, + `pgrep -fl '${language.debuggers.map((d) => `[${d[0]}]${d.slice(1)}`).join("|")}' || true`, + ) + : ""; + + assert.equal(debuggers, "", "a debugger runs on the remote"); + + const memory = await remoteMemory(alias); + + check( + ctx, + id, + `8. the extension host's PATH holds ${language.toolDir} (${hostPath}); the server's is ${serverPath}; ${language.debuggers.length ? `no ${language.debuggers.join(" or ")} process; ` : ""}memory in MB ${JSON.stringify(memory)}`, + ); + + // 9. The window closes; its extension host on the remote ends. + const session = await ctx.browser.newBrowserCDPSession(); + + await Promise.race([ + session.send("Browser.close").catch(() => {}), + sleep(2000), + ]); + await waitFor( + async () => (await extensionHosts(alias)) === 0, + `no extension host on ${alias} after the close`, + 300000, + ); + check( + ctx, + id, + `9. first hover ${timings.firstHover} ms (limit ${FIRST_HOVER_LIMIT_MS}); no extension host on ${alias} after the close`, + ); +} diff --git a/apps/review-desktop/scripts/e2e/remote/aws-hosts.mjs b/apps/review-desktop/scripts/e2e/remote/aws-hosts.mjs index 573a9e6ce..2d546a83c 100644 --- a/apps/review-desktop/scripts/e2e/remote/aws-hosts.mjs +++ b/apps/review-desktop/scripts/e2e/remote/aws-hosts.mjs @@ -1,6 +1,6 @@ /** Hosts on AWS: `aws-up`, and removing a run's instances, key pair and security groups. */ import { addOnce, aws, run, sleep } from "./exec.mjs"; -import { waitForSsh } from "./ssh.mjs"; +import { installFakeAgent, waitForSsh } from "./ssh.mjs"; const maxInstances = 2; @@ -221,6 +221,9 @@ export async function awsUp(runState, name, options) { 60, "test -f /run/systemd/shutdown/scheduled", ); + + if (options["fake-agent"]) await installFakeAgent(runState, host); + console.log(alias); } diff --git a/apps/review-desktop/scripts/e2e/remote/docker-hosts.mjs b/apps/review-desktop/scripts/e2e/remote/docker-hosts.mjs index 946b755eb..137f23aa5 100644 --- a/apps/review-desktop/scripts/e2e/remote/docker-hosts.mjs +++ b/apps/review-desktop/scripts/e2e/remote/docker-hosts.mjs @@ -3,9 +3,10 @@ import { createHash, randomBytes } from "node:crypto"; import { mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; import path from "node:path"; +import { stageVscodeServer } from "../../stage-vscode-server.mjs"; import { addOnce, docker, exec, run, sleep } from "./exec.mjs"; import { hostOf } from "./run-state.mjs"; -import { waitForSsh } from "./ssh.mjs"; +import { installFakeAgent, waitForSsh } from "./ssh.mjs"; const imageDir = path.join(import.meta.dirname, "image"); @@ -14,14 +15,23 @@ const packageDir = path.resolve( "../../../../../packages/review", ); +const TOOLCHAIN_IMAGES = { + rust: "rust:1.98-bookworm", + swift: "swift:6.4-noble", + dotnet: "mcr.microsoft.com/dotnet/sdk:10.0-noble", +}; + /** Tags are per run, so removing one run's tag never touches another run's containers. */ -async function buildImage(runState, { platform, image, node, shell }) { +async function buildImage( + runState, + { platform, image, node, shell, toolchain }, +) { const hash = createHash("sha256"); for (const file of ["Dockerfile", "setup.sh", "start.sh"]) hash.update(await readFile(path.join(imageDir, file))); - hash.update(JSON.stringify([platform, image, node, shell])); + hash.update(JSON.stringify([platform, image, node, shell, toolchain])); const tag = `wb-test-${runState.id}-image:${hash.digest("hex").slice(0, 12)}`; @@ -37,6 +47,8 @@ async function buildImage(runState, { platform, image, node, shell }) { `NODE=${node}`, "--build-arg", `LOGIN_SHELL=${shell}`, + "--build-arg", + `TOOLCHAIN=${toolchain}`, "-t", tag, imageDir, @@ -113,11 +125,20 @@ export async function up(runState, name, options) { if (jump && port) throw new Error("--port and --jump cannot be combined"); + if (options["fake-agent"] && auth !== "key") + throw new Error("--fake-agent needs --auth key"); + + const toolchain = options.toolchain ?? "none"; + + if (toolchain !== "none" && !TOOLCHAIN_IMAGES[toolchain]) + throw new Error(`--toolchain ${toolchain}: expected rust, swift or dotnet`); + const image = await buildImage(runState, { platform: options.platform, - image: options.image ?? "ubuntu:22.04", + image: options.image ?? TOOLCHAIN_IMAGES[toolchain] ?? "ubuntu:22.04", node: options.node ?? "24", shell: options.shell ?? "bash", + toolchain, }); const network = `wb-test-${runState.id}`; @@ -279,6 +300,8 @@ export async function up(runState, name, options) { : [], ); + if (options["fake-agent"]) await installFakeAgent(runState, host); + console.log(alias); } @@ -290,10 +313,8 @@ export function containerOf(runState, name) { return host; } -/** Packs packages/review from this worktree and installs it globally; a sealed host gets its route back meanwhile. */ -export async function install(runState, name, version) { - const host = containerOf(runState, name); - const scratch = await mkdtemp(`${runState.dir}/pack-`); +export async function packStaged(tarball, { version, runtime } = {}) { + const scratch = await mkdtemp(`${path.dirname(tarball)}/pack-`); try { await run("pnpm", [ @@ -305,9 +326,7 @@ export async function install(runState, name, version) { ]); const [packed] = (await readdir(scratch)).filter((f) => f.endsWith(".tgz")); - let tarball = path.join(scratch, packed); - - await run("tar", ["-xzf", tarball, "-C", scratch]); + await run("tar", ["-xzf", path.join(scratch, packed), "-C", scratch]); const manifestPath = path.join(scratch, "package/package.json"); const manifest = JSON.parse(await readFile(manifestPath, "utf8")); @@ -315,10 +334,26 @@ export async function install(runState, name, version) { if (version) { manifest.version = version; await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); - tarball = path.join(scratch, "stamped.tgz"); - await run("tar", ["-czf", tarball, "-C", scratch, "package"]); } + await stageVscodeServer(path.join(scratch, "package"), { runtime }); + await run("tar", ["-czf", tarball, "-C", scratch, "package"], { + env: { ...process.env, COPYFILE_DISABLE: "1" }, + }); + + return `${manifest.name}@${manifest.version}`; + } finally { + await rm(scratch, { recursive: true, force: true }); + } +} + +export async function install(runState, name, { version, runtime } = {}) { + const host = containerOf(runState, name); + const tarball = `${runState.dir}/staged-${randomBytes(3).toString("hex")}.tgz`; + + try { + const packed = await packStaged(tarball, { version, runtime }); + await docker("cp", tarball, `${host.container}:/tmp/wb-test-package.tgz`); if (host.sealed) @@ -345,8 +380,8 @@ export async function install(runState, name, version) { } await docker("exec", host.container, "rm", "/tmp/wb-test-package.tgz"); - console.log(`${manifest.name}@${manifest.version}`); + console.log(packed); } finally { - await rm(scratch, { recursive: true, force: true }); + await rm(tarball, { force: true }); } } diff --git a/apps/review-desktop/scripts/e2e/remote/fake-agent/acp-agent.mjs b/apps/review-desktop/scripts/e2e/remote/fake-agent/acp-agent.mjs new file mode 100644 index 000000000..3665aadb5 --- /dev/null +++ b/apps/review-desktop/scripts/e2e/remote/fake-agent/acp-agent.mjs @@ -0,0 +1,83 @@ +import { createInterface } from "node:readline"; + +const send = (message) => process.stdout.write(`${JSON.stringify(message)}\n`); + +const reply = (id, result) => send({ jsonrpc: "2.0", id, result }); + +const notify = (method, params) => send({ jsonrpc: "2.0", method, params }); + +const cwds = new Map(); + +let toolCalls = 0; + +createInterface({ input: process.stdin }).on("line", (line) => { + if (!line.trim()) return; + + const { id, method, params } = JSON.parse(line); + + switch (method) { + case "initialize": + return reply(id, { + protocolVersion: 1, + agentCapabilities: { + loadSession: false, + promptCapabilities: { image: false }, + }, + authMethods: [], + }); + case "session/new": { + const sessionId = `wb-test-${cwds.size + 1}`; + + cwds.set(sessionId, params.cwd); + + return reply(id, { + sessionId, + modes: { + currentModeId: "build", + availableModes: [ + { id: "build", name: "Build" }, + { id: "plan", name: "Plan" }, + ], + }, + }); + } + + case "session/set_mode": + return reply(id, {}); + case "session/prompt": { + const { sessionId, prompt } = params; + const question = prompt.findLast((block) => block.type === "text")?.text; + + notify("session/update", { + sessionId, + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: `You asked: ${question}. See \`f.ts\`.` }, + }, + }); + notify("session/update", { + sessionId, + update: { + sessionUpdate: "tool_call", + toolCallId: `t${++toolCalls}`, + title: "Read f.ts", + kind: "read", + status: "completed", + locations: [{ path: `${cwds.get(sessionId)}/f.ts`, line: 1 }], + }, + }); + + return reply(id, { stopReason: "end_turn" }); + } + + case "session/cancel": + return; + default: + if (id !== undefined) + send({ + jsonrpc: "2.0", + id, + error: { code: -32601, message: `unknown method ${method}` }, + }); + } +}); diff --git a/apps/review-desktop/scripts/e2e/remote/fake-agent/opencode b/apps/review-desktop/scripts/e2e/remote/fake-agent/opencode new file mode 100755 index 000000000..156c495ee --- /dev/null +++ b/apps/review-desktop/scripts/e2e/remote/fake-agent/opencode @@ -0,0 +1,10 @@ +#!/bin/sh +# wb-test: stands in for OpenCode. `opencode acp` speaks ACP; anything else prints a version. +case "$1" in + acp) + node=$(command -v node) + [ -n "$node" ] || for node in "$HOME"/.dev/whiteboard-remote/node/v*/bin/node; do :; done + [ -x "$node" ] || { echo "opencode (wb-test): no node on PATH or in ~/.dev/whiteboard-remote/node" >&2; exit 127; } + exec "$node" "${0%/*}/acp-agent.mjs" ;; + *) echo "opencode 0.0.0-wb-test" ;; +esac diff --git a/apps/review-desktop/scripts/e2e/remote/image/Dockerfile b/apps/review-desktop/scripts/e2e/remote/image/Dockerfile index 6ba0fff4a..cca95117d 100644 --- a/apps/review-desktop/scripts/e2e/remote/image/Dockerfile +++ b/apps/review-desktop/scripts/e2e/remote/image/Dockerfile @@ -3,6 +3,7 @@ ARG BASE=ubuntu:22.04 FROM ${BASE} ARG NODE=24 ARG LOGIN_SHELL=bash +ARG TOOLCHAIN=none COPY setup.sh start.sh /usr/local/lib/wb-test/ RUN sh /usr/local/lib/wb-test/setup.sh EXPOSE 22 diff --git a/apps/review-desktop/scripts/e2e/remote/image/setup.sh b/apps/review-desktop/scripts/e2e/remote/image/setup.sh index 8514decfd..ff70a81a0 100755 --- a/apps/review-desktop/scripts/e2e/remote/image/setup.sh +++ b/apps/review-desktop/scripts/e2e/remote/image/setup.sh @@ -1,5 +1,6 @@ #!/bin/sh # Build step: sshd, git and Node ($NODE: a major version or "none"), and the user `dev` with $LOGIN_SHELL. +# $TOOLCHAIN (rust, swift or dotnet) names the toolchain its base image carries. set -eu packages="openssh-server git curl ca-certificates procps iproute2 bash" [ "$LOGIN_SHELL" = fish ] && packages="$packages fish" @@ -9,6 +10,11 @@ if command -v apk >/dev/null; then adduser -D -s "$(command -v "$LOGIN_SHELL")" dev else export DEBIAN_FRONTEND=noninteractive + # Bullseye's security pool left the mirrors in 2026 while its index still lists it: use the image's snapshot. + if grep -qs 'VERSION_CODENAME=bullseye' /etc/os-release; then + sed -n 's|^# deb http://snapshot|deb [check-valid-until=no] http://snapshot|p' /etc/apt/sources.list > /tmp/sources.list + mv /tmp/sources.list /etc/apt/sources.list + fi apt-get update -qq apt-get install -y -qq --no-install-recommends $packages xz-utils >/dev/null rm -rf /var/lib/apt/lists/* @@ -20,6 +26,15 @@ else curl -fsSL "$dist/$file" | tar -xJ -C /usr/local --strip-components=1 --exclude CHANGELOG.md --exclude README.md --exclude LICENSE fi fi +# sshd does not pass on the image's ENV: rustup and the .NET SDK reach PATH only through the login shell, +# as a hand install leaves them. Swift is in /usr/bin. +case "$TOOLCHAIN" in + rust) printf '%s\n' 'export RUSTUP_HOME=/usr/local/rustup CARGO_HOME=/usr/local/cargo' 'export PATH="/usr/local/cargo/bin:$PATH"' >> /home/dev/.profile ;; + dotnet) + rm -f /usr/bin/dotnet + printf '%s\n' 'export DOTNET_ROOT=/usr/share/dotnet DOTNET_CLI_TELEMETRY_OPTOUT=1' 'export PATH="/usr/share/dotnet:$PATH"' >> /home/dev/.profile + ;; +esac # A `*` password field is not locked, so key logins work where sshd checks for locked accounts. echo 'dev:*' | chpasswd -e rm -f /etc/ssh/ssh_host_* diff --git a/apps/review-desktop/scripts/e2e/remote/remote.mjs b/apps/review-desktop/scripts/e2e/remote/remote.mjs index 8957ac9ee..ba45a8b7e 100644 --- a/apps/review-desktop/scripts/e2e/remote/remote.mjs +++ b/apps/review-desktop/scripts/e2e/remote/remote.mjs @@ -1,9 +1,10 @@ /** Disposable SSH remotes for live checks, in Docker or on AWS; see ../TESTING.md. */ +import path from "node:path"; import { fileURLToPath } from "node:url"; import { parseArgs } from "node:util"; import { awsUp } from "./aws-hosts.mjs"; -import { containerOf, install, up } from "./docker-hosts.mjs"; +import { containerOf, install, packStaged, up } from "./docker-hosts.mjs"; import { docker, inherit } from "./exec.mjs"; import { down, downEveryRun, downRun } from "./removal.mjs"; import { checkName, hostOf, openRun } from "./run-state.mjs"; @@ -14,17 +15,19 @@ const usage = `usage: remote.mjs up [--platform linux/amd64|linux/arm64] [--image ] [--node 24|20|none] [--auth key|password] [--banner] [--shell bash|fish] [--jump ] [--sealed] [--delay-ms ] [--no-forwarding] [--port ] - install [--version ] + [--toolchain rust|swift|dotnet] [--fake-agent] + install [--version ] [--runtime ] + pack --out [--version ] [--runtime ] ssh -- forward pause | resume | logs - aws-up [--arch x64|arm64] [--sealed] + aws-up [--arch x64|arm64] [--sealed] [--fake-agent] down | down --all | down --every-run verify-clean The run is WB_TEST_RUN, else the only one in /tmp/wbt.*.`; -const commandsWithoutName = new Set(["verify-clean", "down"]); +const commandsWithoutName = new Set(["verify-clean", "down", "pack"]); async function main(argv) { const split = argv.indexOf("--"); @@ -45,7 +48,11 @@ async function main(argv) { "delay-ms": { type: "string" }, "no-forwarding": { type: "boolean" }, port: { type: "string" }, + toolchain: { type: "string" }, + "fake-agent": { type: "boolean" }, version: { type: "string" }, + runtime: { type: "string" }, + out: { type: "string" }, arch: { type: "string" }, all: { type: "boolean" }, "every-run": { type: "boolean" }, @@ -68,7 +75,19 @@ async function main(argv) { case "aws-up": return awsUp(await openRun(true), name, values); case "install": - return install(await openRun(false), name, values.version); + return install(await openRun(false), name, { + version: values.version, + runtime: values.runtime && path.resolve(values.runtime), + }); + case "pack": + if (!values.out) throw new Error("pack needs --out "); + + return console.log( + await packStaged(path.resolve(values.out), { + version: values.version, + runtime: values.runtime && path.resolve(values.runtime), + }), + ); case "ssh": { const runState = await openRun(false); diff --git a/apps/review-desktop/scripts/e2e/remote/source-window-helpers.mjs b/apps/review-desktop/scripts/e2e/remote/source-window-helpers.mjs new file mode 100644 index 000000000..9024b584a --- /dev/null +++ b/apps/review-desktop/scripts/e2e/remote/source-window-helpers.mjs @@ -0,0 +1,166 @@ +/** Helpers the Source-window journeys run against a Source window over CDP. */ + +import assert from "node:assert/strict"; + +/** + * Runs `fn` in the Source window with its services, reached through its + * extension service instance: the journey needs no hook in the product. + */ +export async function inSource(page, fn, ...args) { + const cdp = await page.context().newCDPSession(page); + + try { + await cdp.send("Emulation.setFocusEmulationEnabled", { enabled: true }); + + const prototype = await cdp.send("Runtime.evaluate", { + expression: + 'import(globalThis._VSCODE_FILE_ROOT + "vs/workbench/services/extensions/electron-browser/nativeExtensionService.js").then((m) => m.NativeExtensionService.prototype)', + awaitPromise: true, + }); + + const { objects } = await cdp.send("Runtime.queryObjects", { + prototypeObjectId: prototype.result.objectId, + }); + + const result = await cdp.send("Runtime.callFunctionOn", { + objectId: objects.objectId, + functionDeclaration: `async function (args) { + // Subclass prototypes have the prototype in their chain too. + const service = [...this].reverse().find((s) => Object.hasOwn(s, "_instantiationService")); + const imp = (file) => import(globalThis._VSCODE_FILE_ROOT + file); + const get = async (file, id) => { + const module = await imp(file); + return service._instantiationService.invokeFunction((a) => a.get(module[id])); + }; + const value = await (${fn.toString()})({ service, get, imp }, ...args); + return value === undefined ? undefined : JSON.parse(JSON.stringify(value)); + }`, + arguments: [{ value: args }], + awaitPromise: true, + returnByValue: true, + }); + + if (result.exceptionDetails) + throw new Error( + result.exceptionDetails.exception?.description ?? + result.exceptionDetails.text, + ); + + return result.result.value; + } finally { + await cdp.detach().catch(() => {}); + } +} + +/** Types into f.ts; returns the read-only notice, after checking the text did not change. */ +export async function typeInto(source, ctx) { + const before = await inSource(source, fileText); + + await source.locator(".monaco-editor .view-lines").first().click(); + await source.keyboard.type("x"); + + const message = await ctx.until( + () => + source.evaluate(() => + [...document.querySelectorAll(".monaco-editor-overlaymessage")] + .map((e) => e.innerText.trim()) + .find(Boolean), + ), + "the read-only notice", + 10000, + ); + + assert.equal(await inSource(source, fileText), before, "typing changed f.ts"); + + return { message }; +} + +export async function quickRows(page) { + return page + .locator(".quick-input-list .monaco-list-row") + .allInnerTexts() + .catch(() => []); +} + +// The functions below run in the Source window. + +export async function windowState({ get }) { + const [editors, workspace, environment] = await Promise.all([ + get( + "vs/workbench/services/editor/common/editorService.js", + "IEditorService", + ), + get("vs/platform/workspace/common/workspace.js", "IWorkspaceContextService"), + get( + "vs/workbench/services/environment/common/environmentService.js", + "IWorkbenchEnvironmentService", + ), + ]); + + return { + authority: environment.remoteAuthority, + title: document.title, + folder: workspace.getWorkspace().folders[0]?.uri.toString(), + active: { + resource: editors.activeEditor?.resource?.toString(), + readonly: !!editors.activeEditor?.isReadonly(), + }, + }; +} + +export async function fileText({ get }) { + const models = await get( + "vs/editor/common/services/model.js", + "IModelService", + ); + + return models + .getModels() + .find((m) => m.uri.scheme === "vscode-remote" && m.uri.path.endsWith("/f.ts")) + ?.getValue(); +} + +export async function focusFile({ get, imp }, resource) { + const editors = await get( + "vs/workbench/services/editor/common/editorService.js", + "IEditorService", + ); + + const { URI } = await imp("vs/base/common/uri.js"); + + await editors.openEditor({ + resource: URI.parse(resource), + options: { pinned: true }, + }); + + return true; +} + +export async function pointAt({ get }, line, word) { + const editor = ( + await get("vs/editor/browser/services/codeEditorService.js", "ICodeEditorService") + ).getActiveCodeEditor(); + + const position = { + lineNumber: line, + column: editor.getModel().getLineContent(line).indexOf(word) + 2, + }; + + editor.revealLineInCenter(line); + + let at = editor.getScrolledVisiblePosition(position); + + for (let frame = 0; !at && frame < 120; frame++) { + await new Promise((resolve) => requestAnimationFrame(resolve)); + at = editor.getScrolledVisiblePosition(position); + } + + if (!at) throw new Error(`line ${line} never rendered`); + + const rect = editor.getDomNode().getBoundingClientRect(); + + return { + x: Math.round(rect.left + at.left), + y: Math.round(rect.top + at.top + at.height / 2), + }; +} diff --git a/apps/review-desktop/scripts/e2e/remote/ssh.mjs b/apps/review-desktop/scripts/e2e/remote/ssh.mjs index d28c5e956..67570fafe 100644 --- a/apps/review-desktop/scripts/e2e/remote/ssh.mjs +++ b/apps/review-desktop/scripts/e2e/remote/ssh.mjs @@ -145,3 +145,21 @@ export async function stopTunnels(runState, name) { state.forwards = state.forwards.filter((f) => name && f.host !== name); await runState.save(); } + +/** Puts the fake OpenCode in ~/.opencode/bin, where Ask finds it. */ +export async function installFakeAgent(runState, host) { + const dir = path.join(import.meta.dirname, "fake-agent"); + + await run("ssh", [...sshArgs(runState, host), "mkdir -p ~/.opencode/bin"]); + await run("scp", [ + "-F", + `${runState.dir}/ssh_config`, + path.join(dir, "opencode"), + path.join(dir, "acp-agent.mjs"), + `${host.alias}:.opencode/bin/`, + ]); + await run("ssh", [ + ...sshArgs(runState, host), + "chmod 755 ~/.opencode/bin/opencode ~/.opencode/bin/acp-agent.mjs", + ]); +} diff --git a/apps/review-desktop/scripts/e2e/run.mjs b/apps/review-desktop/scripts/e2e/run.mjs index 41f6325cc..13f8ce840 100644 --- a/apps/review-desktop/scripts/e2e/run.mjs +++ b/apps/review-desktop/scripts/e2e/run.mjs @@ -25,7 +25,9 @@ for (const file of (await readdir(journeysDir)) if (values.list) { console.log( - JSON.stringify(journeys.map(({ name, phase }) => ({ name, phase }))), + JSON.stringify( + journeys.map(({ name, phase, manual }) => ({ name, phase, manual })), + ), ); process.exit(0); } @@ -39,7 +41,8 @@ const selected = values.journey return found; }) : journeys.filter( - (j) => j.phase === 1 || process.env.REVIEW_E2E_NETWORK === "1", + (j) => + !j.manual && (j.phase === 1 || process.env.REVIEW_E2E_NETWORK === "1"), ); if (!values.runtime) diff --git a/apps/review-desktop/scripts/stage-review-runtime.mjs b/apps/review-desktop/scripts/stage-review-runtime.mjs index 675b38772..0dbfac8ad 100644 --- a/apps/review-desktop/scripts/stage-review-runtime.mjs +++ b/apps/review-desktop/scripts/stage-review-runtime.mjs @@ -76,6 +76,7 @@ export function requiredPackagedArtifacts(packagedRoot) { "out/vs/review/review.desktop.main.js", "out/vs/review/review.desktop.main.css", "out/vs/review/electron-utility/reviewDesktopHostMain.js", + "out/vs/review/node/reviewSshAskpassMain.js", "out/vs/review/canvas/canvas-loader.js", ...REQUIRED_RUNTIME_ENTRIES.map((entry) => path.join(RUNTIME_DIRECTORY_NAME, entry), diff --git a/apps/review-desktop/scripts/stage-vscode-server.mjs b/apps/review-desktop/scripts/stage-vscode-server.mjs new file mode 100644 index 000000000..429586ba4 --- /dev/null +++ b/apps/review-desktop/scripts/stage-vscode-server.mjs @@ -0,0 +1,200 @@ +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { + DEFAULT_REMOTE_RUNTIME, + REMOTE_BUILTIN_EXTENSIONS, +} from "./build-remote-runtime.mjs"; +import { + curatedExtensions, + openVsxUrl, + remoteExtensionIds, + remoteTargets, + targetKeyFor, +} from "./curated-extensions.manifest.mjs"; +import { downloadPinned } from "../../../packages/review/src/pinned-download.ts"; + +const DEFAULT_CACHE = path.join( + path.dirname(DEFAULT_REMOTE_RUNTIME), + "remote-server-cache", +); + +const npmTarball = (name, version) => + `https://registry.npmjs.org/${name}/-/${name.split("/").pop()}-${version}.tgz`; + +export const REMOTE_SERVER_DOWNLOADS = [ + { + url: npmTarball("@vscode/ripgrep-universal", "1.18.0"), + sha256: "76a03f429dc13a90f53b60995211a54308ab4f67b2521bac00f9c4be953dff52", + copy: { + "bin/linux-x64/rg": "out/vs/workbench/api/bin/linux-x64/rg", + "bin/linux-arm64/rg": "out/vs/workbench/api/bin/linux-arm64/rg", + LICENSE: "out/vs/workbench/api/bin/LICENSE", + }, + }, + ...[ + [ + "@parcel/watcher", + "2.5.6", + "8daa7285bee7e10bebce179007afcf07108c1b8ce9dbe5081f3db319d5453648", + ], + [ + "@parcel/watcher-linux-x64-glibc", + "2.5.6", + "43eed2f56c7b33d32172cf79513e807b4588ecc4e05d53a306d78fc6c356a42e", + ], + [ + "@parcel/watcher-linux-arm64-glibc", + "2.5.6", + "084d8a2dbea9c6a4c1b49d680abed344294f413a86a2184b7e013c1b395b30a0", + ], + [ + "detect-libc", + "2.1.2", + "270dec0fc06cff86481da8af2dd8f18dee6b602790b14ef0e1c2c18d7da39427", + ], + [ + "is-glob", + "4.0.3", + "3fe453fb193bb58f6f0505dfb1151230935380b5b55e1f9864261c2aafc1bec6", + ], + [ + "is-extglob", + "2.1.1", + "8c5d4286146ad62fc1096981700ce1c22a167708926fca01f9ca74f9bb50bc19", + ], + [ + "picomatch", + "4.0.4", + "515b5ab666558ed9a117483a310892aede54a68dd78f2d8db6604513e578571c", + ], + ].map(([name, version, sha256]) => ({ + url: npmTarball(name, version), + sha256, + copy: { ".": `node_modules/${name}` }, + })), +]; + +async function cached(cacheDir, download) { + const file = path.join( + cacheDir, + `${download.sha256}-${path.basename(download.url)}`, + ); + + fs.mkdirSync(cacheDir, { recursive: true }); + await downloadPinned(download, file); + + return file; +} + +export function remoteCuratedExtensions() { + return { + extensions: remoteExtensionIds.map((id) => { + const extension = curatedExtensions.find((e) => e.id === id); + + return { + id, + version: extension.version, + tier: extension.tier, + group: extension.group, + executables: extension.executables, + stripExtensionPack: extension.stripExtensionPack, + targets: Object.fromEntries( + remoteTargets.map((target) => { + const key = targetKeyFor(extension, target); + const pin = extension.targets[key]; + + if (!pin?.size) + throw new Error(`${id} pins no size for ${target}`); + + return [ + target, + { + universal: key === "universal", + url: + pin.url ?? + openVsxUrl({ + namespace: extension.namespace, + name: extension.name, + version: extension.version, + target: key === "universal" ? undefined : key, + }), + sha256: pin.sha256, + size: pin.size, + }, + ]; + }), + ), + }; + }), + }; +} + +export async function stageVscodeServer( + packageRoot, + { + commit, + runtime = DEFAULT_REMOTE_RUNTIME, + cacheDir = DEFAULT_CACHE, + downloads = REMOTE_SERVER_DOWNLOADS, + } = {}, +) { + const productPath = path.join(runtime, "product.json"); + + if (!fs.existsSync(productPath)) + throw new Error( + `${productPath} is missing. Build it first: node apps/review-desktop/scripts/build-remote-runtime.mjs`, + ); + + const product = JSON.parse(fs.readFileSync(productPath, "utf8")); + + if (!/^[0-9a-f]{40}$/.test(product.commit ?? "")) + throw new Error(`${productPath} has no commit`); + + if (commit && product.commit !== commit) + throw new Error( + `the remote runtime was built from ${product.commit}, not ${commit}; rebuild it with BUILD_SOURCEVERSION=${commit}`, + ); + + for (const name of REMOTE_BUILTIN_EXTENSIONS) + if (!fs.existsSync(path.join(runtime, "extensions", name, "package.json"))) + throw new Error(`the remote runtime has no built-in ${name}`); + + const destination = path.join(packageRoot, "vscode-server"); + fs.rmSync(destination, { recursive: true, force: true }); + fs.cpSync(runtime, destination, { recursive: true }); + + for (const notice of ["LICENSE.txt", "ThirdPartyNotices.txt"]) + fs.copyFileSync( + path.join(import.meta.dirname, "../code-oss", notice), + path.join(destination, notice), + ); + + const scratch = fs.mkdtempSync(path.join(os.tmpdir(), "wb-vscode-server-")); + + try { + for (const download of downloads) { + const unpacked = fs.mkdtempSync(path.join(scratch, "npm-")); + execFileSync("tar", ["-xzf", await cached(cacheDir, download), "-C", unpacked]); + + for (const [from, to] of Object.entries(download.copy)) { + fs.cpSync( + path.join(unpacked, "package", from), + path.join(destination, to), + { recursive: true }, + ); + } + } + } finally { + fs.rmSync(scratch, { recursive: true, force: true }); + } + + fs.writeFileSync( + path.join(destination, "curated.json"), + `${JSON.stringify(remoteCuratedExtensions(), null, 2)}\n`, + ); + + return destination; +} diff --git a/apps/review-desktop/scripts/stamp-remote-package.mjs b/apps/review-desktop/scripts/stamp-remote-package.mjs new file mode 100644 index 000000000..d734a5bb3 --- /dev/null +++ b/apps/review-desktop/scripts/stamp-remote-package.mjs @@ -0,0 +1,147 @@ + +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { readFileSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; + +const APP_DIR = path.resolve(import.meta.dirname, ".."); + +const PRODUCT = path.join(APP_DIR, "code-oss", "product.json"); + +const NVMRC = path.join(APP_DIR, "code-oss", ".nvmrc"); + +const TARGETS = ["linux-x64", "linux-arm64"]; + +const nodeUrl = (version, file) => `https://nodejs.org/dist/v${version}/${file}`; + +export function nodeVersionOf(nvmrc = NVMRC) { + const version = readFileSync(nvmrc, "utf8").trim(); + + if (!/^\d+\.\d+\.\d+$/.test(version)) { + throw new Error(`${nvmrc} holds ${JSON.stringify(version)}, not an exact Node version`); + } + + return version; +} + +async function fetchText(url) { + const response = await fetch(url, { signal: AbortSignal.timeout(60_000) }); + + if (!response.ok) throw new Error(`${url} answered ${response.status}`); + + return response.text(); +} + +export async function remotePin({ tarball, nodeVersion = nodeVersionOf(), fetchText: get = fetchText }) { + const manifest = JSON.parse( + execFileSync("tar", ["-xzOf", tarball, "package/package.json"], { encoding: "utf8" }), + ); + + const shasums = await get(nodeUrl(nodeVersion, "SHASUMS256.txt")); + const node = { version: nodeVersion }; + + for (const target of TARGETS) { + const file = `node-v${nodeVersion}-${target}.tar.xz`; + const line = shasums.split("\n").find((entry) => entry.trim().split(/\s+/)[1] === file); + + if (!line) throw new Error(`SHASUMS256.txt for Node ${nodeVersion} has no line for ${file}`); + node[target] = { url: nodeUrl(nodeVersion, file), sha256: line.trim().split(/\s+/)[0] }; + } + + const pin = { + package: { + name: manifest.name, + version: manifest.version, + integrity: `sha512-${createHash("sha512").update(readFileSync(tarball)).digest("base64")}`, + }, + node, + }; + + checkPin(pin); + + return pin; +} + +function checkPin(pin) { + const fail = (what) => { + throw new Error(`the remote pin's ${what} is malformed`); + }; + + if (pin?.package?.name !== "@dev.fast/whiteboard") fail("package name"); + + if (!/^\d+\.\d+\.\d+(-[0-9A-Za-z.-]+)?$/.test(pin.package.version ?? "")) fail("package version"); + + if (!/^sha512-[A-Za-z0-9+/]{86}==$/.test(pin.package.integrity ?? "")) fail("package integrity"); + const version = pin.node?.version ?? ""; + + if (!/^\d+\.\d+\.\d+$/.test(version)) fail("Node version"); + + for (const target of TARGETS) { + const entry = pin.node[target]; + + if (entry?.url !== nodeUrl(version, `node-v${version}-${target}.tar.xz`)) fail(`${target} URL`); + + if (!/^[0-9a-f]{64}$/.test(entry.sha256 ?? "")) fail(`${target} checksum`); + } +} + +export function writePin(productPath, json) { + let pin; + + try { + pin = JSON.parse(json); + } catch { + throw new Error("the remote pin is not JSON"); + } + + checkPin(pin); + const product = JSON.parse(readFileSync(productPath, "utf8")); + product.whiteboardRemote = pin; + writeFileSync(productPath, `${JSON.stringify(product, null, "\t")}\n`); +} + +export async function stampRemotePackage({ productPath = PRODUCT, ...options }) { + const pin = await remotePin(options); + writePin(productPath, JSON.stringify(pin)); + + return pin; +} + +async function main() { + const { values, positionals } = parseArgs({ + allowPositionals: true, + options: { + product: { type: "string", default: PRODUCT }, + "node-version": { type: "string" }, + pinned: { type: "string" }, + }, + }); + + if (values.pinned !== undefined) { + writePin(values.product, values.pinned); + + return; + } + + if (positionals.length !== 1) { + console.error( + "usage: stamp-remote-package.mjs [--product ] [--node-version ]\n" + + " stamp-remote-package.mjs --pinned [--product ]", + ); + process.exit(2); + } + + const pin = await stampRemotePackage({ + tarball: positionals[0], + productPath: values.product, + nodeVersion: values["node-version"] ?? nodeVersionOf(), + }); + + console.log(JSON.stringify(pin)); +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + await main(); +} diff --git a/apps/review-desktop/scripts/stamp-remote-package.test.mjs b/apps/review-desktop/scripts/stamp-remote-package.test.mjs new file mode 100644 index 000000000..e9767a673 --- /dev/null +++ b/apps/review-desktop/scripts/stamp-remote-package.test.mjs @@ -0,0 +1,129 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { after, test } from "node:test"; + +import { remotePin, stampRemotePackage, writePin } from "./stamp-remote-package.mjs"; + +const roots = []; + +after(async () => { + await Promise.all( + roots.map((root) => rm(root, { recursive: true, force: true })), + ); +}); + +const X64 = "a".repeat(64); + +const ARM64 = "b".repeat(64); + +function shasums(version, lines = ["linux-x64", "linux-arm64"]) { + const sums = { "linux-x64": X64, "linux-arm64": ARM64 }; + + return [ + `${"c".repeat(64)} node-v${version}-darwin-arm64.tar.xz`, + `${"d".repeat(64)} node-v${version}-linux-x64.tar.gz`, + ...lines.map((target) => `${sums[target]} node-v${version}-${target}.tar.xz`), + "", + ].join("\n"); +} + +async function fixture() { + const root = await mkdtemp(path.join(os.tmpdir(), "wb-stamp-remote-")); + roots.push(root); + await mkdir(path.join(root, "package")); + await writeFile( + path.join(root, "package", "package.json"), + '{"name":"@dev.fast/whiteboard","version":"0.1.6"}\n', + ); + const tarball = path.join(root, "dev.fast-whiteboard-0.1.6.tgz"); + execFileSync("tar", ["-czf", tarball, "-C", root, "package"]); + const productPath = path.join(root, "product.json"); + await writeFile( + productPath, + `${JSON.stringify({ nameShort: "Whiteboard", quality: "stable" }, null, "\t")}\n`, + ); + + return { root, tarball, productPath }; +} + +test("pins the tarball's integrity and Node's checksums into product.json", async () => { + const { tarball, productPath } = await fixture(); + const fetched = []; + + const pin = await stampRemotePackage({ + tarball, + productPath, + nodeVersion: "24.18.0", + fetchText: async (url) => { + fetched.push(url); + + return shasums("24.18.0"); + }, + }); + + const expected = `sha512-${createHash("sha512").update(await readFile(tarball)).digest("base64")}`; + const product = JSON.parse(await readFile(productPath, "utf8")); + assert.deepEqual(fetched, [ + "https://nodejs.org/dist/v24.18.0/SHASUMS256.txt", + ]); + assert.deepEqual(product.whiteboardRemote, { + package: { + name: "@dev.fast/whiteboard", + version: "0.1.6", + integrity: expected, + }, + node: { + version: "24.18.0", + "linux-x64": { + url: "https://nodejs.org/dist/v24.18.0/node-v24.18.0-linux-x64.tar.xz", + sha256: X64, + }, + "linux-arm64": { + url: "https://nodejs.org/dist/v24.18.0/node-v24.18.0-linux-arm64.tar.xz", + sha256: ARM64, + }, + }, + }); + assert.deepEqual(pin, product.whiteboardRemote); + assert.equal(product.nameShort, "Whiteboard"); +}); + +test("fails when SHASUMS256.txt has no line for a target, and leaves product.json alone", async () => { + const { tarball, productPath } = await fixture(); + const before = await readFile(productPath, "utf8"); + + await assert.rejects( + stampRemotePackage({ + tarball, + productPath, + nodeVersion: "24.18.0", + fetchText: async () => shasums("24.18.0", ["linux-x64"]), + }), + /node-v24\.18\.0-linux-arm64\.tar\.xz/, + ); + assert.equal(await readFile(productPath, "utf8"), before); +}); + +test("a pin from earlier in the run is written as it is, and a malformed one is refused", async () => { + const { tarball, productPath } = await fixture(); + + const pin = await remotePin({ + tarball, + nodeVersion: "24.18.0", + fetchText: async () => shasums("24.18.0"), + }); + + writePin(productPath, JSON.stringify(pin)); + assert.deepEqual( + JSON.parse(await readFile(productPath, "utf8")).whiteboardRemote, + pin, + ); + + const broken = { ...pin, package: { ...pin.package, integrity: "sha1-abc" } }; + assert.throws(() => writePin(productPath, JSON.stringify(broken)), /integrity/); + assert.throws(() => writePin(productPath, ""), /JSON/); +}); diff --git a/docs/remote-hosts.md b/docs/remote-hosts.md new file mode 100644 index 000000000..3f2739757 --- /dev/null +++ b/docs/remote-hosts.md @@ -0,0 +1,336 @@ +# Remote hosts + +A remote host is a Linux machine that you reach over SSH, such as a +development server or a cloud VM, where your code and your coding agent live. +Whiteboard runs its review server there, next to the code. Whiteboard Desktop +on your laptop connects to it over SSH and shows its reviews beside your local +ones: in Home, each one is labelled with the host, as `devbox: my-repo`. + +Whiteboard uses your own SSH configuration. A host is an alias from it, such +as `devbox`. User names, keys, ports, jump hosts and agents all come from that +configuration; Whiteboard stores only the alias. + +## What the remote needs + +- Linux on x64 or ARM64, with glibc 2.34 or newer (Ubuntu 22.04, Debian 12, + Fedora 35, RHEL 9 or later). +- 1 GB free in your home directory, which must be writable. +- `sha512sum` or `openssl`. If the host has no Node 24, also `tar`, `xz` and + `sha256sum`, so that Desktop can install one. +- An OpenSSH server that allows TCP forwarding. See + [TCP forwarding](#tcp-forwarding). + +Nothing else: not Node, not root access, and not an internet connection. +Desktop installs Whiteboard there itself, as your user. See +[How Desktop installs Whiteboard](#how-desktop-installs-whiteboard). + +Your laptop needs macOS or Linux with the OpenSSH client, and an internet +connection while it installs on a host that has none. A Windows laptop is not +supported yet. + +## Add a host + +Remote hosts are experimental. Turn them on in Whiteboard's `settings.json`: + +```json +"review.experimental.remoteHosts.enabled": true +``` + +Then open Settings. Under **Remote hosts**, type an alias from your SSH +configuration (the field suggests them) and click **Add**. The row shows the +host's state. The first time, Desktop asks to install Whiteboard there. When +the row reads `online`, the host's reviews appear in Home. + +**Remove** takes the host and its reviews out of Whiteboard. Nothing on the +remote is deleted, unless you also check **Also remove Whiteboard**. See +[Remove Whiteboard from a host](#remove-whiteboard-from-a-host). + +The hosts are kept in the setting `review.remote.hosts`, a list of aliases. +Only you can change it: a repository or a remote never adds a host. + +If `ssh` asks for a password, a passphrase or a host key confirmation, +Whiteboard shows the prompt in its window. Your answer goes to `ssh` and is +never stored. + +## How Desktop installs Whiteboard + +When Desktop connects to a host, it first checks the machine: its system, +CPU, glibc, home directory, free space, Node, and which Whiteboard versions it +already has. This check only reads, except that it runs `node --version` on +each Node it finds, and a version manager's shim (such as Volta's) may fetch +its Node when run. + +- **A machine Whiteboard cannot run on** shows `unsupported`, with the + reason, such as "This host runs glibc 2.31; Whiteboard needs 2.34 or + newer." Desktop writes nothing on it. +- **A machine without Desktop's version** gets a question in Desktop's + window: "Install Whiteboard on ``?", with the space it takes. Desktop + asks once per host and remembers the answer until you remove Whiteboard + from that host. + - **Install:** Settings shows each step (preparing, installing Node 24, + installing the Whiteboard package, checking the install, starting the + server), and then the host is `online`, usually well within a minute. + - **Don't install:** the host shows `not-installed`, with the command to + install by hand, and an **Install** button for when you change your mind. + Reconnecting does not ask again. +- **A machine with Desktop's version** is attached at once. + +Desktop installs exactly its own version of the `@dev.fast/whiteboard` +package, and a Node 24 when the host has none. Each release of Desktop pins +both, with their checksums, and the host checks them before it installs +anything. The host downloads them when it can reach the internet. When it +cannot, Desktop downloads them on the laptop and uploads them through the SSH +connection, and the host's npm fetches the package's dependencies through a +relay on the laptop, reached over that same connection only while it installs. +Desktop never uses `sudo`. The install writes only the paths below; npm's +cache and log stay in the unfinished version's directory and go with it. The +review server it starts writes under `~/.dev`, as below, and **Connect** +writes the agent's own configuration. + +A Node 24 already on the host is used, and then Desktop installs no Node: one +on the `PATH` of a non-login shell, in `/usr/local/bin` or `/usr/bin`, or under +a version manager such as nvm, fnm, volta, asdf or mise. A Node only on your +login shell's `PATH` is not seen. + +If the connection drops while installing, the next connection starts the +install again; nothing half-written is left. Two Desktops installing on one +host at once take turns. + +### What Desktop writes on the remote + +| Path | What it is | +|---|---| +| `~/.dev/whiteboard-remote/versions//` | One Whiteboard version: the package, its dependencies and the host's `diffr` for structural diff, a `whiteboard` launcher and a marker file that records what was checked. | +| `~/.dev/whiteboard-remote/node/v/` | Node 24, only when the host has none. | +| `~/.dev/whiteboard-remote/install.lock/` | Present while an install runs. | +| `~/.local/bin/whiteboard` | A launcher for the newest version, so that you and your agents can run `whiteboard` there. Desktop writes it only if that path is free or Desktop's own; a `whiteboard` you installed yourself is left alone. | + +With `DEV_REVIEW_HOME` set in the remote's environment, `whiteboard-remote/` +is under that directory instead of `~/.dev`, as is everything below that +Desktop and the VS Code server keep there. + +As before, the review server keeps its reviews in `~/.dev` +(`review-api.db` and its companion files). + +### Updates + +When Desktop updates, it installs its new version beside the old one without +asking again, and restarts the review server it started on the new version. +Your reviews stay. Desktop keeps two versions on a host, and any older one +that a running process still uses; it removes the rest after the next +install. If the new version fails to install but a `whiteboard` of that version +on `PATH` attaches, the host is `online` and Settings does not show the +failure; the next connection tries the install again. + +Desktop does not stop a review server that you started yourself +(`whiteboard server start`). If that server runs another version, the host is +`incompatible` until you stop it. A newer server, started by a newer Desktop, +is also left running, and this Desktop asks you to update. + +### The setting + +`review.remote.install` decides whether Desktop installs on a host that does +not have its version: + +| Value | What Desktop does | +|---|---| +| `ask` (the default) | Asks once per host. | +| `always` | Installs without asking. | +| `never` | Never installs. It uses the `whiteboard` you installed on the host's `PATH`, and shows `not-installed` with the command when there is none. | + +### Install by hand + +With `never`, or after **Don't install**, install the version that matches +your Desktop on the remote yourself: + +```sh +npm install -g @dev.fast/whiteboard@ +``` + +This needs Node 24 there. Settings shows the exact command, with the version. +You do not need to start anything: Desktop starts the review server when it +connects. On a remote with no Desktop, `whiteboard api` starts one too, so an +agent there can write reviews before you connect. + +### A remote without internet access + +It needs nothing more. Desktop uploads Whiteboard and Node from the laptop and +relays npm through the SSH connection. + +## Agents on the remote + +Once a host is `online`, Desktop looks for coding agents there: Claude Code, +Codex, OpenCode and Pi, by their configuration directories. Looking changes +nothing. Settings lists the agents that are not yet connected to Whiteboard, +with **Connect**, which runs that agent's own install commands on the host, +the same ones `whiteboard connect` gives. An agent whose command is not on the +login shell's `PATH` gets the text to paste into it instead. + +**Ask.** Ask works on a review that lives on a host. The agent runs on the +host, in the review's checkout, with the agents and sign-ins already there +(`claude auth login`, `codex login`, `opencode auth login`, +`cursor-agent login`, Pi's `/login`). Threads are kept on the host, so every +laptop that attaches it sees the same history for its reviews. A file that an +answer names opens in a Source window on the host; see +[Source windows](#source-windows). + +## Remove Whiteboard from a host + +In Settings, click **Remove** on the host, check **Also remove Whiteboard from +``**, and click **Remove host**. Desktop runs the host's own +`whiteboard remote uninstall --keep-reviews` over the connection, which: + +- stops the review server that Desktop or the CLI started, and the VS Code + server and any extension download that `remote attach` started; +- removes `~/.dev/whiteboard-remote/`, and `~/.local/bin/whiteboard` if + Desktop wrote it; +- keeps your reviews in `~/.dev`. + +Desktop also forgets your answer to the install question, so adding the host +again asks again. + +It refuses, and removes nothing, while an install runs there, or while a +server you started yourself or another process (such as an agent's MCP +server) runs from the install. Settings shows why; stop that process and try +again. The host is removed from Desktop either way. + +On the remote, `whiteboard remote uninstall --keep-reviews` does the same, and +`--delete-reviews` also deletes the saved reviews. + +Without the box, **Remove** only takes the host out of Desktop; nothing on the +remote changes, and Desktop keeps your answer. + +## States + +| State | Meaning | What to do | +|---|---|---| +| `connecting` | Whiteboard is opening the SSH connection and starting the review server. | Wait. | +| `installing` | Desktop is installing its version there; the detail names the step. | Wait. | +| `online` | Connected. The host's reviews are listed and open. | Nothing. | +| `offline` | The connection is up, but the review server did not answer. Whiteboard checks every 10 seconds, so a host that hangs shows `offline` within about 15 seconds. | It returns by itself when the server answers. After three checks in a row without an answer, Whiteboard attaches again. A hung server then shows `unreachable`, and its detail names `whiteboard server stop`: run it on the host. | +| `unreachable` | `ssh` could not connect, the connection ended, or the review server there did not start (the detail says why). | Whiteboard tries again by itself, waiting 1 to 60 seconds between tries; **Retry** tries now. Check that `ssh ` works in a terminal, or do what the detail says. | +| `auth-failed` | The login was refused, a prompt was cancelled, or the host key did not match. | Fix the login, then click **Retry**. Whiteboard does not retry this by itself. | +| `not-installed` | You declined the install, `review.remote.install` is `never` and `whiteboard` was not found, or an install failed (the detail says where). | After **Don't install**, click **Install**; otherwise install by hand with the command that Settings shows and click **Retry**. Whiteboard does not retry this by itself. | +| `unsupported` | Whiteboard cannot run on this machine; the detail says why. Nothing was written there. | Use another machine. | +| `incompatible` | The remote runs another version of Whiteboard: a server you started yourself, a newer one started by a newer Desktop, or a version you installed by hand. | Stop your server (`whiteboard server stop`) and click **Retry**, update this Desktop, or install the version that Settings shows. | +| `duplicate` | Two hosts report the same server id. This happens when a review store was copied to a second machine. | If both aliases are one machine, remove one of them. If they are two machines, remove the copy's host, run `whiteboard server stop` and then `whiteboard server reset-id` on the copy, and add it again. | + +While a host is not `online`, its reviews stay in Home, drawn as unavailable. +Opening one says why. + +## Language features + +Hover and go to definition work in a remote review's code, answered on the +remote by the same language extensions a laptop review uses. Desktop runs a +VS Code server and one extension host on each remote it connects to. + +- **Which languages:** TypeScript, JavaScript, JSON, CSS and HTML always. + Python, with ty, Ruff and the Python extension. Go, Rust, Swift and C# when + you have turned their group on in Whiteboard (Settings → Tools → Extensions); + see [Optional languages](#optional-languages). +- **First use:** the remote downloads its language extensions from Open VSX + (`open-vsx.org`) the first time Desktop connects. That takes a few seconds + to a minute, and the remote needs network access to Open VSX for it. A + remote without that access still shows its reviews, without hovers. +- **Memory:** plan on about 1 GB for the VS Code server, its extension host and + the language servers of one TypeScript and one Python project; about 0.8 GB + with TypeScript alone. The server exits 5 minutes after the last window + leaves. +- **Same version:** language features need the same Whiteboard version on + both ends. Otherwise the host stays `online` and its reviews open, and + Settings says under the host why language features are unavailable. + +Settings shows "Language features: available" or why not for each online host. + +Extensions on a remote are trusted the way VS Code Remote trusts them: they +run on that machine and can do in the window what a local extension can. The +code in a review stays read-only. + +An extension on a remote that asks for call or type hierarchies may also get +answers about another remote's files. + +### Optional languages + +A group you turn on in Whiteboard is installed on each remote at its next +connection, at the same version as on your laptop. A group you have not turned +on is never installed on a remote. Turning a group off leaves it on the remote, +unused. Each group needs its toolchain on the remote, on the `PATH` of your +login shell (`~/.profile` or your shell's own start-up file is enough): + +| Group | Needs on the remote | Extensions the remote downloads | +|---|---|---| +| Go | `go` | Go | +| Rust | `cargo` and `rustc` | rust-analyzer, about 16 MB | +| Swift | `swift` | Swift and LLDB DAP, about 16 MB | +| C# | `dotnet` (a .NET SDK) | C# and .NET Runtime, about 80 MB | + +- When a toolchain is missing, the host still connects. Settings says under + the host which tool the login shell could not find, for example + "swift: installed — swift was not found on the login shell's PATH". +- **Memory:** a small Rust project needs about 1 GB for the VS Code server, + its extension host and rust-analyzer. +- **Rust** gives hover and go to definition on remotes. The glibc a remote + needs for Whiteboard (2.34 or newer) is enough for rust-analyzer. +- **Swift and C#** are installed on a remote but do not answer hovers yet, + on a remote or on your laptop. Swift's extension needs the task API, which + Whiteboard does not expose yet, and on a remote also `node-pty`, which the + remote's VS Code server does not include. In a review's Diff view the C# extension + loads the project from the review's base side only, so a hover on the + changed side stays at "Loading...". +- The debuggers in the Swift and C# groups are never started: reviews are + read-only. +- The remote downloads each extension from Open VSX itself, and checks it + against the checksum Whiteboard pins. Whiteboard redistributes none of them. + The C# extension downloads its OmniSharp server from Microsoft the first + time it starts. + +## Source windows + +"Open file", the source tree and Ask's file links open a Source window bound +to the host. The window shows the host's checkout read-only, with the +explorer, quick open, text search and the language features above. Its title +is ` — Source — Whiteboard`. + +When the host stops answering, the status bar reads ` — offline, +reconnecting…` within about 15 seconds, and a warning says the window +reconnects when the host is back. A window restored while its host is down +shows the same. The window reconnects by itself, usually within a few seconds +of the host returning, and the entry goes back to the alias. + +Not available in a Source window: + +- Terminals and source control. +- Watching on musl hosts such as Alpine. The explorer refreshes when the + window gets focus. On glibc hosts it updates as files change. + +Each Source window runs its own extension host on the host, about 130 MB with +TypeScript. Close windows you are done with. + +## Not available for remote reviews yet + +- Sharing. +- Traces. +- Scratchpads. The scratchpad is always the laptop's. + +Also: + +- `whiteboard server stop` on a remote is undone within about 10 seconds while + Desktop is connected to it, because Desktop starts the server again. To stop + it for good, remove the host first. +- Every Desktop connected to a remote receives the reviews its agent opens. + +## TCP forwarding + +Whiteboard reaches the remote's review server through an SSH port forward +(`ssh -L`) to a port on the remote's loopback interface. The server never +listens on a public address. So the remote's `sshd` must allow TCP +forwarding, which is OpenSSH's default. If `sshd_config` sets +`AllowTcpForwarding no`, the host shows `unreachable`, and the detail quotes +OpenSSH's "administratively prohibited" message. + +Language features use a second forward, to the remote's VS Code server on its +loopback interface. That server has its own connection token, new each time it +starts. Unlike the review server's token, it reaches the Desktop window: the +window connects to the VS Code server itself. Desktop hands it only to a window +that asks for an `online` host running the same Whiteboard version, never on a +command line, and never writes it to a log. diff --git a/packages/review-protocol/src/contracts.ts b/packages/review-protocol/src/contracts.ts index e169540df..e693577fc 100644 --- a/packages/review-protocol/src/contracts.ts +++ b/packages/review-protocol/src/contracts.ts @@ -261,6 +261,7 @@ export interface ReviewDiffViewSpec { onToggleSection?: (id: string) => void; lens?: ReviewDiffLens; scope?: ReviewCommitScope; + openFile?: boolean; } export interface ReviewDiffViewHandle extends ReviewDisposable { @@ -583,6 +584,57 @@ export interface ReviewCanvasSettingsContent { // has reviews and no longer shows the Welcome rail. Absent when the // install status endpoint is unavailable. install?: ReviewCanvasInstallContent; + remoteHosts: ReviewRemoteHostsSettings; +} + +export interface ReviewRemoteHostsSettings { + enabled: boolean; + configured: string[]; + suggestions(): Promise; + states(): Promise; + set(aliases: string[]): Promise; + retry(alias: string): Promise; + install(alias: string): Promise; + agents(alias: string): Promise; + connectAgents( + alias: string, + agents: ReviewRemoteAgentId[], + ): Promise; + uninstall(alias: string): Promise; +} + +export const REVIEW_REMOTE_WRAPPER_MARK = + "# Written by Whiteboard Desktop, which replaces it with each install."; + +export const REVIEW_REMOTE_ATTACH_BEGIN = "WHITEBOARD-REMOTE-BEGIN"; + +export const REVIEW_REMOTE_ATTACH_END = "WHITEBOARD-REMOTE-END"; + +export const REVIEW_REMOTE_VERSION = /^\d+\.\d+\.\d+(-[0-9A-Za-z.-]+)?$/; + +export const REVIEW_REMOTE_INSTALL_LOCK = "install.lock"; + +export const REVIEW_REMOTE_LOCK_STALE_SECONDS = 15 * 60; + +export const REVIEW_REMOTE_AGENT_IDS = [ + "claude", + "codex", + "opencode", + "pi", +] as const satisfies readonly ReviewCliInstallTarget[]; + +export type ReviewRemoteAgentId = (typeof REVIEW_REMOTE_AGENT_IDS)[number]; + +export interface ReviewRemoteAgent { + id: ReviewRemoteAgentId; + connected: boolean; + manual?: true; +} + +export interface ReviewRemoteAgentResult { + id: ReviewRemoteAgentId; + connected: boolean; + output: string; } /** Workspace attachment identity is independent of the displayed source generation. */ @@ -594,6 +646,13 @@ export interface ReviewLanguageEnvironment { readonly issue?: string; } +export interface ReviewRemoteLanguageEnvironment { + readonly remoteRootPath: string | null; + readonly identity: string; + readonly serverId: string; + readonly issue?: string; +} + /** Authored version selection is independent of whether source is live or fixed. */ export type ReviewSourceSelection = | { readonly reviewId: string; readonly kind: "current" } @@ -692,6 +751,8 @@ export type ReviewCanvasContent = documentWidth?: ReviewDocumentWidthChoice; reviewId: string; version?: number; + host?: string; + available?: ReviewApiSummary["available"]; bridge: ReviewCanvasBridge; setTitle?(title: string): void; setSourceView?( @@ -724,6 +785,7 @@ export type ReviewCanvasContent = // not support them. dismissReview?(uuid: string): Promise; restoreReview?(uuid: string): Promise; + hostStates?(): Promise; // Opens the review and pins its read-only source tree open. Absent when // the host cannot show the tree. openSourceTree?(uuid: string): void; @@ -848,6 +910,87 @@ export interface ReviewServerHealthWithToken extends ReviewServerHealth { commit: string | null; } +export const REVIEW_REMOTE_INSTALL_STEPS = [ + "preparing", + "waiting-for-lock", + "node", + "package", + "verifying", + "done", +] as const; + +export const ReviewGatewayHostSchema = z.strictObject({ + alias: requiredString, + endpoint: z + .strictObject({ url: requiredString, token: requiredString }) + .optional(), + problem: z + .strictObject({ + state: z.enum([ + "unreachable", + "not-installed", + "auth-failed", + "unsupported", + "incompatible", + ]), + detail: stringAllowEmpty, + }) + .optional(), + declined: z.literal(true).optional(), + asking: requiredString.optional(), + installFailure: stringAllowEmpty.optional(), + installing: z + .strictObject({ + step: z.enum(REVIEW_REMOTE_INSTALL_STEPS), + detail: stringAllowEmpty.optional(), + }) + .optional(), + languageFeatures: z.boolean().optional(), + languageFeaturesDetail: stringAllowEmpty.optional(), + languageGroups: z + .array( + z.strictObject({ + group: requiredString, + installed: z.boolean(), + detail: stringAllowEmpty.optional(), + }), + ) + .optional(), +}); + +export type ReviewGatewayHost = z.infer; + +export interface ReviewGatewayHostState { + alias: string; + serverId?: string; + state: + | "connecting" + | "online" + | "offline" + | "incompatible" + | "duplicate" + | "unreachable" + | "not-installed" + | "auth-failed" + | "unsupported" + | "installing"; + detail?: string; + installCommand?: string; + languageFeatures?: boolean; + languageFeaturesDetail?: string; + languageGroups?: ReviewGatewayHost["languageGroups"]; + declined?: true; +} + +/** `POST /:id/navigator` for a review on another machine: its host paths as + * `vscode-remote` URIs. `emptySide`: the diff side has no file; open one locally. */ +export interface ReviewRemoteNavigatorAnswer { + workspaceUri: string; + fileUri?: string; + remoteAuthority: string; + emptySide?: true; +} + export const ReviewRepositoryIdentitySchema = z.strictObject({ kind: z.enum(["git", "jj", "none"], { error: "must be git, jj, or none", @@ -957,6 +1100,19 @@ export type ReviewCliInstallTarget = z.infer< typeof ReviewCliInstallTargetSchema >; +export const REVIEW_CLI_INSTALL_TARGET_LABELS: Record< + ReviewCliInstallTarget, + string +> = { + claude: "Claude Code", + codex: "Codex", + cursor: "Cursor", + opencode: "OpenCode", + pi: "Pi", + omp: "oh-my-pi", + copilot: "Copilot CLI", +}; + export const ReviewCliInstallStampSchema = z.object({ consent: z.enum(["granted", "declined", "skipped"], { error: "must be granted, declined, or skipped", diff --git a/packages/review-protocol/src/review-api-client.ts b/packages/review-protocol/src/review-api-client.ts index fcef77d69..2f80e2b5d 100644 --- a/packages/review-protocol/src/review-api-client.ts +++ b/packages/review-protocol/src/review-api-client.ts @@ -1,4 +1,6 @@ -import type { JsonValue } from "@dev.fast/json"; +import type { JsonObject } from "@dev.fast/json"; + +import type { ReviewGatewayHostState } from "./contracts.js"; /** List metadata for the authenticated local catalog; document contents stay in snapshots. */ export interface ReviewApiSummary { @@ -45,8 +47,26 @@ export interface ReviewApiSummary { dismissedAt: string | null; /** An agent holds a live lease; absent on shared reviews. */ working?: boolean; + host?: string; + hostState?: Exclude; + available?: { sourceWindows: boolean; languageFeatures: boolean }; } +export type ReviewStreamSnapshot = + Snapshot & { + activity: { workingCount: number; expiresAt: number | null }; + coverageRevision: number; + }; + +export type ReviewStreamLine = + | { + kind: "list"; + mode: "structural" | "textual"; + reviews: ReviewApiSummary[]; + } + | { kind: "review"; reviewId: string; value: ReviewStreamSnapshot } + | { kind: "review"; reviewId: string; error: string }; + export interface ReviewSourceEntry { path: string; kind: "file" | "directory"; @@ -119,16 +139,12 @@ export class ReviewApiClient { }) ).json(); } - async *watch( - reviewId: string | null | Subscription[], + async *watch( + subscriptions: Subscription[], signal: AbortSignal, - ): AsyncGenerator { + ): AsyncGenerator { const response = await this.response( - Array.isArray(reviewId) - ? `/watch?subscriptions=${encodeURIComponent(JSON.stringify(reviewId))}` - : reviewId === null - ? "/watch" - : `/${encodeURIComponent(reviewId)}/watch`, + `/watch?subscriptions=${encodeURIComponent(JSON.stringify(subscriptions))}`, { signal }, ); @@ -154,8 +170,8 @@ export class ReviewApiClient { let end: number; while ((end = pending.indexOf("\n")) !== -1) { - // SAFETY: the authenticated host serializes the snapshot type requested by this caller. - yield JSON.parse(pending.slice(0, end)) as T; + // SAFETY: the authenticated host writes ReviewStreamLine values. + yield JSON.parse(pending.slice(0, end)) as ReviewStreamLine; pending = pending.slice(end + 1); } } @@ -204,15 +220,19 @@ export class ReviewApiClient { } } -type Result = { value?: JsonValue; error?: string } | null; +type Result = + | { value: ReviewStreamSnapshot | ReviewApiSummary[] } + | { error: string }; type Listener = { subscription: Subscription; signal: AbortSignal; - accept(value: JsonValue | undefined): void | Promise; + accept( + value: ReviewStreamSnapshot | ReviewApiSummary[], + ): void | Promise; disconnected(cause: unknown): void; /** The newest undelivered result; replaced rather than queued while a render runs. */ - queued?: Exclude; + queued?: Result; draining?: boolean; }; @@ -234,6 +254,13 @@ function sleep(ms: number, signal: AbortSignal) { }); } +function concerns(line: ReviewStreamLine, subscription: Subscription) { + return line.kind === "list" + ? subscription.reviewId === null && + (subscription.mode ?? "structural") === line.mode + : subscription.reviewId === line.reviewId; +} + function report(listener: Listener, cause: unknown) { if (listener.signal.aborted) return; @@ -320,19 +347,17 @@ class LiveConnection { while (!signal.aborted) { try { - for await (const results of this.client.watch( + for await (const line of this.client.watch( listeners.map((item) => item.subscription), signal, )) { if (signal.aborted) break; delay = 1000; + const result = line.kind === "list" ? { value: line.reviews } : line; - listeners.forEach((listener, index) => { - const result = results[index]; - - // null marks a subscription unchanged since the previous line. - if (result) this.deliver(listener, result); - }); + for (const listener of listeners) + if (concerns(line, listener.subscription)) + this.deliver(listener, result); } if (!signal.aborted) disconnected(new Error("Connection closed.")); @@ -352,7 +377,7 @@ class LiveConnection { } /** Renders never block the stream: a slow tab only delays its own newest state. */ - private deliver(listener: Listener, result: Exclude) { + private deliver(listener: Listener, result: Result) { listener.queued = result; if (listener.draining) return; @@ -363,7 +388,7 @@ class LiveConnection { const next = listener.queued; listener.queued = undefined; - if (next.error) { + if ("error" in next) { report(listener, new Error(next.error)); continue; } diff --git a/packages/review/THIRD_PARTY_NOTICES.md b/packages/review/THIRD_PARTY_NOTICES.md index d6594bd66..c1789c98a 100644 --- a/packages/review/THIRD_PARTY_NOTICES.md +++ b/packages/review/THIRD_PARTY_NOTICES.md @@ -91,3 +91,32 @@ Direct third-party runtime/UI dependencies currently include: - `semver` (`ISC`) - `zod` (`MIT`) - `zustand` (`MIT`) +- `yauzl` (`MIT`) + +## Bundled VS Code Server + +The published package carries `vscode-server/`, staged by +`apps/review-desktop/scripts/stage-vscode-server.mjs`. It is built from the +Code OSS fork (`MIT`) and carries the fork's `LICENSE.txt` and +`ThirdPartyNotices.txt`. It bundles: + +- in the server and extension host (`MIT` unless noted): `@vscode/proxy-agent`, + `agent-base`, `cookie`, `debug`, `http-proxy-agent`, `https-proxy-agent`, + `ip-address`, `minimist`, `ms`, `smart-buffer`, `socks`, + `socks-proxy-agent`, `supports-color`, `undici`, `vscode-regexpp` and + `tslib` (`0BSD`) +- in the built-in language extensions (`MIT` unless noted): `@vscode/l10n`, + `@vscode/sync-api-client`, `@vscode/sync-api-common`, + `@vscode/sync-api-service`, `@vscode/ts-package-manager`, `jsonc-parser`, + `request-light`, `semver` (`ISC`), `vscode-css-languageservice`, + `vscode-html-languageservice`, `vscode-json-languageservice`, + `vscode-languageclient`, `vscode-languageserver`, + `vscode-languageserver-textdocument`, `vscode-tas-client` and `vscode-uri` +- `typescript` (`Apache-2.0`), for the built-in TypeScript extension +- `@vscode/ripgrep-universal` (`MIT`), whose `rg` binaries are ripgrep + (`MIT OR Unlicense`) +- `@parcel/watcher` and its Linux prebuilds (`MIT`), with `detect-libc` + (`Apache-2.0`), `is-glob`, `is-extglob` and `picomatch` (`MIT`) + +Curated extensions are not bundled; `whiteboard remote extensions ensure` +downloads them from Open VSX on the remote. diff --git a/packages/review/app/src/App.tsx b/packages/review/app/src/App.tsx index c4f7cb2dc..3197af8f2 100644 --- a/packages/review/app/src/App.tsx +++ b/packages/review/app/src/App.tsx @@ -295,6 +295,7 @@ function ReviewLayoutContent({ // The scratchpad is a document and nothing else: no source tree to browse, // nothing to share, nothing to dismiss. const scratchpad = session.review?.kind === "scratchpad"; + const sourceWindows = session.review?.available?.sourceWindows !== false; useEffect(() => { if (scratchpad) captureUiEvent(session, "scratchpad_opened"); }, [scratchpad, session]); @@ -412,9 +413,9 @@ function ReviewLayoutContent({ const storedList = useTraceList(); const diffFiles = useReviewDiffFiles(); - // The scratchpad has no repository of its own, so no traces to show. const hasTraceSessions = !scratchpad && + !session.review?.host && ((session.review?.traces.size ?? 0) > 0 || storedList.status !== "loaded" || storedList.sessions.length > 0); @@ -607,7 +608,7 @@ function ReviewLayoutContent({ shellStyles.topbarContext, )} > - {!scratchpad && ( + {!scratchpad && sourceWindows && ( ); })} @@ -296,7 +287,7 @@ function OtherAgentMenu({ return { value: other, - label: TARGET_LABELS[other], + label: REVIEW_CLI_INSTALL_TARGET_LABELS[other], icon: , }; })} @@ -310,7 +301,7 @@ function OtherAgentMenu({ triggerProps={{ "aria-pressed": selected !== undefined }} > {Logo ? : null} - {selected ? TARGET_LABELS[selected] : "Other…"} + {selected ? REVIEW_CLI_INSTALL_TARGET_LABELS[selected] : "Other…"} ); } diff --git a/packages/review/app/src/copy-text.tsx b/packages/review/app/src/copy-text.tsx index b5885d422..78b46f49a 100644 --- a/packages/review/app/src/copy-text.tsx +++ b/packages/review/app/src/copy-text.tsx @@ -1,6 +1,6 @@ import { IconButton } from "@canvas/ui/button"; import * as stylex from "@stylexjs/stylex"; -import { type ReactElement, useEffect, useState } from "react"; +import { type ReactElement, type ReactNode, useEffect, useState } from "react"; import { CheckIcon, CopyIcon as CopyGlyph } from "./icons"; import { useTooltip } from "./use-tooltip"; @@ -72,6 +72,62 @@ export function CopyIcon() { const COPIED_FOR_MS = 1200; +function useCopy(text: string) { + const [copied, setCopied] = useState(false); + + useEffect(() => { + if (!copied) return; + const timer = setTimeout(() => setCopied(false), COPIED_FOR_MS); + + return () => clearTimeout(timer); + }, [copied]); + + // The workbench denies DOM clipboard requests; copyText falls back to + // execCommand and reports whether anything was copied. + const copy = () => + void copyText(text).then((ok) => { + if (ok) setCopied(true); + }); + + return [copied, copy] as const; +} + +/** Text, such as an error, that underlines on hover and copies on click. */ +export function CopyableText({ + text, + children = text, +}: { + text: string; + children?: ReactNode; +}): ReactElement { + const [copied, copy] = useCopy(text); + + const tooltip = useTooltip( + copied ? "Copied" : "Click to copy", + { instant: true }, + ); + + return ( + { + // A drag that selects part of the text should not copy all of it. + if (document.getSelection()?.isCollapsed !== false) copy(); + }} + onKeyDown={(event) => { + if (event.key !== "Enter" && event.key !== " ") return; + event.preventDefault(); + copy(); + }} + > + {children} + + ); +} + /** An icon button that copies `text` and shows a check while it is copied. */ export function CopyButton({ text, @@ -86,16 +142,9 @@ export function CopyButton({ xstyle?: stylex.StyleXStyles; iconStyle: stylex.StyleXStyles; }): ReactElement { - const [copied, setCopied] = useState(false); + const [copied, copy] = useCopy(text); const tooltip = useTooltip(label); - useEffect(() => { - if (!copied) return; - const timer = setTimeout(() => setCopied(false), COPIED_FOR_MS); - - return () => clearTimeout(timer); - }, [copied]); - return ( { - // The workbench denies DOM clipboard requests; copyText falls back to - // execCommand and reports whether anything was copied. - void copyText(text).then((ok) => { - if (ok) setCopied(true); - }); - }} + onClick={copy} > {copied ? ( @@ -121,6 +164,11 @@ export function CopyButton({ } const styles = stylex.create({ + copyable: { + cursor: "pointer", + textDecorationLine: { default: "none", ":hover": "underline" }, + textUnderlineOffset: "2px", + }, icon: { width: "12px", height: "12px", diff --git a/packages/review/app/src/desktop-entry.tsx b/packages/review/app/src/desktop-entry.tsx index b9bf67c62..17d7bf6b8 100644 --- a/packages/review/app/src/desktop-entry.tsx +++ b/packages/review/app/src/desktop-entry.tsx @@ -113,6 +113,7 @@ function Home({ onRestore={ restoreReview ? (review) => restoreReview(review.reviewId) : undefined } + hostStates={content.hostStates} install={content.install} setupActions={content.setupActions} onboarding={content.onboarding} diff --git a/packages/review/app/src/host/review-session-data.ts b/packages/review/app/src/host/review-session-data.ts index 5df4f0740..c12eca17f 100644 --- a/packages/review/app/src/host/review-session-data.ts +++ b/packages/review/app/src/host/review-session-data.ts @@ -1,9 +1,14 @@ import type { LoadedAgentTrace } from "@canvas/use-agent-trace"; -import type { ReviewStackLayer } from "@dev.fast/review-protocol"; +import type { + ReviewApiSummary, + ReviewStackLayer, +} from "@dev.fast/review-protocol"; export interface ReviewSessionData { /** Absent for a review. The scratchpad hides review-only chrome. */ kind?: "scratchpad"; + host?: string; + available?: ReviewApiSummary["available"]; /** Absent for a document whose references all carry their own pins. */ pins?: { base: string; head: string }; /** `worktree` when the head side is the checkout's working files rather diff --git a/packages/review/app/src/remote-hosts-section.browser.test.tsx b/packages/review/app/src/remote-hosts-section.browser.test.tsx new file mode 100644 index 000000000..50717829a --- /dev/null +++ b/packages/review/app/src/remote-hosts-section.browser.test.tsx @@ -0,0 +1,535 @@ +import type { + ReviewCanvasSettingsContent, + ReviewGatewayHostState, + ReviewRemoteHostsSettings, +} from "@dev.fast/review-protocol"; +import { act } from "react"; +import { createRoot } from "react-dom/client"; +import { afterEach, expect, test, vi } from "vitest"; +import { page } from "vitest/browser"; + +import { SettingsPage } from "./settings-page"; + +let root: ReturnType | undefined; + +afterEach(async () => { + await act(async () => root?.unmount()); + root = undefined; + document.body.replaceChildren(); +}); + +function remoteHosts( + configured: string[], + states: ReviewGatewayHostState[] = [], +): ReviewRemoteHostsSettings { + return { + enabled: true, + configured, + suggestions: vi.fn(async () => [ + "devbox", + "other", + ]), + states: vi.fn(async () => states), + set: vi.fn(async (aliases) => aliases), + retry: vi.fn(async () => {}), + install: vi.fn(async () => {}), + agents: vi.fn(async () => null), + connectAgents: vi.fn( + async () => [], + ), + uninstall: vi.fn(async () => {}), + }; +} + +async function render(hosts: ReviewRemoteHostsSettings) { + const reject = () => Promise.reject(new Error("unused")); + + const settings: ReviewCanvasSettingsContent = { + telemetryEnabled: false, + setTelemetryEnabled: reject, + theme: "dark", + setTheme: reject, + keymap: "none", + setKeymap: reject, + ctrlTab: "recent", + setCtrlTab: reject, + documentWidth: "standard", + setDocumentWidth: reject, + readyNotification: "off", + setReadyNotification: reject, + softwareMapEnabled: false, + setSoftwareMapEnabled: reject, + structuralDiffEnabled: false, + setStructuralDiffEnabled: reject, + scratchpadEnabled: false, + setScratchpadEnabled: reject, + diffrConfig: { + read: reject, + set: reject, + saveSummarizer: reject, + testSummarizer: reject, + }, + reloadWindow: async () => {}, + manageExtensions: () => {}, + remoteHosts: hosts, + }; + + const container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); + await act(async () => root!.render()); + + return container; +} + +const section = () => + document.querySelector('section[aria-label="Remote hosts"]'); + +const rows = () => + [...document.querySelectorAll("[data-remote-host]")].map( + (row) => row.textContent, + ); + +test("the section is absent with the setting off", async () => { + await render({ ...remoteHosts(["devbox"]), enabled: false }); + expect(section()).toBeNull(); +}); + +test("lists each host with its state and detail as plain text, and the install command from Desktop as code", async () => { + const install = "npm install -g @dev.fast/whiteboard@0.1.6"; + + await render( + remoteHosts( + ["devbox", "box2", "box3", "box4"], + [ + { alias: "devbox", serverId: "s1", state: "online" }, + { + alias: "box2", + state: "not-installed", + detail: + "Whiteboard is not installed on box2. Install Whiteboard 0.1.6 there; Node 24 is needed.", + installCommand: install, + }, + { + alias: "box3", + state: "incompatible", + detail: "box3 runs Whiteboard x npm install -g evil; run it.", + installCommand: install, + }, + { alias: "box4", state: "connecting" }, + ], + ), + ); + await vi.waitFor(() => expect(rows()).toHaveLength(4)); + await vi.waitFor(() => expect(rows()[0]).toContain("online")); + expect(rows()[1]).toContain("not installed"); + expect(rows()[1]).toContain( + "Whiteboard is not installed on box2. Install Whiteboard 0.1.6 there; Node 24 is needed.", + ); + expect(rows()[2]).toContain("incompatible"); + expect(rows()[2]).toContain( + "box3 runs Whiteboard x npm install -g evil; run it.", + ); + expect(rows()[3]).toContain("connecting"); + expect( + [...section()!.querySelectorAll("code")].map((code) => code.textContent), + ).toEqual([install, install]); + expect(section()!.querySelector("a")).toBeNull(); + expect( + page + .getByRole("button", { name: /^Retry / }) + .elements() + .map((button) => button.getAttribute("aria-label")), + ).toEqual(["Retry box2", "Retry box3"]); +}); + +test("says whether an online host has language features, and why not as one line of plain text", async () => { + await render( + remoteHosts( + ["devbox", "box2", "box3"], + [ + { + alias: "devbox", + serverId: "s1", + state: "online", + languageFeatures: true, + }, + { + alias: "box2", + serverId: "s2", + state: "online", + languageFeatures: false, + languageFeaturesDetail: + "it runs [e10c782](command:x) new\nthis\u0007 Desktop\r\u20283c82a2a", + }, + { + alias: "box3", + state: "offline", + detail: "timed out", + languageFeatures: false, + }, + ], + ), + ); + await vi.waitFor(() => expect(rows()[0]).toContain("online")); + expect(rows()[0]).toContain("Language features: available"); + expect(rows()[1]).toContain( + "Language features: unavailable · it runs [e10c782](command:x) new this Desktop 3c82a2a", + ); + expect(rows()[2]).not.toContain("Language features"); + expect(section()!.querySelector("a, b")).toBeNull(); +}); + +test("lists an online host's language groups after its language features, each with what is missing as plain text", async () => { + await render( + remoteHosts( + ["devbox", "box2"], + [ + { + alias: "devbox", + serverId: "s1", + state: "online", + languageFeatures: true, + languageGroups: [ + { group: "rust", installed: true }, + { + group: "swift", + installed: true, + detail: + "swift was not found\non the login shell's PATH\u0007", + }, + { group: "csharp", installed: false, detail: "download failed" }, + ], + }, + { + alias: "box2", + state: "offline", + languageGroups: [{ group: "rust", installed: true }], + }, + ], + ), + ); + await vi.waitFor(() => expect(rows()[0]).toContain("online")); + expect( + [ + ...document + .querySelector("[data-remote-host]")! + .querySelectorAll(":scope > div > span"), + ].map((line) => line.textContent), + ).toEqual([ + "devbox", + "online", + "Language features: available", + "rust: installed", + "swift: installed · swift was not found on the login shell's PATH", + "csharp: not installed · download failed", + ]); + expect(rows()[1]).not.toContain("rust"); + expect(section()!.querySelector("b")).toBeNull(); +}); + +test("reads the states again only once the last read has answered", async () => { + const pending = Promise.withResolvers(); + const hosts = remoteHosts(["devbox"]); + + vi.mocked(hosts.states).mockReturnValue(pending.promise); + vi.useFakeTimers(); + + try { + await render(hosts); + await act(async () => vi.advanceTimersByTime(10_000)); + expect(hosts.states).toHaveBeenCalledTimes(1); + await act(async () => pending.resolve([])); + await act(async () => vi.advanceTimersByTime(3_000)); + expect(hosts.states).toHaveBeenCalledTimes(2); + } finally { + vi.useRealTimers(); + } +}); + +test("offers Retry to a host that failed to authenticate, is unreachable, not installed or unsupported", async () => { + const hosts = remoteHosts( + ["devbox", "box2", "box3", "box4", "box5"], + [ + { alias: "devbox", state: "auth-failed", detail: "Permission denied" }, + { alias: "box2", state: "unreachable", detail: "timed out" }, + { alias: "box3", state: "not-installed", detail: "Not installed." }, + { + alias: "box4", + state: "unsupported", + detail: "This host runs glibc 2.31; Whiteboard needs 2.34 or newer.", + }, + { alias: "box5", state: "installing" }, + ], + ); + + await render(hosts); + await vi.waitFor(() => + expect( + page.getByRole("button", { name: "Retry devbox" }).elements(), + ).toHaveLength(1), + ); + expect(rows()[3]).toContain("unsupported"); + expect(rows()[3]).toContain("glibc 2.31"); + expect(rows()[4]).toContain("installing"); + expect( + page + .getByRole("button", { name: /^Retry / }) + .elements() + .map((button) => button.getAttribute("aria-label")), + ).toEqual(["Retry devbox", "Retry box2", "Retry box3", "Retry box4"]); + await page.getByRole("button", { name: "Retry box2" }).click(); + expect(hosts.retry).toHaveBeenCalledWith("box2"); + await page.getByRole("button", { name: "Retry box3" }).click(); + expect(hosts.retry).toHaveBeenCalledWith("box3"); +}); + +test("shows the install step, offers Install to a declined host, and Retry after a failed install", async () => { + const hosts = remoteHosts( + ["box1", "box2", "box3", "box4"], + [ + { + alias: "box1", + state: "installing", + detail: "Installing Node 24 (uploaded from this computer).", + }, + { + alias: "box2", + state: "not-installed", + detail: "Whiteboard is not installed on box2.", + installCommand: "npm install -g @dev.fast/whiteboard@0.1.6", + declined: true, + }, + { + alias: "box3", + state: "not-installed", + detail: + "Installing Whiteboard 0.1.6 on box3 failed while installing the package: the package does not match its pinned integrity.", + }, + { + alias: "box4", + state: "incompatible", + detail: + "box4 runs Whiteboard 0.1.5; this Desktop runs 0.1.6. Install Whiteboard 0.1.6 on box4.", + declined: true, + }, + ], + ); + + await render(hosts); + await vi.waitFor(() => + expect(rows()[0]).toContain( + "installing · Installing Node 24 (uploaded from this computer).", + ), + ); + expect( + page + .getByRole("button", { name: /^(Retry|Install) / }) + .elements() + .map((button) => button.getAttribute("aria-label")), + ).toEqual([ + "Install box2", + "Retry box2", + "Retry box3", + "Install box4", + "Retry box4", + ]); + await page.getByRole("button", { name: "Install box2" }).click(); + expect(hosts.install).toHaveBeenCalledWith("box2"); + await page.getByRole("button", { name: "Retry box3" }).click(); + expect(hosts.retry).toHaveBeenCalledWith("box3"); +}); + +test("offers an online host's unconnected agents by Desktop's names, connects them, and shows the result as plain text", async () => { + const hosts = remoteHosts( + ["box1", "box2"], + [ + { alias: "box1", serverId: "s1", state: "online" }, + { alias: "box2", state: "unreachable", detail: "timed out" }, + ], + ); + + vi.mocked(hosts.agents) + .mockResolvedValueOnce([ + { id: "claude", connected: true }, + { id: "codex", connected: false }, + { id: "pi", connected: false }, + { id: "opencode", connected: false, manual: true }, + ]) + .mockResolvedValue([ + { id: "claude", connected: true }, + { id: "codex", connected: false }, + { id: "pi", connected: true }, + { id: "opencode", connected: false, manual: true }, + ]); + vi.mocked(hosts.connectAgents).mockResolvedValue([ + { id: "codex", connected: false, output: "codex: command failed" }, + { id: "pi", connected: true, output: "" }, + ]); + + await render(hosts); + await vi.waitFor(() => + expect(rows()[0]).toContain("Agents on box1: Codex, Pi — Connect"), + ); + expect(hosts.agents).toHaveBeenCalledWith("box1"); + expect(hosts.agents).not.toHaveBeenCalledWith("box2"); + expect(hosts.connectAgents).not.toHaveBeenCalled(); + expect(rows()[0]).toContain( + "Paste into OpenCode on box1: Run `whiteboard connect opencode` and follow the instructions to connect this agent to Whiteboard.", + ); + + await page.getByRole("button", { name: "Connect agents on box1" }).click(); + + expect(hosts.connectAgents).toHaveBeenCalledWith("box1", ["codex", "pi"]); + await vi.waitFor(() => + expect(rows()[0]).toContain( + "Codex was not connected on box1: codex: command failed Pi is connected on box1.", + ), + ); + expect(section()!.querySelector("b")).toBeNull(); + await vi.waitFor(() => + expect(rows()[0]).toContain("Agents on box1: Codex — Connect"), + ); +}); + +test("a refused connect shows why", async () => { + const hosts = remoteHosts(["box1"], [{ alias: "box1", state: "online" }]); + + vi.mocked(hosts.agents).mockResolvedValue([{ id: "pi", connected: false }]); + vi.mocked(hosts.connectAgents).mockRejectedValue( + new Error("box1 is not connected, or its agents could not be read."), + ); + + await render(hosts); + await page.getByRole("button", { name: "Connect agents on box1" }).click(); + await vi.waitFor(() => + expect(rows()[0]).toContain( + "Could not connect agents on box1: box1 is not connected, or its agents could not be read.", + ), + ); +}); + +test("adding -bad is refused with the reason", async () => { + const hosts = remoteHosts([]); + + vi.mocked(hosts.set).mockRejectedValue( + new Error('The SSH alias "-bad" starts with -.'), + ); + await render(hosts); + await page.getByLabelText("SSH alias").fill("-bad"); + await page.getByRole("button", { name: "Add" }).click(); + await vi.waitFor(() => + expect(section()!.querySelector('[role="alert"]')?.textContent).toBe( + 'The SSH alias "-bad" starts with -.', + ), + ); + expect(rows()).toEqual([]); +}); + +test("adds a host from the suggestions, and removes one", async () => { + const hosts = remoteHosts(["other"]); + + await render(hosts); + await vi.waitFor(() => + expect( + [...section()!.querySelectorAll("datalist option")].map((option) => + option.getAttribute("value"), + ), + ).toEqual(["devbox"]), + ); + await page.getByLabelText("SSH alias").fill("devbox"); + await page.getByRole("button", { name: "Add" }).click(); + expect(hosts.set).toHaveBeenLastCalledWith(["other", "devbox"]); + await vi.waitFor(() => expect(rows()).toHaveLength(2)); + await page.getByRole("button", { name: "Remove other" }).click(); + expect(hosts.set).toHaveBeenLastCalledWith(["other", "devbox"]); + await page.getByRole("button", { name: "Remove host" }).click(); + await vi.waitFor(() => + expect(hosts.set).toHaveBeenLastCalledWith(["devbox"]), + ); + await vi.waitFor(() => expect(rows()).toHaveLength(1)); +}); + +test("removing a host leaves Whiteboard on it unless asked", async () => { + const hosts = remoteHosts(["devbox", "box2"]); + + await render(hosts); + await page.getByRole("button", { name: "Remove devbox" }).click(); + const also = page.getByLabelText("Also remove Whiteboard from devbox"); + await expect.element(also).not.toBeChecked(); + await page.getByRole("button", { name: "Remove host" }).click(); + await vi.waitFor(() => + expect(hosts.set).toHaveBeenLastCalledWith(["box2"]), + ); + expect(hosts.uninstall).not.toHaveBeenCalled(); + + await page.getByRole("button", { name: "Remove box2" }).click(); + await expect + .element(page.getByLabelText("Also remove Whiteboard from box2")) + .not.toBeChecked(); + expect(section()!.querySelector("b")).toBeNull(); +}); + +test("also removing Whiteboard uninstalls before the host goes, and a failure is shown while the host is still removed", async () => { + const hosts = remoteHosts(["devbox", "box2"]); + const order: string[] = []; + + vi.mocked(hosts.uninstall).mockImplementation(async (alias) => { + order.push(`uninstall ${alias}`); + + if (alias === "box2") + throw new Error( + "Could not remove Whiteboard from box2: A Whiteboard server you started (process 7) runs from it.", + ); + }); + vi.mocked(hosts.set).mockImplementation(async (aliases) => { + order.push(`set ${aliases.join(",")}`); + + return aliases; + }); + await render(hosts); + + await page.getByRole("button", { name: "Remove devbox" }).click(); + await page.getByLabelText("Also remove Whiteboard from devbox").click(); + await page.getByRole("button", { name: "Remove host" }).click(); + await vi.waitFor(() => expect(rows()).toHaveLength(1)); + expect(order).toEqual(["uninstall devbox", "set box2"]); + expect(section()!.querySelector('[role="alert"]')).toBeNull(); + + await page.getByRole("button", { name: "Remove box2" }).click(); + await page.getByLabelText("Also remove Whiteboard from box2").click(); + await page.getByRole("button", { name: "Remove host" }).click(); + await vi.waitFor(() => + expect(section()!.querySelector('[role="alert"]')?.textContent).toBe( + "Could not remove Whiteboard from box2: A Whiteboard server you started (process 7) runs from it.", + ), + ); + expect(rows()).toEqual([]); + expect(order.slice(2)).toEqual(["uninstall box2", "set "]); +}); + +test("a failed host's detail copies on click; a healthy one does not", async () => { + const writeText = vi + .spyOn(navigator.clipboard, "writeText") + .mockResolvedValue(); + + await render( + remoteHosts( + ["devbox", "box2"], + [ + { alias: "devbox", serverId: "s1", state: "online" }, + { alias: "box2", state: "unreachable", detail: "ssh: timed out" }, + ], + ), + ); + + await vi.waitFor(() => + expect(rows()[1]).toContain("unreachable · ssh: timed out"), + ); + expect(page.getByRole("button", { name: "online" }).query()).toBeNull(); + + await page + .getByRole("button", { name: "unreachable · ssh: timed out" }) + .click(); + expect(writeText).toHaveBeenCalledWith("unreachable · ssh: timed out"); +}); diff --git a/packages/review/app/src/remote-hosts-section.tsx b/packages/review/app/src/remote-hosts-section.tsx new file mode 100644 index 000000000..1432170f0 --- /dev/null +++ b/packages/review/app/src/remote-hosts-section.tsx @@ -0,0 +1,412 @@ +import { Button } from "@canvas/ui/button"; +import { TextField } from "@canvas/ui/text-field"; +import { + REVIEW_CLI_INSTALL_TARGET_LABELS, + type ReviewGatewayHostState, + type ReviewRemoteAgent, + type ReviewRemoteHostsSettings, +} from "@dev.fast/review-protocol"; +import * as stylex from "@stylexjs/stylex"; +import { useEffect, useState } from "react"; + +import { CopyableText } from "./copy-text"; +import { settingsStyles as styles } from "./settings-styles"; +import { tokens } from "./tokens.stylex"; + +const STATES_EVERY_MS = 3000; + +const RETRIED = new Set([ + "auth-failed", + "incompatible", + "not-installed", + "unreachable", + "unsupported", +]); + +const plain = (text: string) => + text + .replaceAll(/[\t\n\u2028\u2029]/g, " ") + .replaceAll(/[\x00-\x08\x0b-\x1f\x7f]/g, ""); + +export function RemoteHostsSection({ + hosts, +}: { + hosts: ReviewRemoteHostsSettings; +}) { + const [configured, setConfigured] = useState(hosts.configured); + const [states, setStates] = useState(); + const [suggestions, setSuggestions] = useState([]); + const [alias, setAlias] = useState(""); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(); + const [removing, setRemoving] = useState(); + const [uninstall, setUninstall] = useState(false); + + useEffect(() => { + void hosts + .suggestions() + .then(setSuggestions) + .catch(() => undefined); + }, [hosts]); + + useEffect(() => { + let live = true; + let reading = false; + + const read = () => { + if (document.hidden || reading) return; + reading = true; + void hosts + .states() + .then((next) => { + if (live) setStates(next); + }) + .catch(() => undefined) + .finally(() => { + reading = false; + }); + }; + + read(); + const timer = setInterval(read, STATES_EVERY_MS); + + return () => { + live = false; + clearInterval(timer); + }; + }, [hosts]); + + const save = async (next: string[]) => { + setBusy(true); + setError(undefined); + + try { + setConfigured(await hosts.set(next)); + + return true; + } catch (cause) { + setError(cause instanceof Error ? cause.message : String(cause)); + + return false; + } finally { + setBusy(false); + } + }; + + const add = async () => { + if (await save([...configured, alias.trim()])) setAlias(""); + }; + + const remove = async (name: string) => { + setRemoving(undefined); + let problem: string | undefined; + + if (uninstall) { + setBusy(true); + + try { + await hosts.uninstall(name); + } catch (cause) { + problem = cause instanceof Error ? cause.message : String(cause); + } + } + + await save(configured.filter((other) => other !== name)); + + if (problem) setError(problem); + }; + + return ( +
+

Remote hosts

+ {configured.map((name) => { + const state = states?.find((host) => host.alias === name); + + return ( +
+
+ {name} + {states ? ( + + ) : null} + {state?.state === "online" ? ( + + ) : null} + {state?.state === "online" + ? state.languageGroups?.map(({ group, installed, detail }) => ( + + )) + : null} + {state?.state === "online" ? ( + + ) : null} + {state?.installCommand ? ( + + + {state.installCommand} + + + ) : null} +
+
+ {state?.declined ? ( + + ) : null} + {state && RETRIED.has(state.state) ? ( + + ) : null} + +
+ {removing === name ? ( +
+ + + +
+ ) : null} +
+ ); + })} +
{ + event.preventDefault(); + void add(); + }} + > +
+ Add a host + + An alias from your SSH configuration. Whiteboard offers to install + itself there. + +
+
+ setAlias(event.target.value)} + /> + + {suggestions + .filter((suggestion) => !configured.includes(suggestion)) + .map((suggestion) => ( + + +
+
+ {error ? ( +

+ +

+ ) : null} +
+ ); +} + +function RemoteHostAgents({ + hosts, + alias, +}: { + hosts: ReviewRemoteHostsSettings; + alias: string; +}) { + const [agents, setAgents] = useState(null); + const [busy, setBusy] = useState(false); + const [result, setResult] = useState<{ text: string; failed: boolean }>(); + + useEffect(() => { + let live = true; + + void hosts + .agents(alias) + .then((next) => { + if (live) setAgents(next); + }) + .catch(() => undefined); + + return () => { + live = false; + }; + }, [hosts, alias]); + + const offered = agents?.filter((agent) => !agent.connected && !agent.manual); + const manual = agents?.filter((agent) => !agent.connected && agent.manual); + + const connect = async (ids: ReviewRemoteAgent["id"][]) => { + setBusy(true); + setResult(undefined); + + try { + const results = await hosts.connectAgents(alias, ids); + + setResult({ + failed: results.some((done) => !done.connected), + text: results + .map((done) => + done.connected + ? `${REVIEW_CLI_INSTALL_TARGET_LABELS[done.id]} is connected on ${alias}.` + : `${REVIEW_CLI_INSTALL_TARGET_LABELS[done.id]} was not connected on ${alias}: ${done.output || "it printed nothing"}`, + ) + .join(" "), + }); + setAgents(await hosts.agents(alias)); + } catch (cause) { + setResult({ + failed: true, + text: `Could not connect agents on ${alias}: ${cause instanceof Error ? cause.message : String(cause)}`, + }); + } finally { + setBusy(false); + } + }; + + return ( + <> + {offered?.length ? ( + + Agents on {alias}:{" "} + {offered + .map((agent) => REVIEW_CLI_INSTALL_TARGET_LABELS[agent.id]) + .join(", ")}{" "} + —{" "} + + + ) : null} + {manual?.map((agent) => ( + + Paste into {REVIEW_CLI_INSTALL_TARGET_LABELS[agent.id]} on {alias}:{" "} + + Run `whiteboard connect {agent.id}` and follow the instructions to + connect this agent to Whiteboard. + + + ))} + {result ? ( + + ) : null} + + ); +} + +/** A row's detail line; a failure reads as an error and copies on click. */ +function Detail({ + text, + failed, + role, +}: { + text: string; + failed: boolean; + role?: "status"; +}) { + return ( + + {failed ? : text} + + ); +} + +const local = stylex.create({ + detail: { + overflowWrap: "anywhere", + userSelect: "text", + }, + failed: { + color: tokens.changeRemoved, + }, + actions: { + gap: "8px", + }, + confirm: { + display: "flex", + gridColumn: "1 / -1", + flexWrap: "wrap", + alignItems: "center", + gap: "8px", + }, + alias: { + minWidth: 0, + flex: 1, + }, + command: { + color: tokens.ink, + fontFamily: tokens.fontMono, + userSelect: "all", + }, +}); diff --git a/packages/review/app/src/review-doc-meta.browser.test.tsx b/packages/review/app/src/review-doc-meta.browser.test.tsx index b1523d1ea..c3b9e7a21 100644 --- a/packages/review/app/src/review-doc-meta.browser.test.tsx +++ b/packages/review/app/src/review-doc-meta.browser.test.tsx @@ -163,4 +163,69 @@ describe("ReviewDocumentMetaLine", () => { await render(2); await vi.waitFor(() => expect(container.querySelector("a")).toBeNull()); }); + + it.each([ + ["file:///etc/passwd", false], + ["javascript:alert(1)", false], + ["https://github.com/o/r/pull/12/../../../evil", false], + ["https://github.com/o/r/pull/12", true], + ["https://github.example.com/o/r/pull/12", true], + ])("links the PR only for a canonical URL: %s", async (url, linked) => { + const session = testReviewSession(); + session.review = { + ...session.review!, + pullRequestNumber: 12, + pullRequestUrl: url, + }; + const container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); + await act(async () => + root?.render( + + + + + + + , + ), + ); + + expect(container.textContent).toContain("PR #12"); + expect(container.querySelector("a")?.getAttribute("href") ?? null).toBe( + linked ? url : null, + ); + }); + + it.each([ + ["https://github.com/o/r/pull/12", "devbox: o / r·PR #12"], + [undefined, "devbox"], + ])("leads with the SSH alias of a remote review: %s", async (url, top) => { + const session = testReviewSession(); + session.review = { + ...session.review!, + host: "devbox", + pullRequestNumber: url ? 12 : undefined, + pullRequestUrl: url, + }; + const container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); + await act(async () => + root?.render( + + + + + + + , + ), + ); + + expect(container.querySelector("header > div > div")?.textContent).toBe( + top, + ); + }); }); diff --git a/packages/review/app/src/review-doc-meta.tsx b/packages/review/app/src/review-doc-meta.tsx index 8f4327884..d1d8933ad 100644 --- a/packages/review/app/src/review-doc-meta.tsx +++ b/packages/review/app/src/review-doc-meta.tsx @@ -3,6 +3,7 @@ import { type ReviewDiffStats, summarizeReviewDiffFiles, } from "@dev.fast/review-protocol"; +import { PULL_REQUEST_URL } from "@review/review-api/origin"; import * as stylex from "@stylexjs/stylex"; import { Fragment, @@ -152,18 +153,21 @@ export function ReviewDocumentMetaLine({ >
- {repository ? ( + {review.host || repository ? ( - {repository[1]} / {repository[2]} + {review.host} + {review.host && repository ? ": " : null} + {repository ? `${repository[1]} / ${repository[2]}` : null} ) : null} - {repository && meta.pullRequestNumber != null ? ( + {(review.host || repository) && meta.pullRequestNumber != null ? ( ) : null} {meta.pullRequestNumber != null && - (meta.pullRequestUrl ? ( + (meta.pullRequestUrl && + PULL_REQUEST_URL.test(meta.pullRequestUrl) ? ( { expect(document.activeElement).toBe(search); }, ); + + it("puts a remote review's host in front of its repository name", async () => { + const reviews = [ + remote("devbox", { reviewId: uuid(1), title: "Remote" }), + summary({ + reviewId: uuid(2), + title: "Laptop", + createdAt: "2026-07-28T11:54:00.000Z", + repositoryGroup: { key: "git:/repo/.git", label: "my-repo" }, + }), + ]; + + await act(async () => + renderWithHost( {}} />), + ); + expect(repositories()).toEqual(["devbox: my-repo", "my-repo"]); + }); + + it("keeps two machines' repositories of one name in two groups", async () => { + const reviews = [ + remote("devbox", { reviewId: uuid(1), title: "On devbox" }), + remote("other", { reviewId: uuid(2), title: "On other" }), + ]; + + await act(async () => + renderWithHost( {}} />), + ); + await act(async () => + container + .querySelector( + 'button[aria-label="Filter by repository"]', + )! + .click(), + ); + expect(host.menu.items.map((item) => item.label)).toEqual([ + "All repos", + "devbox: my-repo", + "other: my-repo", + ]); + await act(async () => host.select("other:git:/repo/.git")); + expect(titles()).toEqual(["On other"]); + }); + + it.each([ + ["offline", "offline"], + ["connecting", "connecting"], + ["not-installed", "not installed"], + ] as const)( + "draws a %s host's review as unavailable, and opening it shows the host's detail", + async (hostState, words) => { + const review = remote("devbox", { hostState }); + const onOpen = vi.fn<(review: ReviewApiSummary) => void>(); + + const hostStates = vi.fn<() => Promise>( + async () => [ + { + alias: "devbox", + state: hostState, + detail: "devbox is offline: it did not answer within 3 seconds.", + }, + ], + ); + + await act(async () => + renderWithHost( + , + ), + ); + + const row = container.querySelector("tbody tr")!; + + const open = container.querySelector( + "td:nth-child(2) > button", + )!; + + expect(row.hasAttribute("data-unavailable")).toBe(true); + expect(row.textContent).toContain(words); + expect(open.getAttribute("aria-disabled")).toBe("true"); + expect(open.title).toBe(`devbox is ${words}`); + await act(async () => + container + .querySelector("td:nth-child(2) > button")! + .click(), + ); + expect(onOpen).not.toHaveBeenCalled(); + expect(container.querySelector('[role="alert"]')?.textContent).toBe( + "devbox is offline: it did not answer within 3 seconds.", + ); + }, + ); + + it("says only the state while the host's detail loads or when it has none", async () => { + const answer = Promise.withResolvers(); + const review = remote("devbox", { hostState: "offline" }); + + await act(async () => + renderWithHost( + {}} + hostStates={() => answer.promise} + />, + ), + ); + await act(async () => + container + .querySelector("td:nth-child(2) > button")! + .click(), + ); + expect(alert()).toBe("devbox is offline."); + await act(async () => + answer.resolve([{ alias: "devbox", state: "offline" }]), + ); + expect(alert()).toBe("devbox is offline."); + }); + + it("keeps the message of the last unavailable review opened, and drops it when the list changes", async () => { + const answers = [ + Promise.withResolvers(), + Promise.withResolvers(), + ]; + + let calls = 0; + + const hostStates = vi.fn<() => Promise>( + () => answers[calls++]!.promise, + ); + + const reviews = [ + remote("devbox", { + reviewId: uuid(1), + title: "One", + hostState: "offline", + }), + remote("other", { + reviewId: uuid(2), + title: "Two", + hostState: "offline", + }), + ]; + + const states = [ + { alias: "devbox", state: "offline" as const, detail: "devbox detail" }, + { alias: "other", state: "offline" as const, detail: "other detail" }, + ]; + + await act(async () => + renderWithHost( + {}} + hostStates={hostStates} + />, + ), + ); + + const open = (title: string) => + [ + ...container.querySelectorAll( + "td:nth-child(2) > button", + ), + ] + .find((button) => button.textContent?.includes(title))! + .click(); + + await act(async () => open("One")); + await act(async () => open("Two")); + await act(async () => answers[1]!.resolve(states)); + await act(async () => answers[0]!.resolve(states)); + expect(alert()).toBe("other detail"); + await act(async () => + renderWithHost( + {}} + hostStates={hostStates} + />, + ), + ); + expect(alert()).toBeUndefined(); + }); + + function alert() { + return container.querySelector('[role="alert"]')?.textContent; + } + + it("draws an online remote review as available", async () => { + const onOpen = vi.fn<(review: ReviewApiSummary) => void>(); + const review = remote("devbox"); + + await act(async () => + renderWithHost(), + ); + expect( + container.querySelector("tbody tr")!.hasAttribute("data-unavailable"), + ).toBe(false); + await act(async () => + container + .querySelector("td:nth-child(2) > button")! + .click(), + ); + expect(onOpen).toHaveBeenCalledWith(review); + }); + + function titles() { + return [ + ...container.querySelectorAll("tbody button > span:first-child"), + ].map((element) => element.textContent); + } + + function repositories() { + return [ + ...container.querySelectorAll("tbody button > span:nth-child(2)"), + ].map((element) => element.textContent); + } }); +function remote( + alias: string, + overrides: Partial = {}, +): ReviewApiSummary { + return summary({ + repositoryGroup: { key: `${alias}:git:/repo/.git`, label: "my-repo" }, + host: alias, + hostState: "online", + available: { sourceWindows: false, languageFeatures: false }, + ...overrides, + }); +} + describe("formatRelativeTime", () => { const now = Date.parse("2026-07-29T12:00:00.000Z"); diff --git a/packages/review/app/src/review-home-view.tsx b/packages/review/app/src/review-home-view.tsx index 2f0523bcd..174417afb 100644 --- a/packages/review/app/src/review-home-view.tsx +++ b/packages/review/app/src/review-home-view.tsx @@ -14,6 +14,7 @@ import type { ReviewCanvasInstallContent, ReviewCanvasOnboarding, ReviewCanvasSetupActions, + ReviewGatewayHostState, } from "@dev.fast/review-protocol"; import { fuzzyMatches, fuzzySegments } from "@review/fuzzy-match"; import * as stylex from "@stylexjs/stylex"; @@ -28,6 +29,7 @@ import { useState, } from "react"; +import { CopyableText } from "./copy-text"; import { homeStyles } from "./home-styles"; import { CanvasUiContext, useCanvasMenu } from "./host/canvas-ui"; import { OptionMenu } from "./option-menu"; @@ -46,6 +48,7 @@ interface ReviewHomeProps { // support them. onDismiss?(review: ReviewApiSummary): Promise; onRestore?(review: ReviewApiSummary): Promise; + hostStates?(): Promise; // Present only while the list is empty: Home then renders Welcome. install?: ReviewCanvasInstallContent; setupActions?: ReviewCanvasSetupActions; @@ -100,6 +103,7 @@ export function ReviewHome({ onDelete, onDismiss, onRestore, + hostStates, install, setupActions, onboarding, @@ -117,6 +121,34 @@ export function ReviewHome({ ); const [deleteError, setDeleteError] = useState(); + const [hostMessage, setHostMessage] = useState(); + const hostMessageGeneration = useRef(0); + + useEffect(() => { + hostMessageGeneration.current++; + setHostMessage(undefined); + }, [reviews]); + + const open = useCallback( + async (review: ReviewApiSummary) => { + const generation = ++hostMessageGeneration.current; + + if (!unavailable(review)) { + setHostMessage(undefined); + onOpen(review); + + return; + } + + setHostMessage(`${review.host} is ${hostStateWords(review)}.`); + const states = await hostStates?.().catch(() => undefined); + const detail = states?.find((host) => host.alias === review.host)?.detail; + + if (detail && generation === hostMessageGeneration.current) + setHostMessage(detail); + }, + [onOpen, hostStates], + ); // Keep successful deletions hidden until the catalog acknowledges removal. useEffect(() => { @@ -250,6 +282,11 @@ export function ReviewHome({
{deleteError ?

{deleteError}

: null} + {hostMessage ? ( +

+ +

+ ) : null} {/* Keyed off the active list, not the whole result: a query that hits only dismissed reviews empties the main area, and the collapsed Dismissed count alone does not explain why. */} @@ -268,14 +305,14 @@ export function ReviewHome({ ) : null} {active.length > 0 ? ( - + ) : null} {dismissed.length > 0 ? ( setShowDismissed((open) => !open)} - onOpen={onOpen} + onToggle={() => setShowDismissed((shown) => !shown)} + onOpen={open} onDelete={actions.onDelete} /> ) : null} @@ -452,10 +489,15 @@ function ReviewTable({ }) { const [repository, setRepository] = useState(""); const [sort, setSort] = useState("newest"); - const repositories = [...new Set(reviews.map(repositoryLabel))].sort(); + + const repositories = [ + ...new Map( + reviews.map((review) => [repositoryKey(review), repositoryLabel(review)]), + ), + ].sort(([, left], [, right]) => left.localeCompare(right)); const filtered = reviews.filter( - (review) => !repository || repositoryLabel(review) === repository, + (review) => !repository || repositoryKey(review) === repository, ); const sorted = [...filtered].sort((left, right) => { @@ -490,7 +532,7 @@ function ReviewTable({ value={repository} options={[ { value: "", label: "All repos" }, - ...repositories.map((name) => ({ value: name, label: name })), + ...repositories.map(([value, label]) => ({ value, label })), ]} onChange={setRepository} /> @@ -548,7 +590,12 @@ function ReviewTable({ return ( onOpen(review)} > @@ -583,6 +635,11 @@ function ReviewTable({ } > + {unavailable(review) ? ( + + {hostStateWords(review)} + + ) : null} @@ -943,7 +1000,25 @@ function matchesQuery(review: ReviewApiSummary, query: string): boolean { ); } +function unavailable(review: ReviewApiSummary): boolean { + return review.hostState !== undefined && review.hostState !== "online"; +} + +function hostStateWords(review: ReviewApiSummary): string { + return review.hostState?.replace("-", " ") ?? ""; +} + +function repositoryKey(review: ReviewApiSummary): string { + return review.repositoryGroup?.key ?? repositoryLabel(review); +} + function repositoryLabel(review: ReviewApiSummary): string { + const label = localRepositoryLabel(review); + + return review.host ? `${review.host}: ${label}` : label; +} + +function localRepositoryLabel(review: ReviewApiSummary): string { if (review.repositoryGroup) return review.repositoryGroup.label; if (review.shared?.cloneUrl) { @@ -1512,4 +1587,7 @@ const styles = stylex.create({ display: "flex", justifyContent: "center", }, + unavailableRow: { + opacity: 0.55, + }, }); diff --git a/packages/review/app/src/settings-page.tsx b/packages/review/app/src/settings-page.tsx index 0ab436b3b..577de0aea 100644 --- a/packages/review/app/src/settings-page.tsx +++ b/packages/review/app/src/settings-page.tsx @@ -15,6 +15,7 @@ import { type ReactNode, useEffect, useState } from "react"; import { ConnectCard, LegacySkillsRow } from "./connect-card"; import { DiffrConfigSection } from "./diffr-config-section"; import { homeStyles } from "./home-styles"; +import { RemoteHostsSection } from "./remote-hosts-section"; import { Choice } from "./settings-choice"; import { settingsStyles as styles } from "./settings-styles"; import { withClass } from "./stylex-props"; @@ -300,6 +301,10 @@ export function SettingsPage({ + {settings.remoteHosts.enabled ? ( + + ) : null} +
readTraceList(session.fetch, storageOverride, signal), - enabled: !usesProvided, + enabled: !usesProvided && !session.review?.host, staleTime: 0, }); diff --git a/packages/review/package.json b/packages/review/package.json index 5f3b31460..1c67b2e74 100644 --- a/packages/review/package.json +++ b/packages/review/package.json @@ -23,7 +23,8 @@ "src", "!src/**/*.test.ts", "!src/**/*test-utils*", - "THIRD_PARTY_NOTICES.md" + "THIRD_PARTY_NOTICES.md", + "vscode-server" ], "type": "module", "exports": { @@ -79,6 +80,8 @@ "semver": "7.8.5", "sharp": "0.35.4", "smol-toml": "1.8.0", + "undici": "7.29.0", + "yauzl": "3.4.0", "zod": "catalog:" }, "devDependencies": { @@ -90,6 +93,7 @@ "@dev.fast/trace-protocol": "workspace:*", "@types/node": "24.12.2", "@types/semver": "7.7.1", + "@types/yauzl": "3.4.0", "micromark-extension-math": "3.1.0", "pino-pretty": "13.1.3", "tsdown": "0.22.14", diff --git a/packages/review/src/ask/agents.test.ts b/packages/review/src/ask/agents.test.ts new file mode 100644 index 000000000..fafb8f329 --- /dev/null +++ b/packages/review/src/ask/agents.test.ts @@ -0,0 +1,43 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { afterEach, beforeEach, expect, it, vi } from "vitest"; + +import { askAgentTakesMcp } from "./agents.js"; + +let root: string; + +beforeEach(async () => { + root = await mkdtemp(path.join(tmpdir(), "review-ask-agents-")); + vi.stubEnv("HOME", root); + vi.stubEnv("PATH", "/usr/bin:/bin"); +}); + +afterEach(async () => { + vi.unstubAllEnvs(); + await rm(root, { recursive: true, force: true }); +}); + +it.skipIf(process.platform === "win32")( + "gives MCP to a Pi found, and run, only on the PATH it is handed", + async () => { + const bin = path.join(root, "login-bin"); + await mkdir(bin); + await writeFile( + path.join(bin, "wbtest-node"), + '#!/bin/sh\nexec /bin/sh "$@"\n', + { mode: 0o755 }, + ); + await writeFile( + path.join(bin, "pi"), + "#!/usr/bin/env wbtest-node\necho 0.99.0\n", + { mode: 0o755 }, + ); + + expect(await askAgentTakesMcp("pi", { PATH: `${bin}:/usr/bin:/bin` })).toBe( + true, + ); + expect(await askAgentTakesMcp("pi")).toBe(false); + }, +); diff --git a/packages/review/src/ask/agents.ts b/packages/review/src/ask/agents.ts index b1a444cc6..354960610 100644 --- a/packages/review/src/ask/agents.ts +++ b/packages/review/src/ask/agents.ts @@ -56,7 +56,7 @@ interface AskAgentSpec { interface AskAgentMcp { /** Whether this install can take them. */ - supported(executable: string): Promise; + supported(executable: string, env: NodeJS.ProcessEnv): Promise; /** What the adapter runs in place of the user's CLI to give them, with * what it needs in `env`. */ launch( @@ -200,6 +200,8 @@ export type AskAgentLauncher = ( bypass?: boolean; /** The MCP servers its sessions are given. */ mcpServers?: McpServer[]; + /** Where it is found, and the environment it runs in. */ + env?: NodeJS.ProcessEnv; }, ) => Promise; @@ -250,13 +252,16 @@ async function findAgent( } /** Whether the agent's model gets the MCP servers its sessions are given. */ -export async function askAgentTakesMcp(agent: AskAgentId): Promise { +export async function askAgentTakesMcp( + agent: AskAgentId, + env: NodeJS.ProcessEnv = process.env, +): Promise { const spec = askAgents[agent]; if (!spec.mcp) return true; - const executable = await findAgent(spec); + const executable = await findAgent(spec, env); - return executable !== undefined && spec.mcp.supported(executable); + return executable !== undefined && spec.mcp.supported(executable, env); } export async function detectAskAgents( @@ -280,15 +285,15 @@ const STDERR_LIMIT = 8_000; export const launchAskAgent: AskAgentLauncher = async ( agent, cwd, - { bypass = false, mcpServers = [] } = {}, + { bypass = false, mcpServers = [], env: base = process.env } = {}, ) => { const spec = askAgents[agent]; - const executable = await findAgent(spec); + const executable = await findAgent(spec, base); if (!executable) throw new Error(`${spec.name} is not installed.`); const env: NodeJS.ProcessEnv = { - ...process.env, + ...base, ...((bypass && spec.bypass?.env) || spec.env), }; @@ -303,7 +308,9 @@ export const launchAskAgent: AskAgentLauncher = async ( if ("adapter" in launch) { env[launch.executableEnv] = - spec.mcp && mcpServers.length && (await spec.mcp.supported(executable)) + spec.mcp && + mcpServers.length && + (await spec.mcp.supported(executable, env)) ? await spec.mcp.launch(executable, mcpServers, env) : executable; diff --git a/packages/review/src/ask/pi-mcp.ts b/packages/review/src/ask/pi-mcp.ts index 4f7de4427..0a1a479f9 100644 --- a/packages/review/src/ask/pi-mcp.ts +++ b/packages/review/src/ask/pi-mcp.ts @@ -49,13 +49,14 @@ const versions = new Map< { read: number; version: Promise } >(); -function piVersion(executable: string) { +function piVersion(executable: string, env: NodeJS.ProcessEnv) { const cached = versions.get(executable); if (cached && Date.now() - cached.read < VERSION_TTL_MS) return cached.version; const version = promisify(execFile)(executable, ["--version"], { + env, timeout: 5_000, }) .then(({ stdout }) => @@ -69,9 +70,12 @@ function piVersion(executable: string) { } /** Whether this Pi can be given MCP servers. */ -export async function piTakesMcp(executable: string): Promise { +export async function piTakesMcp( + executable: string, + env: NodeJS.ProcessEnv, +): Promise { if (process.platform === "win32") return false; - const version = await piVersion(executable); + const version = await piVersion(executable, env); if (!version) return false; diff --git a/packages/review/src/ask/thread.test.ts b/packages/review/src/ask/thread.test.ts index 9f223073f..27c028ed3 100644 --- a/packages/review/src/ask/thread.test.ts +++ b/packages/review/src/ask/thread.test.ts @@ -22,9 +22,13 @@ import { applyAskChange, askUpdateSchema, } from "@review/ask/thread-state.js"; -import { AskThread, type AskThreadLimits } from "@review/ask/thread.js"; +import { + AskThread, + type AskThreadLimits, + askThreadLimits, +} from "@review/ask/thread.js"; import { AskThreads } from "@review/ask/threads.js"; -import { expect, it, vi } from "vitest"; +import { expect, it, onTestFinished, vi } from "vitest"; const permissionOptions = [ { optionId: "allow", name: "Allow once", kind: "allow_once" as const }, @@ -1507,3 +1511,150 @@ it("accepts Cursor's todo list, so it carries on with the answer", async () => { expect(state.entries.at(-1)).toMatchObject({ text: "It is safe." }); thread.close(); }); + +function idleThreads(turn: (prompt: string) => Promise = async () => {}) { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + onTestFinished(() => { + vi.useRealTimers(); + }); + const fake = fakeAgent(async (_client, prompt) => turn(prompt)); + const stopped = vi.fn<() => void>(); + + const launch: AskAgentLauncher = async (...args) => { + const process = await fake.launch(...args); + + return { + ...process, + stop: () => { + stopped(); + process.stop(); + }, + }; + }; + + return { + threads: new AskThreads( + launch, + {}, + { ...askThreadLimits, idleCloseMs: 1_000 }, + ), + desktop: new AskThreads(launch), + stopped, + }; +} + +const idleAsk = (onSave?: (entries: AskEntry[]) => void) => ({ + reviewId: "review", + agent: "claude" as const, + cwd: "/checkouts/payments", + head: "abc123", + selection: { title: "Payments" }, + context: "", + question: { text: "Why?" }, + onSave, +}); + +it("closes a thread once it has waited idle a while with no one watching, saving what it showed", async () => { + const { threads, desktop, stopped } = idleThreads(); + const saved = vi.fn<(entries: AskEntry[]) => void>(); + const thread = threads.open(idleAsk(saved)); + const kept = desktop.open(idleAsk()); + const unwatch = thread.subscribe(() => {}); + + await vi.waitFor(() => expect(thread.read().status).toBe("idle")); + await vi.waitFor(() => expect(kept.read().status).toBe("idle")); + await vi.advanceTimersByTimeAsync(5_000); + expect(threads.get(thread.id)).toBe(thread); + + unwatch(); + await vi.advanceTimersByTimeAsync(999); + thread.subscribe(() => {})(); + await vi.advanceTimersByTimeAsync(999); + expect(threads.get(thread.id)).toBe(thread); + expect(stopped).not.toHaveBeenCalled(); + + await vi.advanceTimersByTimeAsync(1); + expect(threads.get(thread.id)).toBeUndefined(); + expect(stopped).toHaveBeenCalled(); + expect(saved).toHaveBeenLastCalledWith(thread.read().entries); + expect(thread.read().status).toBe("idle"); + expect(desktop.get(kept.id)).toBe(kept); + kept.close(); +}); + +it("waits for an answer to end before closing a thread no one watches", async () => { + const answered = Promise.withResolvers(); + + const { threads, stopped } = idleThreads(async (prompt) => { + if (prompt.includes("And then?")) await answered.promise; + }); + + const thread = threads.open(idleAsk()); + + await vi.waitFor(() => expect(thread.read().status).toBe("idle")); + await vi.advanceTimersByTimeAsync(500); + const asked = thread.ask({ text: "And then?" }); + await vi.waitFor(() => expect(thread.read().status).toBe("running")); + await vi.advanceTimersByTimeAsync(5_000); + expect(threads.get(thread.id)).toBe(thread); + + answered.resolve(); + await asked; + await vi.advanceTimersByTimeAsync(999); + expect(threads.get(thread.id)).toBe(thread); + await vi.advanceTimersByTimeAsync(1); + expect(threads.get(thread.id)).toBeUndefined(); + expect(stopped).toHaveBeenCalled(); +}); + +it.each([ + [ + "failed", + async () => { + throw new Error("Overloaded."); + }, + ], + [ + "waiting", + async (client: AgentContext) => { + await askPermission(client, "execute"); + }, + ], +] as const)( + "closes a thread no one watches that is %s", + async (status, turn) => { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + onTestFinished(() => { + vi.useRealTimers(); + }); + const fake = fakeAgent(turn); + const stopped = vi.fn<() => void>(); + + const threads = new AskThreads( + async (...args) => { + const process = await fake.launch(...args); + + return { + ...process, + stop: () => { + stopped(); + process.stop(); + }, + }; + }, + {}, + { ...askThreadLimits, idleCloseMs: 1_000 }, + ); + + const thread = threads.open(idleAsk()); + + while (thread.read().status !== status) + await new Promise((resolve) => setImmediate(resolve)); + + await vi.advanceTimersByTimeAsync(999); + expect(stopped).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + expect(threads.get(thread.id)).toBeUndefined(); + expect(stopped).toHaveBeenCalled(); + }, +); diff --git a/packages/review/src/ask/thread.ts b/packages/review/src/ask/thread.ts index b1eba4483..373c61d13 100644 --- a/packages/review/src/ask/thread.ts +++ b/packages/review/src/ask/thread.ts @@ -156,6 +156,7 @@ export class AskThread { /** The number of changes so far; a snapshot carries the one it includes. */ private seq = 0; private readonly listeners = new Set<(update: AskUpdate) => void>(); + private watched?: (unwatched: boolean) => void; private readonly closers = new Set<() => void>(); private readonly decisions = new Map< string, @@ -246,8 +247,27 @@ export class AskThread { /** Every change after the current `seq`, in order. */ subscribe(listener: (update: AskUpdate) => void) { this.listeners.add(listener); + this.noteWatchers(); - return () => this.listeners.delete(listener); + return () => { + this.listeners.delete(listener); + this.noteWatchers(); + }; + } + + /** Tells `watch`, on each change, whether the thread is unwatched and not working. */ + onUnwatched(watch: (unwatched: boolean) => void) { + this.watched = watch; + this.noteWatchers(); + } + + private noteWatchers() { + if (!this.closed) + this.watched?.( + !this.listeners.size && + this.state.status !== "starting" && + this.state.status !== "running", + ); } /** Starts the agent and asks the first question, or loads an earlier @@ -1396,5 +1416,6 @@ export class AskThread { const update = { seq: this.seq, change }; for (const listener of this.listeners) listener(update); + this.noteWatchers(); } } diff --git a/packages/review/src/ask/threads.ts b/packages/review/src/ask/threads.ts index 545bbd2b8..86481ebea 100644 --- a/packages/review/src/ask/threads.ts +++ b/packages/review/src/ask/threads.ts @@ -1,5 +1,5 @@ import { PROTOCOL_VERSION, client, methods } from "@agentclientprotocol/sdk"; -import { errorMessage } from "@dev.fast/trace-core"; +import { errorMessage, shellQuote } from "@dev.fast/trace-core"; import { type AskAgentLauncher, askAgentTakesMcp, @@ -138,6 +138,52 @@ export interface AskTools { takesMcp?: (agent: AskAgentId) => Promise; } +/** Whiteboard's CLI as the tools, with `selector` in its env so it reaches + * this server and no other. */ +export function cliAskTools( + cliPath: () => string | undefined, + selector: { name: string; value: string }, +): AskTools { + const env = () => [ + selector, + ...(process.versions.electron + ? [{ name: "ELECTRON_RUN_AS_NODE", value: "1" }] + : []), + ...(process.env.DEV_REVIEW_HOME + ? [{ name: "DEV_REVIEW_HOME", value: process.env.DEV_REVIEW_HOME }] + : []), + ]; + + return { + mcpServers: () => { + const cli = cliPath(); + + return cli + ? [ + { + name: "whiteboard", + command: process.execPath, + args: [cli, "mcp"], + env: env(), + }, + ] + : []; + }, + cli: () => { + const cli = cliPath(); + + return ( + cli && + [ + ...env().map(({ name, value }) => `${name}=${shellQuote(value)}`), + shellQuote(process.execPath), + shellQuote(cli), + ].join(" ") + ); + }, + }; +} + /** How an agent reaches Whiteboard's tools, which its first prompt says. */ export type AskToolsReach = | { kind: "mcp" } @@ -156,7 +202,9 @@ export class AskThreads { constructor( private readonly launch: AskAgentLauncher, private readonly tools: AskTools = {}, - private readonly limits: AskThreadLimits = askThreadLimits, + private readonly limits: AskThreadLimits & { + idleCloseMs?: number; + } = askThreadLimits, ) { this.mcpServers = tools.mcpServers ?? (() => []); } @@ -183,6 +231,22 @@ export class AskThreads { ); this.threads.set(thread.id, thread); + const { idleCloseMs } = this.limits; + + if (idleCloseMs !== undefined) { + let timer: ReturnType | undefined; + + thread.onUnwatched((unwatched) => { + if (unwatched) + timer ??= setTimeout(() => this.close(thread.id), idleCloseMs); + else { + clearTimeout(timer); + timer = undefined; + } + }); + thread.onClose(() => clearTimeout(timer)); + } + void thread.open(); return thread; diff --git a/packages/review/src/cli-install.ts b/packages/review/src/cli-install.ts index 58461b4fd..052c74318 100644 --- a/packages/review/src/cli-install.ts +++ b/packages/review/src/cli-install.ts @@ -6,6 +6,7 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; import { + REVIEW_REMOTE_WRAPPER_MARK, type ReviewCliInstallStamp, ReviewCliInstallStampSchema, type ReviewCliInstallStatus, @@ -58,7 +59,8 @@ function hasManagedShimMarker(source: string): boolean { return ( source.includes(SHIM_MARKER) || source.includes("Managed by Review Desktop") || - source.includes("Managed by Whiteboard Desktop") + source.includes("Managed by Whiteboard Desktop") || + source.includes(REVIEW_REMOTE_WRAPPER_MARK) ); } diff --git a/packages/review/src/cli-runner.ts b/packages/review/src/cli-runner.ts index 72bb944da..840926f9d 100644 --- a/packages/review/src/cli-runner.ts +++ b/packages/review/src/cli-runner.ts @@ -2,9 +2,14 @@ import { existsSync } from "node:fs"; import { realpath } from "node:fs/promises"; import path from "node:path"; import { createInterface } from "node:readline/promises"; -import type { Writable } from "node:stream"; +import { Writable } from "node:stream"; import { fileURLToPath } from "node:url"; +import { + REVIEW_CLI_INSTALL_TARGET_LABELS, + REVIEW_REMOTE_ATTACH_BEGIN, + REVIEW_REMOTE_ATTACH_END, +} from "@dev.fast/review-protocol"; import { StoreApiError, processIsAlive, @@ -42,6 +47,12 @@ import { windowsInstallerCommand, } from "./cli-install"; import { cliRuntimeInfo, describeCliRuntime } from "./cli-runtime-info"; +import { + AGENT_CONNECT_TARGETS, + type AgentConnectTarget, + connectAgents, + detectAgents, +} from "./connect-agents"; import { connectPrompts } from "./connect-prompts"; import { ReviewInstanceUnavailableError, @@ -86,6 +97,7 @@ import { type ReviewTelemetryErrorName, } from "./review-telemetry"; import { + type ReviewServerDiscovery, readReviewServerDiscovery, readReviewServerHealth, reviewServerIsHealthy, @@ -265,6 +277,16 @@ export async function runReviewCli(input: ReviewCliInput): Promise { ? { ...env, DEV_REVIEW_SERVER_DIR: path.resolve(cwd, stateDir) } : env; + const refuse = (message: string, json?: boolean) => { + if (json) + emitReviewEvent(input.stdout, { + event: "error", + error: { name: "Error", message }, + }); + else input.stderr.write(`${message}\n`); + state.exitCode = 1; + }; + const serverCommand = configureOutput( program .command("server") @@ -291,6 +313,16 @@ export async function runReviewCli(input: ReviewCliInput): Promise { "--software-maps", "allow the authoring skill to generate optional software maps", ) + .option( + "--detach", + "start the server in the background, logging to review-server/server.log", + ) + .addOption( + new Option("--started-by ") + .choices(["user", "cli", "desktop"]) + .default("user") + .hideHelp(), + ) // Batch authoring was removed; name that instead of "unknown option". .addOption(new Option("--authoring-mode ").hideHelp()), "plain", @@ -299,6 +331,8 @@ export async function runReviewCli(input: ReviewCliInput): Promise { stateDir?: string; port: string; softwareMaps?: boolean; + detach?: boolean; + startedBy: "user" | "cli" | "desktop"; authoringMode?: string; json?: boolean; }>(); @@ -314,17 +348,40 @@ export async function runReviewCli(input: ReviewCliInput): Promise { "--port must be an integer between 0 and 65535.", ); const stateDir = reviewServerStateDir(authoringEnv(options.stateDir)); + + if (options.detach) { + const { ensureBackgroundServer } = + await import("./server/background-server.js"); + + const { discovery, started } = await ensureBackgroundServer({ + stateDir, + env, + args: [ + "--port", + `${port}`, + ...(options.softwareMaps ? ["--software-maps"] : []), + ], + }); + + await writeServerStatus(discovery, stateDir, options.json, { started }); + + return; + } + const controller = new AbortController(); const stop = () => controller.abort(); process.once("SIGINT", stop); process.once("SIGTERM", stop); + const { HeadlessServerBusyError, runHeadlessServer } = + await import("./server/headless-host.js"); + try { - const { runHeadlessServer } = await import("./server/headless-host.js"); await runHeadlessServer({ stateDir, port, softwareMapEnabled: options.softwareMaps, + startedBy: options.startedBy, signal: controller.signal, telemetry, onReady: ({ url, serverPid }) => { @@ -335,6 +392,9 @@ export async function runReviewCli(input: ReviewCliInput): Promise { ); }, }); + } catch (error) { + if (!(error instanceof HeadlessServerBusyError)) throw error; + refuse(error.message, options.json); } finally { process.off("SIGINT", stop); process.off("SIGTERM", stop); @@ -358,15 +418,64 @@ export async function runReviewCli(input: ReviewCliInput): Promise { const stateDir = reviewServerStateDir(authoringEnv(options.stateDir)); const discovery = await readReviewServerDiscovery(stateDir); - const health = discovery && (await readReviewServerHealth(discovery)); - if (!discovery || !health) throw serverNotReady(stateDir); - const { url, serverPid } = discovery; - const { version, serverId } = health; + if (!discovery) throw serverNotReady(stateDir); + await writeServerStatus(discovery, stateDir, options.json); + }); + + configureJsonOutput( + serverCommand + .command("stop") + .description( + "Stop a background server started by --detach, the CLI, or Desktop", + ) + .option( + "--state-dir ", + "directory selected when starting the server", + ), + "plain", + ).action(async (_options, command: Command) => { + const options = command.optsWithGlobals<{ + stateDir?: string; + json?: boolean; + }>(); + + const stateDir = reviewServerStateDir(authoringEnv(options.stateDir)); + + const { headlessServerOwner, stopBackgroundServer } = + await import("./server/background-server.js"); + + const owner = await headlessServerOwner(stateDir); + const discovery = await readReviewServerDiscovery(stateDir); + + if (owner === undefined) { + input.stdout.write( + options.json + ? `${JSON.stringify({ event: "server.stop", stopped: false, stateDir })}\n` + : `No Whiteboard server is running in ${stateDir}.\n`, + ); + + return; + } + + const serverPid = owner; + + if (discovery?.serverPid !== owner) + throw new Error( + `Process ${owner} holds the Whiteboard server's lock in ${stateDir} but has published no server; it may be starting. Try again, or end that process.`, + ); + + if (discovery.startedBy === "user") + throw new Error( + `The Whiteboard server in ${stateDir} (process ${serverPid}) runs in the foreground of \`whiteboard server start\`. Stop it there with Ctrl-C.`, + ); + + await stopBackgroundServer(discovery); + input.stdout.write( options.json - ? `${JSON.stringify({ event: "server.status", ready: true, url, serverPid, stateDir, version, serverId })}\n` - : `Whiteboard server ready at ${url}\nSaved reviews: ${stateDir}\n`, + ? `${JSON.stringify({ event: "server.stop", stopped: true, serverPid, stateDir })}\n` + : `Stopped the Whiteboard server (process ${serverPid}).\n`, ); }); @@ -407,10 +516,7 @@ export async function runReviewCli(input: ReviewCliInput): Promise { `Cannot tell whether a Whiteboard Desktop is using ${stateDir}: ${problem.message}`, ); - const inUse = () => - new Error( - `A Whiteboard server is using ${stateDir}. Stop it first, then run whiteboard server reset-id again.`, - ); + const inUse = `A Whiteboard server is using ${stateDir}. Stop it first, then run whiteboard server reset-id again.`; if ( desktops.instances.some(({ discovery }) => @@ -419,7 +525,7 @@ export async function runReviewCli(input: ReviewCliInput): Promise { ), ) ) - throw inUse(); + return refuse(inUse, options.json); const [{ openReviewProfile }, { withHeadlessServerLock }] = await Promise.all([ @@ -445,7 +551,7 @@ export async function runReviewCli(input: ReviewCliInput): Promise { }, ); - if (!reset.acquired) throw inUse(); + if (!reset.acquired) return refuse(inUse, options.json); const serverId = reset.result; input.stdout.write( @@ -455,6 +561,239 @@ export async function runReviewCli(input: ReviewCliInput): Promise { ); }); + const remote = configureOutput( + program + .command("remote") + .description("Serve reviews to Whiteboard Desktop over SSH"), + "plain", + ); + + configureJsonOutput( + remote + .command("attach") + .description( + "Start a background server if none is healthy and report how to reach it", + ) + .option( + "--state-dir ", + "directory for saved reviews and server discovery", + ) + .option( + "--groups ", + "comma-separated optional extension groups the Desktop has enabled, such as go", + ) + .option( + "--replace", + "stop a running server of another version that the CLI or Desktop started, and start this one", + ), + "plain", + ).action(async (_options, command: Command) => { + const options = command.optsWithGlobals<{ + stateDir?: string; + json?: boolean; + groups?: string; + replace?: boolean; + }>(); + + const stateDir = reviewServerStateDir(authoringEnv(options.stateDir)); + + const { remoteAttach } = await import("./remote-attach.js"); + + let attach: Awaited>; + + try { + attach = await remoteAttach({ + stateDir, + env, + groups: options.groups?.split(",").map((group) => group.trim()), + replace: options.replace, + }); + } catch (error) { + if (!options.json) throw error; + + let line = ""; + + emitReviewEvent( + new Writable({ + write(chunk, _encoding, done) { + line += chunk; + done(); + }, + }), + { event: "error", error: serializeReviewError(error) }, + ); + input.stdout.write( + `${REVIEW_REMOTE_ATTACH_BEGIN}\n${ensureTrailingNewline(line)}${REVIEW_REMOTE_ATTACH_END}\n`, + ); + state.exitCode = 1; + + return; + } + + input.stdout.write( + options.json + ? `${REVIEW_REMOTE_ATTACH_BEGIN}\n${JSON.stringify(attach)}\n${REVIEW_REMOTE_ATTACH_END}\n` + : `Whiteboard server ${attach.startedServer ? "started" : "already running"} at ${attach.url}\nLanguage features: ${attach.languageServer ? `VS Code server on port ${attach.languageServer.port}` : `unavailable (${attach.languageServerDetail})`}\n`, + ); + }); + + configureJsonOutput( + remote + .command("extensions") + .description("Language extensions for the VS Code server on a remote") + .command("ensure") + .description( + "Download the curated extensions for this machine and list them for the VS Code server", + ) + .option( + "--groups ", + "comma-separated optional extension groups the Desktop has enabled, such as go", + ), + "plain", + ).action(async (_options, command: Command) => { + const { json, groups } = command.optsWithGlobals<{ + json?: boolean; + groups?: string; + }>(); + + const { ensureRemoteExtensions } = await import("./remote-extensions.js"); + + const result = await ensureRemoteExtensions({ + env, + groups: groups?.split(",").map((group) => group.trim()), + }); + + input.stdout.write( + json + ? `${JSON.stringify(result)}\n` + : [ + ...result.installed.map((id) => `Installed ${id}`), + ...result.skipped.map( + ({ id, reason }) => `Skipped ${id}: ${reason}`, + ), + ...result.failed.map(({ id, error }) => `Failed ${id}: ${error}`), + ] + .map((line) => `${line}\n`) + .join(""), + ); + state.exitCode = result.failed.length > 0 ? 1 : 0; + }); + + configureJsonOutput( + remote + .command("uninstall") + .description( + "Remove what Whiteboard Desktop installed on this host, stopping a server it started", + ) + .addOption( + new Option( + "--keep-reviews", + "leave the saved reviews in place", + ).conflicts("deleteReviews"), + ) + .addOption( + new Option("--delete-reviews", "also delete the saved reviews"), + ) + .option( + "--state-dir ", + "directory for saved reviews and server discovery", + ), + "plain", + ).action(async (_options, command: Command) => { + const options = command.optsWithGlobals<{ + stateDir?: string; + json?: boolean; + keepReviews?: boolean; + deleteReviews?: boolean; + }>(); + + const stateDir = reviewServerStateDir(authoringEnv(options.stateDir)); + const { remoteUninstall } = await import("./remote-uninstall.js"); + + let deleteReviews = options.deleteReviews + ? true + : options.keepReviews + ? false + : undefined; + + if (deleteReviews === undefined && !options.json && input.stdin?.isTTY) { + const prompt = createInterface({ + input: input.stdin, + output: input.stderr, + }); + + try { + deleteReviews = /^y(es)?$/i.test( + ( + await prompt.question( + `Also delete the saved reviews in ${stateDir}? [y/N] `, + ) + ).trim(), + ); + } finally { + prompt.close(); + } + } + + const result = + deleteReviews === undefined + ? { + event: "remote.uninstall" as const, + ok: false as const, + reason: + "Say what happens to the saved reviews: pass --keep-reviews or --delete-reviews.", + } + : await remoteUninstall({ + home: env.HOME ?? "", + env, + stateDir, + deleteReviews, + }); + + if (!result.ok) { + if (!options.json) throw new Error(result.reason); + input.stdout.write(`${JSON.stringify(result)}\n`); + state.exitCode = 1; + + return; + } + + input.stdout.write( + options.json + ? `${JSON.stringify(result)}\n` + : [ + ...(result.stoppedServer + ? [ + `Stopped the Whiteboard server (process ${result.stoppedServer.pid}).`, + ] + : []), + ...result.removed.map((removed) => `Removed ${removed}`), + result.keptReviews + ? `Saved reviews kept in ${stateDir}` + : "Saved reviews deleted.", + "", + ].join("\n"), + ); + }); + + async function writeServerStatus( + discovery: ReviewServerDiscovery, + stateDir: string, + json: boolean | undefined, + extra: { started?: boolean } = {}, + ) { + const health = await readReviewServerHealth(discovery); + + if (!health) throw serverNotReady(stateDir); + const { url, serverPid } = discovery; + const { version, serverId } = health; + input.stdout.write( + json + ? `${JSON.stringify({ event: "server.status", ready: true, url, serverPid, stateDir, version, serverId, ...extra })}\n` + : `Whiteboard server ${extra.started === false ? "already running" : "ready"} at ${url}\nSaved reviews: ${stateDir}\n`, + ); + } + configureJsonOutput( program .command("version") @@ -662,53 +1001,125 @@ export async function runReviewCli(input: ReviewCliInput): Promise { "copilot", "all", ]), + ) + .addOption( + new Option( + "--detect", + "list the agents on this machine and whether each is connected; changes nothing", + ).conflicts("yes"), + ) + .option( + "--yes", + "run the commands the prompt gives, without an agent: claude, codex, opencode or pi", ), "plain", ); - connect.action(async (targets: string[], options: { json?: boolean }) => { - const selected = parseTargets(targets); + connect.action( + async ( + targets: string[], + options: { json?: boolean; detect?: boolean; yes?: boolean }, + ) => { + const output = { + json: options.json, + stdout: input.stdout, + stderr: input.stderr, + }; + + if (options.detect) { + const agents = await detectAgents({ homeDir: scope.homeDir, env }); + + if (options.json) { + emitJsonEvent(output, { event: "connect.detect", agents }); + + return; + } + + humanStream(output).write( + agents.length + ? `${agents.map((agent) => `${agent.name}: ${agent.connected ? "connected" : "not connected"}${agent.manual ? " (its command is not on PATH)" : ""}`).join("\n")}\n` + : "No agents found.\n", + ); - const { homeDir, devHome } = scope; + return; + } + + if (options.yes) { + const agents = parseTargets(targets); + + if ( + targets.length === 0 || + !agents.every((target): target is AgentConnectTarget => + AGENT_CONNECT_TARGETS.some((known) => known === target), + ) + ) + throw new Error( + "connect --yes takes one or more of claude, codex, opencode or pi.", + ); - const prompts = connectPrompts({ - legacyPaths: await scanLegacySkills(homeDir), - hasShim: - (await isOwnedShim(pathShimPath(homeDir))) || - (await windowsInstallerCommand( - findReviewPackageRoot(import.meta.url), + const results = await connectAgents({ + agents, + homeDir: scope.homeDir, env, - )) !== undefined, - traceEnabled: await traceMachineEnabled({ homeDir, env }), - fffBinaryPath: path.join(homeDir, ".local", "bin", "fff-mcp"), - fffCorpusRoot: path.join(devHome, "trace-search"), - }); + }); - const output = { - json: options.json, - stdout: input.stdout, - stderr: input.stderr, - }; + state.exitCode = results.every((result) => result.connected) ? 0 : 1; - if (options.json) { - emitJsonEvent(output, { - event: "connect", - prompts: Object.fromEntries( - selected.map((target) => [target, prompts[target]]), - ), + if (options.json) { + emitJsonEvent(output, { event: "connect.run", agents: results }); + + return; + } + + humanStream(output).write( + results + .map( + (result) => + `${result.output}${result.name}: ${result.connected ? "connected" : "not connected"}\n`, + ) + .join(""), + ); + + return; + } + + const selected = parseTargets(targets); + + const { homeDir, devHome } = scope; + + const prompts = connectPrompts({ + legacyPaths: await scanLegacySkills(homeDir), + hasShim: + (await isOwnedShim(pathShimPath(homeDir))) || + (await windowsInstallerCommand( + findReviewPackageRoot(import.meta.url), + env, + )) !== undefined, + traceEnabled: await traceMachineEnabled({ homeDir, env }), + fffBinaryPath: path.join(homeDir, ".local", "bin", "fff-mcp"), + fffCorpusRoot: path.join(devHome, "trace-search"), }); - return; - } + if (options.json) { + emitJsonEvent(output, { + event: "connect", + prompts: Object.fromEntries( + selected.map((target) => [target, prompts[target]]), + ), + }); - const sections = selected.map((target) => - selected.length > 1 - ? `## ${TARGET_LABELS[target]}\n\n${prompts[target]}` - : prompts[target], - ); + return; + } - humanStream(output).write(`${sections.join("\n\n")}\n`); - }); + const sections = selected.map((target) => + selected.length > 1 + ? `## ${REVIEW_CLI_INSTALL_TARGET_LABELS[target]}\n\n${prompts[target]}` + : prompts[target], + ); + + humanStream(output).write(`${sections.join("\n\n")}\n`); + }, + ); const migrate = configureOutput( program.command("migrate", { hidden: true }), @@ -1137,16 +1548,6 @@ export async function runReviewCli(input: ReviewCliInput): Promise { } } -const TARGET_LABELS: Record = { - claude: "Claude Code", - codex: "Codex", - cursor: "Cursor", - opencode: "OpenCode", - pi: "Pi", - omp: "oh-my-pi", - copilot: "Copilot CLI", -}; - function parseTargets(targets: readonly string[]): InstallTarget[] { if (targets.length === 0 || targets.includes("all")) { return [...ALL_INSTALL_TARGETS]; diff --git a/packages/review/src/cli.test.ts b/packages/review/src/cli.test.ts index c228b95da..5525d511e 100644 --- a/packages/review/src/cli.test.ts +++ b/packages/review/src/cli.test.ts @@ -4,12 +4,14 @@ import { readFile, readdir, rm, + stat, writeFile, } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { PassThrough, Readable } from "node:stream"; +import { REVIEW_REMOTE_WRAPPER_MARK } from "@dev.fast/review-protocol"; import { StoreClient, runTraceSessions as runTraceSessionsActual, @@ -116,6 +118,110 @@ describe("Whiteboard CLI", () => { expect(Object.keys(event.prompts)).toEqual(["claude"]); }); + it("counts the launcher Desktop writes on a remote host as the managed command", async () => { + const { code, stdout } = await runConnect( + ["connect", "claude", "--json"], + async (home) => { + await mkdir(path.join(home, ".local", "bin"), { recursive: true }); + await writeFile( + path.join(home, ".local", "bin", "whiteboard"), + `#!/bin/sh\n${REVIEW_REMOTE_WRAPPER_MARK}\nexec '${home}/.dev/whiteboard-remote/versions/0.1.6/whiteboard' "$@"\n`, + ); + }, + ); + + expect(code).toBe(0); + const prompt = JSON.parse(stdout).prompts.claude; + expect(prompt).toContain("claude plugin install whiteboard@devfast"); + expect(prompt).not.toContain("install the whiteboard command"); + }); + + it("lists the agents whose configuration exists, as one JSON line", async () => { + const { code, stdout } = await runConnect( + ["connect", "--detect", "--json"], + async (home) => { + await mkdir(path.join(home, ".codex")); + }, + ); + + expect(code).toBe(0); + expect(stdout.trimEnd().split("\n")).toHaveLength(1); + expect(JSON.parse(stdout)).toEqual({ + event: "connect.detect", + agents: [ + { + id: "codex", + name: "Codex", + connected: false, + manual: true, + }, + ], + }); + }); + + it("detects without changing any file", async () => { + let before = ""; + let after = ""; + + const { code, stdout } = await runConnect( + ["connect", "--detect", "--json"], + async (home) => { + await installTestShim(home); + await mkdir(path.join(home, ".claude", "plugins"), { recursive: true }); + await writeFile( + path.join(home, ".claude", "plugins", "installed_plugins.json"), + "{}", + ); + await mkdir(path.join(home, ".pi")); + before = await fileTree(home); + }, + { after: async (home) => void (after = await fileTree(home)) }, + ); + + expect(code).toBe(0); + expect( + JSON.parse(stdout).agents.map((agent: { id: string }) => agent.id), + ).toEqual(["claude", "pi"]); + expect(after).toBe(before); + }); + + it("connects one agent with its prompt's commands, and reports it connected", async () => { + const { code, stdout } = await runConnect( + ["connect", "--yes", "pi", "--json"], + async (home) => { + await installTestShim(home); + await mkdir(path.join(home, ".pi")); + await mkdir(path.join(home, "bin")); + await writeFile( + path.join(home, "bin", "pi"), + `#!/bin/sh\nmkdir -p "$HOME/.pi/agent" && printf '{"packages":["%s"]}' "$2" > "$HOME/.pi/agent/settings.json"\n`, + { mode: 0o755 }, + ); + }, + { env: (home) => ({ PATH: `${path.join(home, "bin")}:/usr/bin:/bin` }) }, + ); + + expect(code).toBe(0); + expect(JSON.parse(stdout)).toMatchObject({ + event: "connect.run", + agents: [{ id: "pi", name: "Pi", connected: true }], + }); + }); + + it("refuses to run commands for an agent it cannot connect alone", async () => { + const { code, stdout } = await runConnect([ + "connect", + "--yes", + "cursor", + "--json", + ]); + + expect(code).toBe(1); + expect(JSON.parse(stdout).error.message).toContain( + "claude, codex, opencode or pi", + ); + }); + it("routes own-upload status filters without requesting trace content", async () => { const runTraceStatus = vi.fn(async () => 0); @@ -756,18 +862,20 @@ describe("Whiteboard CLI", () => { await mkdir(directory, { recursive: true }); await writeFile(path.join(directory, "review.json"), record); }, - async (home) => { - const directory = path.join( - home, - ".dev", - "reviews", - "11111111-1111-4111-8111-111111111111", - ); - - expect( - await readFile(path.join(directory, "review.json"), "utf8"), - ).toBe(record); - expect(await readdir(directory)).toEqual(["review.json"]); + { + after: async (home) => { + const directory = path.join( + home, + ".dev", + "reviews", + "11111111-1111-4111-8111-111111111111", + ); + + expect( + await readFile(path.join(directory, "review.json"), "utf8"), + ).toBe(record); + expect(await readdir(directory)).toEqual(["review.json"]); + }, }, ); @@ -788,6 +896,23 @@ function outputStream(): PassThrough { return new PassThrough(); } +async function fileTree(root: string): Promise { + const entries = await readdir(root, { recursive: true, withFileTypes: true }); + + const lines = await Promise.all( + entries.map(async (entry) => { + const file = path.join(entry.parentPath, entry.name); + const info = await stat(file); + + return `${path.relative(root, file)} ${info.size} ${info.mode} ${info.mtimeMs}`; + }), + ); + + const top = await stat(root); + + return [`. ${top.mtimeMs}`, ...lines.sort()].join("\n"); +} + async function installTestShim(home: string): Promise { const bin = path.join(home, ".local", "bin"); await mkdir(bin, { recursive: true }); @@ -797,7 +922,10 @@ async function installTestShim(home: string): Promise { async function runConnect( argv: string[], setup?: (homeDir: string) => Promise, - verify?: (homeDir: string) => Promise, + extra?: { + env?: (homeDir: string) => NodeJS.ProcessEnv; + after?: (homeDir: string) => Promise; + }, ): Promise<{ code: number; stdout: string; stderr: string }> { const homeDir = await mkdtemp(path.join(os.tmpdir(), "review-connect-")); @@ -820,12 +948,13 @@ async function runConnect( HOME: homeDir, TRACE_HOME_DIR: homeDir, DEV_REVIEW_HOME: path.join(homeDir, ".dev"), + ...extra?.env?.(homeDir), }, stdout, stderr, }); - await verify?.(homeDir); + await extra?.after?.(homeDir); return { code, stdout: stdoutText, stderr: stderrText }; } finally { diff --git a/packages/review/src/cli.ts b/packages/review/src/cli.ts index 0f9131613..6a4f766c0 100644 --- a/packages/review/src/cli.ts +++ b/packages/review/src/cli.ts @@ -71,7 +71,7 @@ async function maybeDelegateToDesktopCli( if ( argv.some( (argument) => - ["api", "mcp", "server", "instances"].includes(argument) || + ["api", "mcp", "server", "remote", "instances"].includes(argument) || /^--state-dir(?:=|$)/.test(argument), ) || env.DEV_REVIEW_SERVER_DIR?.trim() diff --git a/packages/review/src/connect-agents.test.ts b/packages/review/src/connect-agents.test.ts new file mode 100644 index 000000000..caee7669f --- /dev/null +++ b/packages/review/src/connect-agents.test.ts @@ -0,0 +1,298 @@ +import { + chmod, + mkdir, + mkdtemp, + readFile, + rm, + writeFile, +} from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { connectAgents, detectAgents } from "./connect-agents"; + +const homes: string[] = []; + +afterEach(async () => { + await Promise.all( + homes.splice(0).map((home) => rm(home, { recursive: true, force: true })), + ); +}); + +async function fakeHome(): Promise<{ + homeDir: string; + bin: string; + env: NodeJS.ProcessEnv; +}> { + const homeDir = await mkdtemp(path.join(os.tmpdir(), "wb-agents-")); + homes.push(homeDir); + const bin = path.join(homeDir, "bin"); + await mkdir(bin); + + return { + homeDir, + bin, + env: { HOME: homeDir, PATH: `${bin}:/usr/bin:/bin`, XDG_CONFIG_HOME: "" }, + }; +} + +async function fakeCli(bin: string, name: string, body = ""): Promise { + const file = path.join(bin, name); + await writeFile( + file, + `#!/bin/sh\necho "${name} $*" >> "$HOME/calls.log"\n${body}\n`, + ); + await chmod(file, 0o755); +} + +const write = async (file: string, text: string) => { + await mkdir(path.dirname(file), { recursive: true }); + await writeFile(file, text); +}; + +describe("detectAgents", () => { + it("lists only the harnesses whose configuration directory exists", async () => { + const { homeDir, bin, env } = await fakeHome(); + await mkdir(path.join(homeDir, ".codex")); + await fakeCli(bin, "codex"); + + expect(await detectAgents({ homeDir, env })).toEqual([ + { id: "codex", name: "Codex", connected: false }, + ]); + }); + + it("reads each harness's own record of the plugin or package", async () => { + const { homeDir, bin, env } = await fakeHome(); + + for (const cli of ["claude", "codex", "opencode", "pi"]) + await fakeCli(bin, cli); + await write( + path.join(homeDir, ".claude", "plugins", "installed_plugins.json"), + JSON.stringify({ + version: 2, + plugins: { "whiteboard@devfast": [{ scope: "user" }] }, + }), + ); + await write( + path.join(homeDir, ".codex", "config.toml"), + 'model = "x"\n\n[plugins."whiteboard@devfast"]\nenabled = true\n', + ); + await write( + path.join(homeDir, ".config", "opencode", "opencode.jsonc"), + '{\n // mine\n "plugin": [\n "@dev.fast/opencode-whiteboard@0.1.0"\n ]\n}\n', + ); + await write( + path.join(homeDir, ".pi", "agent", "settings.json"), + JSON.stringify({ packages: [{ source: "npm:@dev.fast/pi-whiteboard" }] }), + ); + + expect( + (await detectAgents({ homeDir, env })).map(({ id, connected }) => [ + id, + connected, + ]), + ).toEqual([ + ["claude", true], + ["codex", true], + ["opencode", true], + ["pi", true], + ]); + }); + + it("finds OpenCode's whiteboard MCP server in its JSONC config", async () => { + const opencode = async (text: string) => { + const { homeDir, bin, env } = await fakeHome(); + await fakeCli(bin, "opencode"); + await write( + path.join(homeDir, ".config", "opencode", "opencode.jsonc"), + text, + ); + + return (await detectAgents({ homeDir, env }))[0]?.connected; + }; + + expect( + await opencode( + '{\n "$schema": "https://opencode.ai/config.json", // mine\n "mcp": {\n /* added */ "whiteboard": { "type": "local" },\n },\n}\n', + ), + ).toBe(true); + expect( + await opencode('{ "mcp": { "servers": { "whiteboard": {} } } }'), + ).toBe(true); + expect( + await opencode('{ "mcp": { "other": { "url": "//whiteboard" } } }'), + ).toBe(false); + }); + + it("is not connected by a disabled or other plugin, and a harness without its CLI on PATH is manual", async () => { + const { homeDir, env } = await fakeHome(); + await write( + path.join(homeDir, ".codex", "config.toml"), + '[plugins."whiteboard@devfast"]\nenabled = false\n[plugins."other@devfast"]\nenabled = true\n', + ); + await write( + path.join(homeDir, ".pi", "agent", "settings.json"), + JSON.stringify({ packages: ["npm:@dev.fast/pi-whiteboard-extra"] }), + ); + + expect(await detectAgents({ homeDir, env })).toEqual([ + { + id: "codex", + name: "Codex", + connected: false, + manual: true, + }, + { id: "pi", name: "Pi", connected: false, manual: true }, + ]); + }); + + it("reads unreadable records as not connected", async () => { + const { homeDir, bin, env } = await fakeHome(); + await fakeCli(bin, "claude"); + await write( + path.join(homeDir, ".claude", "plugins", "installed_plugins.json"), + "{not json", + ); + + expect(await detectAgents({ homeDir, env })).toEqual([ + { id: "claude", name: "Claude Code", connected: false }, + ]); + }); +}); + +describe("connectAgents", () => { + it("runs one harness's prompt commands, and changes no other harness", async () => { + const { homeDir, bin, env } = await fakeHome(); + await mkdir(path.join(homeDir, ".codex")); + const codexConfig = path.join(homeDir, ".codex", "config.toml"); + await writeFile(codexConfig, 'model = "x"\n'); + await fakeCli(bin, "codex"); + await fakeCli( + bin, + "pi", + `mkdir -p "$HOME/.pi/agent" && printf '{"packages":["%s"]}' "$2" > "$HOME/.pi/agent/settings.json"`, + ); + await mkdir(path.join(homeDir, ".pi")); + + const results = await connectAgents({ agents: ["pi"], homeDir, env }); + + expect(results).toEqual([ + expect.objectContaining({ id: "pi", connected: true }), + ]); + expect(await readFile(path.join(homeDir, "calls.log"), "utf8")).toBe( + "pi install npm:@dev.fast/pi-whiteboard\n", + ); + expect(await readFile(codexConfig, "utf8")).toBe('model = "x"\n'); + }); + + it("adds OpenCode's MCP server with the whiteboard launch", async () => { + const { homeDir, bin, env } = await fakeHome(); + await mkdir(path.join(homeDir, ".config", "opencode"), { recursive: true }); + await fakeCli( + bin, + "opencode", + `printf '{"mcp":{"whiteboard":{}}}' > "$HOME/.config/opencode/opencode.json"`, + ); + + const [result] = await connectAgents({ + agents: ["opencode"], + homeDir, + env, + }); + + expect(result).toMatchObject({ id: "opencode", connected: true }); + expect(await readFile(path.join(homeDir, "calls.log"), "utf8")).toBe( + 'opencode mcp add --global whiteboard -- sh -c exec "$HOME/.local/bin/whiteboard" mcp\n', + ); + }); + + async function claudeHome(installExit: number) { + const home = await fakeHome(); + const registration = path.join(home.homeDir, ".claude.json"); + await mkdir(path.join(home.homeDir, ".claude")); + await writeFile(registration, '{"mcpServers":{"whiteboard":{}}}'); + await fakeCli( + home.bin, + "claude", + `case "$2" in +install) [ ${installExit} = 0 ] || { echo "network down" >&2; exit ${installExit}; } + mkdir -p "$HOME/.claude/plugins" && echo '{"version":2,"plugins":{"whiteboard@devfast":[{"scope":"user"}]}}' > "$HOME/.claude/plugins/installed_plugins.json" ;; +remove) rm "$HOME/.claude.json"; echo "Removed" ;; +esac`, + ); + + return { ...home, registration }; + } + + it("replaces an old registration only once the plugin is installed", async () => { + const { homeDir, env, registration } = await claudeHome(0); + + const [result] = await connectAgents({ agents: ["claude"], homeDir, env }); + + expect(result).toMatchObject({ id: "claude", connected: true }); + await expect(readFile(registration, "utf8")).rejects.toThrow("ENOENT"); + }); + + it("stops at a failed install, and keeps the old registration", async () => { + const { homeDir, env, registration } = await claudeHome(1); + + const [result] = await connectAgents({ agents: ["claude"], homeDir, env }); + + expect(result).toMatchObject({ id: "claude", connected: false }); + expect(result?.output).toContain("network down"); + expect(result?.output).toContain( + "Stopped: claude plugin install whiteboard@devfast --scope user failed.", + ); + expect(await readFile(registration, "utf8")).toContain("whiteboard"); + expect( + await readFile(path.join(homeDir, "calls.log"), "utf8"), + ).not.toContain("mcp remove"); + }); + + it("tolerates a cleanup that finds nothing to remove", async () => { + const { homeDir, bin, env } = await fakeHome(); + await mkdir(path.join(homeDir, ".codex")); + await fakeCli( + bin, + "codex", + `case "$1 $2" in +"plugin add") printf '[plugins."whiteboard@devfast"]\nenabled = true\n' > "$HOME/.codex/config.toml" ;; +"mcp remove") echo "No MCP server named whiteboard" >&2; exit 1 ;; +esac`, + ); + + const [result] = await connectAgents({ agents: ["codex"], homeDir, env }); + + expect(result).toMatchObject({ id: "codex", connected: true }); + expect(result?.output).not.toContain("Stopped"); + }); + + it("judges by the harness's record afterwards, and reports a failure with its output", async () => { + const { homeDir, bin, env } = await fakeHome(); + await mkdir(path.join(homeDir, ".codex")); + await fakeCli(bin, "codex", 'echo "network down" >&2; exit 2'); + + const [result] = await connectAgents({ agents: ["codex"], homeDir, env }); + + expect(result).toMatchObject({ id: "codex", connected: false }); + expect(result?.output).toContain("network down"); + }); + + it("ends a command that outlives the bound", async () => { + const { homeDir, bin, env } = await fakeHome(); + await mkdir(path.join(homeDir, ".pi")); + await fakeCli(bin, "pi", "sleep 30"); + + const [result] = await connectAgents({ + agents: ["pi"], + homeDir, + env, + timeoutMs: 200, + }); + + expect(result).toMatchObject({ id: "pi", connected: false }); + expect(result?.output).toContain("did not finish within"); + }); +}); diff --git a/packages/review/src/connect-agents.ts b/packages/review/src/connect-agents.ts new file mode 100644 index 000000000..f7498a599 --- /dev/null +++ b/packages/review/src/connect-agents.ts @@ -0,0 +1,285 @@ +import { spawn } from "node:child_process"; +import { constants } from "node:fs"; +import { access, readFile } from "node:fs/promises"; +import path from "node:path"; + +import { + REVIEW_CLI_INSTALL_TARGET_LABELS, + REVIEW_REMOTE_AGENT_IDS, + type ReviewRemoteAgentId, +} from "@dev.fast/review-protocol"; +import { + type AgentTraceHookAgent, + agentTraceHomeDirectory, +} from "@dev.fast/trace-core"; +import { parse as parseToml } from "smol-toml"; +import { z } from "zod"; + +import { CONNECT_COMMANDS, type ConnectCommand } from "./connect-prompts"; +import { isDirectory } from "./fs-utils"; + +export const AGENT_CONNECT_TARGETS: readonly (ReviewRemoteAgentId & + AgentTraceHookAgent)[] = REVIEW_REMOTE_AGENT_IDS; + +export type AgentConnectTarget = ReviewRemoteAgentId; + +export interface DetectedAgent { + id: AgentConnectTarget; + name: string; + connected: boolean; + manual?: true; +} + +export interface ConnectedAgent { + id: AgentConnectTarget; + name: string; + connected: boolean; + output: string; +} + +const OUTPUT_LIMIT = 4000; + +export const AGENT_CONNECT_TIMEOUT_MS = 120_000; + +interface Scope { + homeDir: string; + env: NodeJS.ProcessEnv; +} + +export async function detectAgents(scope: Scope): Promise { + const found: DetectedAgent[] = []; + + for (const id of AGENT_CONNECT_TARGETS) { + if ( + !(await isDirectory( + agentTraceHomeDirectory(id, scope.homeDir, scope.env), + )) + ) + continue; + + const manual = !(await onPath(CONNECT_COMMANDS[id][0].argv[0], scope.env)); + + found.push({ + id, + name: REVIEW_CLI_INSTALL_TARGET_LABELS[id], + connected: await connected(id, scope), + ...(manual && { manual: true as const }), + }); + } + + return found; +} + +export async function connectAgents( + input: Scope & { + agents: readonly AgentConnectTarget[]; + timeoutMs?: number; + }, +): Promise { + const results: ConnectedAgent[] = []; + + for (const id of input.agents) { + const deadline = Date.now() + (input.timeoutMs ?? AGENT_CONNECT_TIMEOUT_MS); + let output = ""; + + const commands: readonly ConnectCommand[] = CONNECT_COMMANDS[id]; + + for (const { argv, ifAny } of commands) { + const left = deadline - Date.now(); + + if (left <= 0) break; + const step = await run(argv, input.env, left); + + output += `$ ${argv.join(" ")}\n${step.output}`; + + if (!step.ok && !ifAny) { + output += `Stopped: ${argv.join(" ")} failed.\n`; + break; + } + } + + results.push({ + id, + name: REVIEW_CLI_INSTALL_TARGET_LABELS[id], + connected: await connected(id, input), + output: output.slice(-OUTPUT_LIMIT), + }); + } + + return results; +} + +const WHITEBOARD_PLUGIN = "whiteboard@devfast"; + +const ClaudePluginsSchema = z.object({ + plugins: z.record(z.string(), z.unknown()), +}); + +const CodexConfigSchema = z.object({ + plugins: z.record(z.string(), z.unknown()).optional(), +}); + +const CodexPluginSchema = z.object({ enabled: z.boolean().optional() }); + +const PiSettingsSchema = z.object({ + packages: z.array( + z.union([ + z.string(), + z.object({ source: z.string() }).transform((entry) => entry.source), + ]), + ), +}); + +const PI_PACKAGE = /^npm:@dev\.fast\/pi-whiteboard(@.*)?$/; + +const OPENCODE_PLUGIN = + /"plugin"\s*:\s*\[[^\]]*"@dev\.fast\/opencode-whiteboard(@[^"]*)?"/; + +const OpenCodeConfigSchema = z.object({ + mcp: z.object({ + whiteboard: z.unknown().optional(), + servers: z.object({ whiteboard: z.unknown().optional() }).optional(), + }), +}); + +/** OpenCode's config is JSONC: drop comments and trailing commas, keep strings. */ +const JSONC_EXTRAS = + /("(?:\\.|[^"\\])*")|\/\/[^\n]*|\/\*[\s\S]*?\*\/|,(?=\s*[}\]])/g; + +function openCodeConnected(text: string): boolean { + if (OPENCODE_PLUGIN.test(text)) return true; + + try { + const { mcp } = OpenCodeConfigSchema.parse( + JSON.parse(text.replace(JSONC_EXTRAS, "$1")), + ); + + return ( + mcp.whiteboard !== undefined || mcp.servers?.whiteboard !== undefined + ); + } catch { + return false; + } +} + +async function connected( + id: AgentConnectTarget, + scope: Scope, +): Promise { + const home = agentTraceHomeDirectory(id, scope.homeDir, scope.env); + + const read = (...parts: string[]) => + readFile(path.join(home, ...parts), "utf8"); + + try { + switch (id) { + case "claude": { + const installs = ClaudePluginsSchema.parse( + JSON.parse(await read("plugins", "installed_plugins.json")), + ).plugins[WHITEBOARD_PLUGIN]; + + return Array.isArray(installs) + ? installs.length > 0 + : Boolean(installs); + } + + case "codex": { + const entry = CodexConfigSchema.parse( + parseToml(await read("config.toml")), + ).plugins?.[WHITEBOARD_PLUGIN]; + + return ( + entry !== undefined && + CodexPluginSchema.parse(entry).enabled !== false + ); + } + + case "opencode": { + for (const name of ["opencode.json", "opencode.jsonc", "config.json"]) { + if (openCodeConnected(await read(name).catch(() => ""))) return true; + } + + return false; + } + + case "pi": + return PiSettingsSchema.parse( + JSON.parse(await read("agent", "settings.json")), + ).packages.some((entry) => PI_PACKAGE.test(entry)); + } + } catch { + return false; + } +} + +async function onPath( + command: string, + env: NodeJS.ProcessEnv, +): Promise { + for (const directory of (env.PATH ?? "").split(path.delimiter)) { + if (!path.isAbsolute(directory)) continue; + const candidate = path.join(directory, command); + + try { + await access(candidate, constants.X_OK); + + return candidate; + } catch {} + } + + return undefined; +} + +async function run( + argv: readonly string[], + env: NodeJS.ProcessEnv, + timeoutMs: number, +): Promise<{ output: string; ok: boolean }> { + const [command = "", ...args] = argv; + const file = await onPath(command, env); + + if (!file) + return { output: `${command} was not found on PATH.\n`, ok: false }; + + return new Promise((resolve) => { + let output = ""; + + const keep = (chunk: Buffer) => { + output = (output + chunk.toString("utf8")).slice(-OUTPUT_LIMIT); + }; + + const child = spawn(file, args, { + env, + stdio: ["ignore", "pipe", "pipe"], + detached: true, + }); + + const timer = setTimeout(() => { + output += `\n${command} did not finish within ${Math.round(timeoutMs / 1000)} seconds.\n`; + + try { + process.kill(-child.pid!, "SIGKILL"); + } catch { + child.kill("SIGKILL"); + } + }, timeoutMs); + + child.stdout.on("data", keep); + child.stderr.on("data", keep); + child.once("error", (error) => { + clearTimeout(timer); + resolve({ output: `${output}${error.message}\n`, ok: false }); + }); + child.once("close", (code) => { + clearTimeout(timer); + resolve( + code === 0 + ? { output, ok: true } + : { + output: `${output}${output.endsWith("\n") || !output ? "" : "\n"}(exit ${code ?? "signal"})\n`, + ok: false, + }, + ); + }); + }); +} diff --git a/packages/review/src/connect-prompts.ts b/packages/review/src/connect-prompts.ts index 6dc718158..17a903235 100644 --- a/packages/review/src/connect-prompts.ts +++ b/packages/review/src/connect-prompts.ts @@ -89,6 +89,64 @@ export const CLAUDE_WINDOWS_MCP_ADD = `claude mcp add -s user whiteboard -- ${WI /** Copilot CLI loads the Claude plugin, so on Windows it too registers the server directly. */ export const COPILOT_WINDOWS_MCP_ADD = `copilot mcp add whiteboard -- ${WINDOWS_MCP_LAUNCH.command} ${WINDOWS_MCP_LAUNCH.args.join(" ")}`; +export interface ConnectCommand { + argv: readonly string[]; + ifAny?: true; +} + +export const CONNECT_COMMANDS = { + claude: [ + { + argv: ["claude", "plugin", "marketplace", "add", "devdotfast/whiteboard"], + }, + { + argv: [ + "claude", + "plugin", + "install", + "whiteboard@devfast", + "--scope", + "user", + ], + }, + { + argv: ["claude", "mcp", "remove", "-s", "user", "whiteboard"], + ifAny: true, + }, + ], + codex: [ + { + argv: ["codex", "plugin", "marketplace", "add", "devdotfast/whiteboard"], + }, + { argv: ["codex", "plugin", "add", "whiteboard@devfast"] }, + { argv: ["codex", "mcp", "remove", "whiteboard"], ifAny: true }, + ], + opencode: [ + { + argv: [ + "opencode", + "mcp", + "add", + "--global", + "whiteboard", + "--", + REVIEW_MCP_LAUNCH.command, + ...REVIEW_MCP_LAUNCH.args, + ], + }, + ], + pi: [{ argv: ["pi", "install", PI_WHITEBOARD_PACKAGE] }], +} as const satisfies Record; + +function runBlock(commands: readonly ConnectCommand[]): string { + const lines = commands.map( + ({ argv, ifAny }) => + `${argv.join(" ")}${ifAny ? " # old manual registration, if any" : ""}`, + ); + + return `Run:\n\n\`\`\`sh\n${lines.join("\n")}\n\`\`\``; +} + function pluginSteps( target: Exclude, platform: NodeJS.Platform, @@ -100,13 +158,9 @@ function pluginSteps( `Run:\n\n\`\`\`sh\nclaude plugin uninstall whiteboard@devfast # its launch cannot start on Windows, if installed\nclaude mcp remove -s user whiteboard # old registration, if any\n${CLAUDE_WINDOWS_MCP_ADD}\n\`\`\``, ]; - return [ - "Run:\n\n```sh\nclaude plugin marketplace add devdotfast/whiteboard\nclaude plugin install whiteboard@devfast --scope user\nclaude mcp remove -s user whiteboard # old manual registration, if any\n```", - ]; + return [runBlock(CONNECT_COMMANDS.claude)]; case "codex": - return [ - "Run:\n\n```sh\ncodex plugin marketplace add devdotfast/whiteboard\ncodex plugin add whiteboard@devfast\ncodex mcp remove whiteboard # old manual registration, if any\n```", - ]; + return [runBlock(CONNECT_COMMANDS.codex)]; case "opencode": { const launch = reviewMcpLaunch(true, platform); diff --git a/packages/review/src/pinned-download.ts b/packages/review/src/pinned-download.ts new file mode 100644 index 000000000..211aa3219 --- /dev/null +++ b/packages/review/src/pinned-download.ts @@ -0,0 +1,115 @@ +// Plain Node and zod only: apps/review-desktop/scripts imports this file directly. +import { createHash } from "node:crypto"; +import { createReadStream, createWriteStream } from "node:fs"; +import { rename, rm } from "node:fs/promises"; +import { Readable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import type { ReadableStream as WebReadableStream } from "node:stream/web"; + +import { z } from "zod"; + +const DOWNLOAD_TIMEOUT_MS = 120_000; + +interface DownloadOptions { + size?: number; + timeoutMs?: number; + signal?: AbortSignal; +} + +/** Streams `url` into `file` through a temporary file and returns its sha256. */ +export async function downloadFile( + url: string, + file: string, + { size, timeoutMs = DOWNLOAD_TIMEOUT_MS, signal }: DownloadOptions = {}, +) { + const { host } = new URL(url); + const partial = `${file}.${process.pid}.part`; + let response: Response; + + try { + response = await fetch(url, { + signal: AbortSignal.any([ + AbortSignal.timeout(timeoutMs), + ...(signal ? [signal] : []), + ]), + }); + } catch (error) { + throw new Error( + `Network error reaching ${host}: ${error instanceof Error ? networkCause(error) : String(error)}`, + ); + } + + if (!response.ok || !response.body) + throw new Error(`${host} answered ${response.status} for ${url}`); + + const hash = createHash("sha256"); + let received = 0; + + const tooLarge = new Error(`${url} is larger than its pinned ${size} bytes`); + + try { + await pipeline( + // SAFETY: Node's fetch body is its own web stream; the DOM type only + // names the same object. + Readable.fromWeb(response.body as WebReadableStream), + async function* (chunks: AsyncIterable) { + for await (const chunk of chunks) { + received += chunk.length; + + if (size !== undefined && received > size) throw tooLarge; + + hash.update(chunk); + yield chunk; + } + }, + createWriteStream(partial), + ); + await rename(partial, file); + } catch (error) { + await rm(partial, { force: true }); + + if (error === tooLarge) throw error; + + throw new Error( + `Network error downloading from ${host}: ${error instanceof Error ? networkCause(error) : String(error)}`, + ); + } + + return hash.digest("hex"); +} + +/** Leaves a matching `file` in place; otherwise downloads it and checks the pin. */ +export async function downloadPinned( + pin: { url: string; sha256: string; size?: number }, + file: string, + options: Omit = {}, +) { + if ((await sha256File(file).catch(() => undefined)) === pin.sha256) return; + + const actual = await downloadFile(pin.url, file, { + ...options, + size: pin.size, + }); + + if (actual !== pin.sha256) { + await rm(file, { force: true }); + throw new Error( + `Checksum mismatch for ${pin.url}: expected ${pin.sha256}, got ${actual}. The download was deleted.`, + ); + } +} + +export async function sha256File(file: string) { + const hash = createHash("sha256"); + + for await (const chunk of createReadStream(file)) hash.update(chunk); + + return hash.digest("hex"); +} + +const networkCauseSchema = z.object({ code: z.string() }); + +const networkCause = (error: Error) => + error.name === "TimeoutError" + ? "timed out" + : (networkCauseSchema.safeParse(error.cause).data?.code ?? error.message); diff --git a/packages/review/src/remote-attach.ts b/packages/review/src/remote-attach.ts new file mode 100644 index 000000000..aab05f3fd --- /dev/null +++ b/packages/review/src/remote-attach.ts @@ -0,0 +1,130 @@ +import { gt as greaterVersion, valid as validVersion } from "semver"; + +import { readReviewPackageVersion } from "./package-paths"; +import { + type EnsureRemoteLanguageServerInput, + ensureRemoteLanguageServer, +} from "./remote-language-server"; +import { missingToolchains } from "./remote-toolchains"; +import { readReviewServerHealth, serverNotReady } from "./server-discovery"; +import { + type EnsureBackgroundServerInput, + ensureBackgroundServer, + recordedBackgroundServer, + stopBackgroundServer, +} from "./server/background-server"; + +export async function remoteAttach(input: { + stateDir: string; + env: NodeJS.ProcessEnv; + packageRoot?: string; + cli?: EnsureBackgroundServerInput["cli"]; + groups?: string[]; + ensureExtensions?: EnsureRemoteLanguageServerInput["ensure"]; + installTimeoutMs?: number; + replace?: boolean; + version?: string; +}) { + const extensions = new AbortController(); + + const language = ensureRemoteLanguageServer({ + env: input.env, + packageRoot: input.packageRoot, + groups: input.groups, + signal: extensions.signal, + ensure: input.ensureExtensions, + installTimeoutMs: input.installTimeoutMs, + cli: input.cli, + }); + + const toolchains = missingToolchains(input.groups ?? [], input.env); + + let server: Awaited>; + let previousVersion: string | undefined; + + let incompatibleRunning: + | { version: string; pid: number; startedBy: "user" | "cli" | "desktop" } + | undefined; + + try { + const version = input.version ?? readReviewPackageVersion(import.meta.url); + + const other = input.replace + ? await otherVersionRunning(input.stateDir, version) + : undefined; + + if (other && (other.startedBy === "user" || newer(other.version, version))) + incompatibleRunning = other; + else if (other) { + await stopBackgroundServer({ serverPid: other.pid }); + previousVersion = other.version; + } + + server = await ensureBackgroundServer({ + stateDir: input.stateDir, + env: input.env, + startedBy: "desktop", + cli: input.cli, + }); + } catch (error) { + extensions.abort(); + await Promise.all([language, toolchains]); + throw error; + } + + const { discovery, started } = server; + + const { + languageServer, + languageServerDetail, + languageServerPending, + languageGroups, + } = await language; + + const missing = await toolchains; + + const health = await readReviewServerHealth(discovery); + + if (!health) throw serverNotReady(input.stateDir); + + return { + event: "remote.attach" as const, + version: health.version ?? null, + commit: health.commit ?? null, + serverId: health.serverId ?? null, + url: discovery.url, + token: discovery.token, + startedServer: started, + languageServer, + ...(languageServerDetail !== undefined && { languageServerDetail }), + ...(languageServerPending && { languageServerPending }), + languageGroups: languageGroups.map(({ group, installed, detail }) => { + const details = [detail, missing.get(group)].filter(Boolean).join("; "); + + return { group, installed, ...(details && { detail: details }) }; + }), + ...(previousVersion !== undefined && { replaced: true, previousVersion }), + ...(incompatibleRunning && { incompatibleRunning }), + }; +} + +const newer = (running: string, own: string) => + validVersion(running) !== null && + validVersion(own) !== null && + greaterVersion(running, own); + +async function otherVersionRunning(stateDir: string, version: string) { + const recorded = await recordedBackgroundServer(stateDir); + + if (!recorded) return undefined; + const { discovery, health } = recorded; + const running = health.version ?? "unknown"; + + return running === version + ? undefined + : { + version: running, + pid: discovery.serverPid, + startedBy: discovery.startedBy, + }; +} diff --git a/packages/review/src/remote-extensions.test.ts b/packages/review/src/remote-extensions.test.ts new file mode 100644 index 000000000..8daef5377 --- /dev/null +++ b/packages/review/src/remote-extensions.test.ts @@ -0,0 +1,557 @@ +import { createHash } from "node:crypto"; +import { existsSync } from "node:fs"; +import { + mkdir, + mkdtemp, + readFile, + readdir, + rm, + stat, + writeFile, +} from "node:fs/promises"; +import { type Server, createServer } from "node:http"; +import type { AddressInfo } from "node:net"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { crc32, deflateRawSync } from "node:zlib"; + +import { + type CuratedRemoteExtension, + ensureRemoteExtensions, + remoteExtensionTarget, + remoteServerPaths, +} from "@review/remote-extensions.js"; +import { afterEach, beforeEach, expect, it } from "vitest"; + +let root: string; + +let env: NodeJS.ProcessEnv; + +let server: Server; + +let base: string; + +let requests: string[]; + +const files = new Map(); + +function vsix(entries: Record) { + const locals: Buffer[] = []; + const centrals: Buffer[] = []; + let offset = 0; + + for (const [name, { data, mode = 0o644 }] of Object.entries(entries)) { + const raw = Buffer.from(data); + const packed = deflateRawSync(raw); + const fileName = Buffer.from(name); + const common = Buffer.alloc(26); + common.writeUInt16LE(20, 0); + common.writeUInt16LE(8, 4); + common.writeUInt32LE(crc32(raw), 10); + common.writeUInt32LE(packed.length, 14); + common.writeUInt32LE(raw.length, 18); + common.writeUInt16LE(fileName.length, 22); + + const local = Buffer.concat([ + Buffer.from([0x50, 0x4b, 3, 4]), + common, + fileName, + packed, + ]); + + const central = Buffer.alloc(46); + central.writeUInt32LE(0x02014b50, 0); + central.writeUInt16LE(0x0314, 4); + common.copy(central, 6); + central.writeUInt32LE(((0o100000 | mode) << 16) >>> 0, 38); + central.writeUInt32LE(offset, 42); + locals.push(local); + centrals.push(central, fileName); + offset += local.length; + } + + const directory = Buffer.concat(centrals); + const end = Buffer.alloc(22); + end.writeUInt32LE(0x06054b50, 0); + end.writeUInt16LE(locals.length, 8); + end.writeUInt16LE(locals.length, 10); + end.writeUInt32LE(directory.length, 12); + end.writeUInt32LE(offset, 16); + + return Buffer.concat([...locals, directory, end]); +} + +function serve(name: string, data: Buffer) { + files.set(`/${name}`, data); + + return { + universal: false, + url: `${base}/${name}`, + sha256: createHash("sha256").update(data).digest("hex"), + size: data.length, + }; +} + +function tyExtension( + overrides: Partial = {}, +): CuratedRemoteExtension { + const download = serve( + "ty.vsix", + vsix({ + "extension/package.json": { + data: JSON.stringify({ + publisher: "astral-sh", + name: "ty", + version: "1.0.0", + scripts: { build: "x" }, + dependencies: { a: "1" }, + extensionPack: ["ms-python.vscode-pylance"], + }), + }, + "extension/bundled/libs/bin/ty": { + data: "#!/bin/sh\necho ty 1.0.0\n", + mode: 0o755, + }, + }), + ); + + return { + id: "astral-sh.ty", + version: "1.0.0", + tier: "bundled", + group: "python", + executables: ["bundled/libs/bin/ty"], + stripExtensionPack: true, + targets: { "linux-x64": download, "linux-arm64": download }, + ...overrides, + }; +} + +function goExtension(): CuratedRemoteExtension { + const download = serve( + "go.vsix", + vsix({ + "extension/package.json": { + data: JSON.stringify({ + publisher: "golang", + name: "go", + version: "1.0.0", + }), + }, + }), + ); + + return { + id: "golang.go", + version: "1.0.0", + tier: "optional", + group: "go", + executables: [], + stripExtensionPack: false, + targets: { "linux-x64": download, "linux-arm64": download }, + }; +} + +beforeEach(async () => { + root = await mkdtemp(path.join(tmpdir(), "wb-remote-ext-")); + env = { ...process.env, DEV_REVIEW_HOME: root }; + requests = []; + files.clear(); + server = createServer((request, response) => { + requests.push(request.url ?? ""); + const data = files.get(request.url ?? ""); + response.writeHead(data ? 200 : 404).end(data); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + base = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; +}); + +afterEach(async () => { + await new Promise((resolve) => server.close(resolve)); + await rm(root, { recursive: true, force: true }); +}); + +it("installs the extension, applies the manifest's changes, lists it for the scanner and clears the scan cache", async () => { + const { extensionsDir, serverDataDir } = remoteServerPaths(env); + + const cache = path.join( + serverDataDir, + "data", + "CachedProfilesData", + "__default__profile__", + ); + + await mkdir(cache, { recursive: true }); + await writeFile(path.join(cache, "extensions.user.cache"), "{}"); + + const result = await ensureRemoteExtensions({ + env, + curated: [tyExtension()], + target: "linux-x64", + }); + + expect(result).toMatchObject({ + event: "remote.extensions", + installed: ["astral-sh.ty"], + skipped: [], + failed: [], + }); + + const directory = path.join(extensionsDir, "astral-sh.ty-1.0.0"); + + const manifest = JSON.parse( + await readFile(path.join(directory, "package.json"), "utf8"), + ); + + expect(manifest).not.toHaveProperty("scripts"); + expect(manifest).not.toHaveProperty("dependencies"); + expect(manifest).not.toHaveProperty("extensionPack"); + expect( + (await stat(path.join(directory, "bundled/libs/bin/ty"))).mode & 0o111, + ).not.toBe(0); + + expect( + JSON.parse( + await readFile(path.join(extensionsDir, "extensions.json"), "utf8"), + ), + ).toEqual([ + { + identifier: { id: "astral-sh.ty" }, + version: "1.0.0", + location: { $mid: 1, scheme: "file", path: directory }, + relativeLocation: "astral-sh.ty-1.0.0", + metadata: { + installedTimestamp: expect.any(Number), + targetPlatform: "linux-x64", + }, + }, + ]); + expect( + existsSync(path.join(serverDataDir, "data", "CachedProfilesData")), + ).toBe(false); + expect((await readdir(extensionsDir)).sort()).toEqual([ + "astral-sh.ty-1.0.0", + "extensions.json", + ]); +}); + +it("downloads nothing and leaves the list alone the second time", async () => { + const curated = [tyExtension()]; + await ensureRemoteExtensions({ env, curated, target: "linux-x64" }); + + const list = await readFile( + path.join(remoteServerPaths(env).extensionsDir, "extensions.json"), + "utf8", + ); + + requests = []; + + const result = await ensureRemoteExtensions({ + env, + curated, + target: "linux-x64", + }); + + expect(result).toMatchObject({ + installed: [], + skipped: [{ id: "astral-sh.ty", reason: "up to date" }], + failed: [], + }); + expect(requests).toEqual([]); + expect( + await readFile( + path.join(remoteServerPaths(env).extensionsDir, "extensions.json"), + "utf8", + ), + ).toBe(list); +}); + +it("removes the folders of other versions of a curated extension", async () => { + await ensureRemoteExtensions({ + env, + curated: [tyExtension()], + target: "linux-x64", + }); + + const { extensionsDir } = remoteServerPaths(env); + await mkdir(path.join(extensionsDir, "astral-sh.ty-0.9.0")); + await mkdir(path.join(extensionsDir, "someone.else-1.0.0")); + + const result = await ensureRemoteExtensions({ + env, + curated: [tyExtension({ version: "1.1.0" })], + target: "linux-x64", + }); + + expect(result).toMatchObject({ installed: ["astral-sh.ty"], failed: [] }); + expect((await readdir(extensionsDir)).sort()).toEqual([ + "astral-sh.ty-1.1.0", + "extensions.json", + "someone.else-1.0.0", + ]); +}); + +it("deletes a download that fails its checksum and installs nothing", async () => { + const extension = tyExtension(); + const wrong = { ...extension.targets["linux-x64"], sha256: "0".repeat(64) }; + + const result = await ensureRemoteExtensions({ + env, + curated: [ + { ...extension, targets: { "linux-x64": wrong, "linux-arm64": wrong } }, + ], + target: "linux-x64", + }); + + expect(result.installed).toEqual([]); + expect(result.failed).toEqual([ + { id: "astral-sh.ty", error: expect.stringMatching(/Checksum mismatch/) }, + ]); + const { extensionsDir } = remoteServerPaths(env); + expect(await readdir(extensionsDir)).toEqual(["extensions.json"]); + expect( + JSON.parse( + await readFile(path.join(extensionsDir, "extensions.json"), "utf8"), + ), + ).toEqual([]); +}); + +it("names the network when the download cannot reach it", async () => { + const extension = tyExtension(); + await new Promise((resolve) => server.close(resolve)); + + const result = await ensureRemoteExtensions({ + env, + curated: [extension], + target: "linux-x64", + }); + + expect(result.failed).toEqual([ + { + id: "astral-sh.ty", + error: expect.stringMatching( + /^Network error reaching 127\.0\.0\.1:\d+: ECONNREFUSED$/, + ), + }, + ]); + expect(await readdir(remoteServerPaths(env).extensionsDir)).toEqual([ + "extensions.json", + ]); +}); + +it("stops a stalled download when its signal aborts, and leaves nothing behind", async () => { + const extension = tyExtension(); + server.removeAllListeners("request"); + server.on("request", (_request, response) => + response.writeHead(200).write("x"), + ); + const stop = new AbortController(); + setTimeout(() => stop.abort(new Error("stopped for the test")), 200); + + const started = Date.now(); + + const result = await ensureRemoteExtensions({ + env, + curated: [extension], + target: "linux-x64", + signal: stop.signal, + }); + + expect(Date.now() - started).toBeLessThan(5_000); + expect(result.failed).toEqual([ + { + id: "astral-sh.ty", + error: expect.stringMatching(/stopped for the test$/), + }, + ]); + expect(await readdir(remoteServerPaths(env).extensionsDir)).toEqual([ + "extensions.json", + ]); + server.closeAllConnections(); +}); + +it("fails an extension whose executable does not run", async () => { + const extension = tyExtension({ executables: ["bundled/libs/bin/missing"] }); + + const result = await ensureRemoteExtensions({ + env, + curated: [extension], + target: "linux-x64", + }); + + expect(result.failed).toEqual([ + { + id: "astral-sh.ty", + error: expect.stringMatching( + /bundled\/libs\/bin\/missing does not run here/, + ), + }, + ]); + expect(await readdir(remoteServerPaths(env).extensionsDir)).toEqual([ + "extensions.json", + ]); +}); + +it("installs an optional extension only when its group is requested", async () => { + const curated = [tyExtension(), goExtension()]; + + const without = await ensureRemoteExtensions({ + env, + curated, + target: "linux-x64", + }); + + expect(without).toMatchObject({ + installed: ["astral-sh.ty"], + skipped: [{ id: "golang.go", reason: 'optional group "go" not requested' }], + failed: [], + }); + expect(requests).toEqual(["/ty.vsix"]); + + const withGo = await ensureRemoteExtensions({ + env, + curated, + groups: ["go"], + target: "linux-x64", + }); + + expect(withGo).toMatchObject({ installed: ["golang.go"], failed: [] }); + expect(requests).toEqual(["/ty.vsix", "/go.vsix"]); + + const list = JSON.parse( + await readFile( + path.join(remoteServerPaths(env).extensionsDir, "extensions.json"), + "utf8", + ), + ); + + expect( + list.map((entry: { identifier: { id: string } }) => entry.identifier.id), + ).toEqual(["astral-sh.ty", "golang.go"]); +}); + +function optionalExtension( + id: string, + group: string, + overrides: Partial = {}, +): CuratedRemoteExtension { + const [publisher, name] = id.split("."); + + const download = serve( + `${id}.vsix`, + vsix({ + "extension/package.json": { + data: JSON.stringify({ publisher, name, version: "1.0.0" }), + }, + }), + ); + + return { + id, + version: "1.0.0", + tier: "optional", + group, + executables: [], + stripExtensionPack: false, + targets: { "linux-x64": download, "linux-arm64": download }, + ...overrides, + }; +} + +it("reports each requested group, and installs nothing of a group the Desktop has not turned on", async () => { + const lldb = optionalExtension("llvm-vs-code-extensions.lldb-dap", "swift"); + + const curated = [ + optionalExtension("rust-lang.rust-analyzer", "rust"), + optionalExtension("swiftlang.swift-vscode", "swift"), + lldb, + optionalExtension("muhammad-sammy.csharp", "csharp", { + targets: { + "linux-x64": { ...lldb.targets["linux-x64"], sha256: "0".repeat(64) }, + "linux-arm64": lldb.targets["linux-arm64"], + }, + }), + ]; + + const result = await ensureRemoteExtensions({ + env, + curated, + groups: ["swift", "csharp"], + target: "linux-x64", + }); + + expect(result.installed).toEqual([ + "swiftlang.swift-vscode", + "llvm-vs-code-extensions.lldb-dap", + ]); + expect(result.groups).toEqual([ + { group: "swift", installed: true }, + { + group: "csharp", + installed: false, + detail: expect.stringMatching( + /^muhammad-sammy\.csharp: Checksum mismatch/, + ), + }, + ]); + expect(requests).not.toContain("/rust-lang.rust-analyzer.vsix"); + expect((await readdir(remoteServerPaths(env).extensionsDir)).sort()).toEqual([ + "extensions.json", + "llvm-vs-code-extensions.lldb-dap-1.0.0", + "swiftlang.swift-vscode-1.0.0", + ]); +}); + +it("refuses a VSIX with a path that leaves its folder, or a symlink, and writes nothing", async () => { + const manifest = { + data: JSON.stringify({ + publisher: "astral-sh", + name: "ty", + version: "1.0.0", + }), + }; + + for (const [name, entry] of [ + ["escape.vsix", { "extension/../../escaped.txt": { data: "x" } }], + [ + "symlink.vsix", + { "extension/link": { data: "/etc/passwd", mode: 0o120777 } }, + ], + ] as const) { + const download = serve( + name, + vsix({ "extension/package.json": manifest, ...entry }), + ); + + const result = await ensureRemoteExtensions({ + env, + curated: [ + tyExtension({ + executables: [], + targets: { "linux-x64": download, "linux-arm64": download }, + }), + ], + target: "linux-x64", + }); + + expect(result.failed).toEqual([ + { + id: "astral-sh.ty", + error: expect.stringMatching( + /invalid relative path|VSIX contains a symlink/, + ), + }, + ]); + } + + const { extensionsDir } = remoteServerPaths(env); + expect(await readdir(extensionsDir)).toEqual(["extensions.json"]); + expect(await readdir(path.dirname(extensionsDir))).toEqual(["extensions"]); +}); + +it("refuses a machine that is not Linux on x64 or arm64", () => { + expect(remoteExtensionTarget("linux", "arm64")).toBe("linux-arm64"); + expect(() => remoteExtensionTarget("darwin", "arm64")).toThrow( + "Language features on a remote need Linux on x64 or arm64; this machine is darwin-arm64.", + ); +}); diff --git a/packages/review/src/remote-extensions.ts b/packages/review/src/remote-extensions.ts new file mode 100644 index 000000000..82ac8d25a --- /dev/null +++ b/packages/review/src/remote-extensions.ts @@ -0,0 +1,359 @@ +import { execFile } from "node:child_process"; +import { + mkdir, + readFile, + readdir, + rename, + rm, + writeFile, +} from "node:fs/promises"; +import path from "node:path"; +import { promisify } from "node:util"; + +import { parseJsonText } from "@dev.fast/json"; +import { z } from "zod"; + +import { findReviewPackageRoot } from "./package-paths"; +import { downloadPinned } from "./pinned-download"; +import { devReviewHome } from "./review-home-paths"; +import { extractVsix, sanitizeVsixManifest } from "./vsix"; + +const EXECUTABLE_TIMEOUT_MS = 15_000; + +const STAMP_FILE = ".curated.json"; + +type RemoteTarget = "linux-x64" | "linux-arm64"; + +const curatedDownloadSchema = z.object({ + universal: z.boolean(), + url: z.string(), + sha256: z.string(), + size: z.number(), +}); + +type CuratedDownload = z.infer; + +const curatedExtensionSchema = z.object({ + id: z.string(), + version: z.string(), + tier: z.enum(["bundled", "optional"]), + group: z.string(), + executables: z.array(z.string()), + stripExtensionPack: z.boolean(), + targets: z.object({ + "linux-x64": curatedDownloadSchema, + "linux-arm64": curatedDownloadSchema, + }), +}); + +export type CuratedRemoteExtension = z.infer; + +const stampSchema = z.object({ + id: z.string(), + version: z.string(), + target: z.string(), + sha256: z.string(), + installedTimestamp: z.number(), +}); + +type Stamp = z.infer; + +export function whiteboardRemoteHome(env: NodeJS.ProcessEnv = process.env) { + return path.join(devReviewHome(env), "whiteboard-remote"); +} + +export function remoteServerPaths(env: NodeJS.ProcessEnv = process.env) { + const home = whiteboardRemoteHome(env); + + return { + extensionsDir: path.join(home, "extensions"), + serverDataDir: path.join(home, "server"), + }; +} + +export function remoteExtensionTarget( + platform: string = process.platform, + arch: string = process.arch, +): RemoteTarget { + if (platform === "linux" && (arch === "x64" || arch === "arm64")) + return `linux-${arch}`; + + throw new Error( + `Language features on a remote need Linux on x64 or arm64; this machine is ${platform}-${arch}.`, + ); +} + +interface EnsureRemoteExtensionsInput { + env?: NodeJS.ProcessEnv; + curated?: CuratedRemoteExtension[]; + groups?: string[]; + target?: RemoteTarget; + timeoutMs?: number; + signal?: AbortSignal; +} + +export async function ensureRemoteExtensions( + input: EnsureRemoteExtensionsInput = {}, +) { + const target = input.target ?? remoteExtensionTarget(); + const curated = input.curated ?? (await readPackagedCurated()); + const { extensionsDir, serverDataDir } = remoteServerPaths(input.env); + const installed: string[] = []; + const skipped: { id: string; reason: string }[] = []; + const failed: { id: string; error: string }[] = []; + const listed: StoredExtension[] = []; + + await mkdir(extensionsDir, { recursive: true }); + + for (const extension of curated) { + if ( + extension.tier === "optional" && + !input.groups?.includes(extension.group) + ) { + skipped.push({ + id: extension.id, + reason: `optional group "${extension.group}" not requested`, + }); + continue; + } + + const download = extension.targets[target]; + const folder = `${extension.id}-${extension.version}`; + const directory = path.join(extensionsDir, folder); + + const stamp = { + id: extension.id, + version: extension.version, + target, + sha256: download.sha256, + }; + + try { + const current = await readStamp(directory); + + if ( + current && + sameStamp(current, stamp) && + (await checkExecutables(directory, extension).then( + () => true, + () => false, + )) + ) { + skipped.push({ id: extension.id, reason: "up to date" }); + listed.push( + storedExtension( + extension, + directory, + folder, + target, + current.installedTimestamp, + ), + ); + continue; + } + + const installedTimestamp = Date.now(); + await install( + extension, + download, + directory, + { ...stamp, installedTimestamp }, + input.timeoutMs, + input.signal, + ); + installed.push(extension.id); + listed.push( + storedExtension( + extension, + directory, + folder, + target, + installedTimestamp, + ), + ); + } catch (error) { + failed.push({ + id: extension.id, + error: error instanceof Error ? error.message : String(error), + }); + } + } + + // The scanner caches its last scan; a changed list is read only once the cache is gone. + const list = `${JSON.stringify(listed)}\n`; + const listFile = path.join(extensionsDir, "extensions.json"); + + if ((await readFile(listFile, "utf8").catch(() => undefined)) !== list) { + const temporary = `${listFile}.${process.pid}.tmp`; + + await writeFile(temporary, list); + await rename(temporary, listFile); + await rm(path.join(serverDataDir, "data", "CachedProfilesData"), { + recursive: true, + force: true, + }); + } + + const kept = new Set(listed.map((entry) => entry.relativeLocation)); + + for (const folder of await readdir(extensionsDir)) + if ( + !kept.has(folder) && + curated.some( + ({ id }) => + folder.startsWith(`${id}-`) && + /^\d/.test(folder.slice(id.length + 1)), + ) + ) + await rm(path.join(extensionsDir, folder), { + recursive: true, + force: true, + }); + + return { + event: "remote.extensions" as const, + target, + installed, + skipped, + failed, + groups: requestedGroups(curated, input.groups ?? [], failed), + }; +} + +function requestedGroups( + curated: CuratedRemoteExtension[], + groups: string[], + failed: { id: string; error: string }[], +) { + return groups.flatMap((group) => { + const members = curated.filter( + (extension) => extension.tier === "optional" && extension.group === group, + ); + + if (members.length === 0) return []; + + const errors = failed.filter(({ id }) => + members.some((extension) => extension.id === id), + ); + + return [ + errors.length === 0 + ? { group, installed: true } + : { + group, + installed: false, + detail: errors.map(({ id, error }) => `${id}: ${error}`).join("; "), + }, + ]; + }); +} + +async function readPackagedCurated(): Promise { + const file = path.join( + findReviewPackageRoot(import.meta.url), + "vscode-server", + "curated.json", + ); + + const text = await readFile(file, "utf8").catch(() => { + throw new Error(`This package has no VS Code server (${file} is missing).`); + }); + + return z + .object({ extensions: z.array(curatedExtensionSchema) }) + .parse(JSON.parse(text)).extensions; +} + +async function readStamp(directory: string): Promise { + return readFile(path.join(directory, STAMP_FILE), "utf8") + .then((text) => stampSchema.safeParse(parseJsonText(text)).data) + .catch(() => undefined); +} + +function sameStamp( + current: Stamp, + expected: Omit, +) { + return ( + current.id === expected.id && + current.version === expected.version && + current.target === expected.target && + current.sha256 === expected.sha256 + ); +} + +async function install( + extension: CuratedRemoteExtension, + download: CuratedDownload, + directory: string, + stamp: Stamp, + timeoutMs?: number, + signal?: AbortSignal, +) { + const part = `${directory}.${process.pid}.vsix`; + const staging = `${directory}.${process.pid}.staging`; + + try { + await downloadPinned(download, part, { timeoutMs, signal }); + await rm(staging, { recursive: true, force: true }); + await extractVsix(part, staging, download.size * 8); + await sanitizeVsixManifest(staging, extension); + + await checkExecutables(staging, extension); + + await writeFile( + path.join(staging, STAMP_FILE), + `${JSON.stringify(stamp)}\n`, + ); + await rm(directory, { recursive: true, force: true }); + await rename(staging, directory); + } finally { + await rm(part, { force: true }); + await rm(staging, { recursive: true, force: true }); + } +} + +async function checkExecutables( + directory: string, + extension: CuratedRemoteExtension, +) { + for (const relative of extension.executables) { + await promisify(execFile)(path.join(directory, relative), ["--version"], { + timeout: EXECUTABLE_TIMEOUT_MS, + killSignal: "SIGKILL", + }).catch((error: Error) => { + throw new Error( + `${extension.id}: ${relative} does not run here: ${error.message}`, + ); + }); + } +} + +interface StoredExtension { + identifier: { id: string }; + version: string; + location: { $mid: 1; scheme: "file"; path: string }; + relativeLocation: string; + metadata: { installedTimestamp: number; targetPlatform: string }; +} + +function storedExtension( + extension: CuratedRemoteExtension, + directory: string, + folder: string, + target: RemoteTarget, + installedTimestamp: number, +): StoredExtension { + return { + identifier: { id: extension.id }, + version: extension.version, + location: { $mid: 1, scheme: "file", path: directory }, + relativeLocation: folder, + metadata: { + installedTimestamp, + targetPlatform: extension.targets[target].universal + ? "universal" + : target, + }, + }; +} diff --git a/packages/review/src/remote-language-server.test.ts b/packages/review/src/remote-language-server.test.ts new file mode 100644 index 000000000..2e21df8a9 --- /dev/null +++ b/packages/review/src/remote-language-server.test.ts @@ -0,0 +1,345 @@ +import { spawnSync } from "node:child_process"; +import { + mkdir, + mkdtemp, + readFile, + realpath, + rm, + stat, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { withFileLock } from "@dev.fast/trace-core"; +import { afterEach, beforeEach, expect, it } from "vitest"; + +import { remoteServerPaths } from "./remote-extensions.js"; +import { ensureRemoteLanguageServer } from "./remote-language-server.js"; +import { + isolatedEnv, + stopServersUnder, +} from "./server/background-server-test-utils.js"; + +const COMMIT = "0123456789abcdef0123456789abcdef01234567"; + +let root: string; + +let packageRoot: string; + +let env: NodeJS.ProcessEnv; + +const noExtensions = async () => ({ failed: [] }); + +beforeEach(async () => { + root = await realpath(await mkdtemp(path.join(tmpdir(), "wb-ls-"))); + packageRoot = path.join(root, "package"); + env = isolatedEnv(root); + await standInServer(COMMIT); +}); + +afterEach(async () => { + await stopServersUnder(root); + await rm(root, { recursive: true, force: true }); +}); + +it("starts the server on loopback with a private token, and a second call reports the same port", async () => { + const first = await ensureRemoteLanguageServer({ + env, + packageRoot, + ensure: noExtensions, + }); + + expect(first).toEqual({ + languageServer: { + port: expect.any(Number), + connectionToken: expect.stringMatching(/^[0-9a-f]{64}$/), + commit: COMMIT, + }, + languageGroups: [], + }); + const { port, connectionToken } = first.languageServer!; + expect(await text(port, "/version")).toBe(COMMIT); + expect(await text(port, "/token")).toBe(connectionToken); + + const { tokenFile, logFile } = serverFiles(); + expect((await stat(tokenFile)).mode & 0o777).toBe(0o600); + expect(await readFile(logFile, "utf8")).not.toContain(connectionToken); + expect( + spawnSync("ps", ["-eo", "args"], { encoding: "utf8" }).stdout, + ).not.toContain(connectionToken); + + const pid = await text(port, "/pid"); + + const second = await ensureRemoteLanguageServer({ + env, + packageRoot, + ensure: noExtensions, + }); + + expect(second).toEqual(first); + expect(await text(port, "/pid")).toBe(pid); +}, 30_000); + +it("starts a new server when the running one stopped answering", async () => { + const first = await ensureRemoteLanguageServer({ + env, + packageRoot, + ensure: noExtensions, + }); + + const pid = Number(await text(first.languageServer!.port, "/pid")); + process.kill(pid, "SIGKILL"); + + const second = await ensureRemoteLanguageServer({ + env, + packageRoot, + ensure: noExtensions, + }); + + expect(second.languageServer?.port).not.toBe(first.languageServer?.port); + expect(second.languageServer?.connectionToken).not.toBe( + first.languageServer?.connectionToken, + ); + expect(await text(second.languageServer!.port, "/version")).toBe(COMMIT); +}, 30_000); + +it("replaces a running server of another commit once the package is reinstalled", async () => { + const first = await ensureRemoteLanguageServer({ + env, + packageRoot, + ensure: noExtensions, + }); + + const other = "f".repeat(40); + await standInServer(other); + + const second = await ensureRemoteLanguageServer({ + env, + packageRoot, + ensure: noExtensions, + }); + + expect(second.languageServer?.commit).toBe(other); + expect(await text(second.languageServer!.port, "/version")).toBe(other); + await expect(text(first.languageServer!.port, "/version")).rejects.toThrow( + "fetch failed", + ); +}, 30_000); + +it("runs extensions ensure first, with the groups, and starts nothing when it fails", async () => { + const calls: unknown[] = []; + + const result = await ensureRemoteLanguageServer({ + env, + packageRoot, + groups: ["go"], + ensure: async (input) => { + calls.push(input.groups); + + return { + failed: [ + { + id: "astral-sh.ty", + error: "Network error reaching open-vsx.org: ENETUNREACH", + }, + ], + }; + }, + }); + + expect(calls).toEqual([["go"]]); + expect(result).toEqual({ + languageServer: null, + languageServerDetail: + "Could not install the language extensions: astral-sh.ty: Network error reaching open-vsx.org: ENETUNREACH", + languageGroups: [{ group: "go", installed: false }], + }); + expect(await stat(serverFiles().runningFile).catch(() => null)).toBeNull(); +}); + +it("hands downloads that outlast the attach to one detached install, and reports pending until it is done", async () => { + const cli = path.join(root, "cli.mjs"); + const runs = path.join(root, "runs"); + await writeFile( + cli, + `import { appendFileSync } from "node:fs";\nappendFileSync(${JSON.stringify(runs)}, process.argv.slice(2).join(" ") + "\\n");\nsetTimeout(() => {}, 30_000);\n`, + ); + let ensured = 0; + + const stalled = async ({ signal }: { signal?: AbortSignal }) => { + ensured++; + await new Promise((resolve) => signal?.addEventListener("abort", resolve)); + + const error = + "Network error reaching open-vsx.org: This operation was aborted"; + + return { + failed: [{ id: "golang.go", error }], + groups: [ + { group: "go", installed: false, detail: `golang.go: ${error}` }, + ], + }; + }; + + const attach = () => + ensureRemoteLanguageServer({ + env, + packageRoot, + groups: ["go"], + ensure: stalled, + installTimeoutMs: 200, + cli: [process.execPath, cli], + }); + + const pending = { + languageServer: null, + languageServerDetail: + "Installing the language extensions on this host; they will be available on the next connection.", + languageServerPending: true, + languageGroups: [{ group: "go", installed: false }], + }; + + expect(await attach()).toEqual(pending); + await expect + .poll(() => readFile(runs, "utf8").catch(() => "")) + .toBe("remote extensions ensure --json --groups go\n"); + + expect(await attach()).toEqual(pending); + expect(ensured).toBe(1); + expect(await readFile(runs, "utf8")).toBe( + "remote extensions ensure --json --groups go\n", + ); + + const { installLog } = serverFiles(); + expect((await stat(installLog)).mode & 0o777).toBe(0o600); + + await stopServersUnder(root); + + const done = await ensureRemoteLanguageServer({ + env, + packageRoot, + ensure: noExtensions, + }); + + expect(done.languageServer?.commit).toBe(COMMIT); +}, 30_000); + +it("reports the download failure, not pending, when the detached install's lock cannot be had", async () => { + const error = + "Network error reaching open-vsx.org: This operation was aborted"; + + const stalled = async ({ signal }: { signal?: AbortSignal }) => { + await new Promise((resolve) => signal?.addEventListener("abort", resolve)); + + return { + failed: [{ id: "golang.go", error }], + groups: [ + { group: "go", installed: false, detail: `golang.go: ${error}` }, + ], + }; + }; + + const held = await withFileLock( + serverFiles().installLock, + { retryMs: 100, staleMs: 60_000, unownedGraceMs: 5_000, timeoutMs: 1_000 }, + () => + ensureRemoteLanguageServer({ + env, + packageRoot, + groups: ["go"], + ensure: stalled, + installTimeoutMs: 200, + cli: [process.execPath, "-e", "process.exit(9)"], + }), + ); + + expect(held).toEqual({ + acquired: true, + result: { + languageServer: null, + languageServerDetail: `Could not install the language extensions: golang.go: ${error}`, + languageGroups: [ + { group: "go", installed: false, detail: `golang.go: ${error}` }, + ], + }, + }); +}, 30_000); + +it("reports a package without a VS Code server", async () => { + const result = await ensureRemoteLanguageServer({ + env, + packageRoot: path.join(root, "empty"), + ensure: noExtensions, + }); + + expect(result.languageServer).toBeNull(); + expect(result.languageServerDetail).toMatch(/has no VS Code server/); +}); + +it("reports a server that exits at start with the end of its log", async () => { + await writeFile( + path.join(packageRoot, "vscode-server", "out", "server-main.js"), + 'console.error("cannot start"); process.exit(3);\n', + ); + + const result = await ensureRemoteLanguageServer({ + env, + packageRoot, + ensure: noExtensions, + }); + + expect(result.languageServer).toBeNull(); + expect(result.languageServerDetail).toMatch( + /did not start\. The end of .*server\.log:\ncannot start/, + ); +}); + +function serverFiles() { + const file = (name: string) => + path.join(remoteServerPaths(env).serverDataDir, name); + + return { + tokenFile: file("connection-token"), + logFile: file("server.log"), + runningFile: file("server.json"), + installLog: file("install.log"), + installLock: file("install.lock"), + }; +} + +async function standInServer(commit: string) { + const server = path.join(packageRoot, "vscode-server"); + await mkdir(path.join(server, "out"), { recursive: true }); + await writeFile( + path.join(server, "product.json"), + JSON.stringify({ commit }), + ); + await writeFile( + path.join(server, "out", "server-main.js"), + [ + 'import { readFileSync } from "node:fs";', + 'import http from "node:http";', + "const arg = (name) => process.argv[process.argv.indexOf(name) + 1];", + "const expected = new Map([", + ` ["--server-data-dir", ${JSON.stringify(remoteServerPaths(env).serverDataDir)}],`, + ` ["--extensions-dir", ${JSON.stringify(remoteServerPaths(env).extensionsDir)}],`, + ' ["--host", "127.0.0.1"],', + "]);", + "for (const [name, value] of expected) if (arg(name) !== value) process.exit(2);", + 'const token = readFileSync(arg("--connection-token-file"), "utf8");', + `const commit = ${JSON.stringify(commit)};`, + "const server = http.createServer((request, response) => response.end(", + ' request.url === "/version" ? commit : request.url === "/token" ? token : String(process.pid)));', + 'server.listen(Number(arg("--port")), arg("--host"), () =>', + " console.log(`Extension host agent listening on ${server.address().port}`));", + "", + ].join("\n"), + ); +} + +async function text(port: number, route: string) { + const response = await fetch(`http://127.0.0.1:${port}${route}`); + + return response.text(); +} diff --git a/packages/review/src/remote-language-server.ts b/packages/review/src/remote-language-server.ts new file mode 100644 index 000000000..5122c657e --- /dev/null +++ b/packages/review/src/remote-language-server.ts @@ -0,0 +1,436 @@ +import { randomBytes } from "node:crypto"; +import { + chmod, + mkdir, + readFile, + rename, + stat, + writeFile, +} from "node:fs/promises"; +import path from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; + +import { processStartIdentity, withFileLock } from "@dev.fast/trace-core"; +import { z } from "zod"; + +import { findReviewPackageRoot } from "./package-paths"; +import { ensureRemoteExtensions, remoteServerPaths } from "./remote-extensions"; +import { cliSpawn, spawnDetached } from "./server/background-server"; + +const START_TIMEOUT_MS = 15_000; + +const VERSION_TIMEOUT_MS = 3_000; + +const INSTALL_TIMEOUT_MS = 35_000; + +const SHUTDOWN_WITHOUT_DELAY_ENV = + "DEV_FAST_REVIEW_REMOTE_SHUTDOWN_WITHOUT_DELAY"; + +const GRACE_ENV = "DEV_FAST_REVIEW_REMOTE_RECONNECTION_GRACE_SECONDS"; + +const PENDING = { + languageServer: null, + languageServerDetail: + "Installing the language extensions on this host; they will be available on the next connection.", + languageServerPending: true, +} as const; + +function idleArgs(env: NodeJS.ProcessEnv) { + const grace = env[GRACE_ENV] ?? ""; + + return [ + "--enable-remote-auto-shutdown", + "--reconnection-grace-time", + /^[1-9]\d{0,3}$/.test(grace) ? grace : "600", + ]; +} + +const LISTENING = /Extension host agent listening on (\d+)/; + +const LOCK = { + retryMs: 100, + staleMs: 60_000, + unownedGraceMs: 5_000, + identifyOwner: true, +}; + +const installingSchema = z.object({ + pid: z.number(), + started: z.string().nullable(), +}); + +const runningSchema = z.object({ + pid: z.number(), + started: z.string().nullable(), + port: z.number(), +}); + +type Running = z.infer; + +export interface RemoteLanguageGroup { + group: string; + installed: boolean; + detail?: string; +} + +export interface RemoteLanguageServer { + port: number; + connectionToken: string; + commit: string; +} + +interface EnsureExtensions { + (input: { + env: NodeJS.ProcessEnv; + groups?: string[]; + signal?: AbortSignal; + }): Promise<{ + failed: { id: string; error: string }[]; + groups?: RemoteLanguageGroup[]; + }>; +} + +export interface EnsureRemoteLanguageServerInput { + env: NodeJS.ProcessEnv; + packageRoot?: string; + groups?: string[]; + signal?: AbortSignal; + ensure?: EnsureExtensions; + timeoutMs?: number; + installTimeoutMs?: number; + cli?: readonly string[]; +} + +function remoteLanguageServerFiles(env: NodeJS.ProcessEnv) { + const { serverDataDir } = remoteServerPaths(env); + + return { + serverDataDir, + tokenFile: path.join(serverDataDir, "connection-token"), + runningFile: path.join(serverDataDir, "server.json"), + logFile: path.join(serverDataDir, "server.log"), + lock: path.join(serverDataDir, "start.lock"), + installingFile: path.join(serverDataDir, "install.json"), + installLog: path.join(serverDataDir, "install.log"), + installLock: path.join(serverDataDir, "install.lock"), + }; +} + +export async function ensureRemoteLanguageServer( + input: EnsureRemoteLanguageServerInput, +): Promise<{ + languageServer: RemoteLanguageServer | null; + languageServerDetail?: string; + languageServerPending?: true; + languageGroups: RemoteLanguageGroup[]; +}> { + const root = path.join( + input.packageRoot ?? findReviewPackageRoot(import.meta.url), + "vscode-server", + ); + + const files = remoteLanguageServerFiles(input.env); + const capped = new AbortController(); + + let languageGroups: RemoteLanguageGroup[] = (input.groups ?? []).map( + (group) => ({ group, installed: false }), + ); + + const cap = setTimeout( + () => capped.abort(), + input.installTimeoutMs ?? INSTALL_TIMEOUT_MS, + ); + + try { + const commit = await readCommit(root); + const ensure = input.ensure ?? ensureRemoteExtensions; + await mkdir(files.serverDataDir, { recursive: true, mode: 0o700 }); + + if (await installing(files)) return { ...PENDING, languageGroups }; + + const { failed, groups } = await ensure({ + env: input.env, + groups: input.groups, + signal: AbortSignal.any([ + capped.signal, + ...(input.signal ? [input.signal] : []), + ]), + }); + + if ( + failed.length > 0 && + capped.signal.aborted && + !input.signal?.aborted && + (await installDetached(files, input)) + ) + return { ...PENDING, languageGroups }; + + if (groups) languageGroups = groups; + + if (failed.length > 0) + throw new Error( + `Could not install the language extensions: ${failed.map(({ id, error }) => `${id}: ${error}`).join("; ")}`, + ); + + const outcome = await withFileLock( + files.lock, + { ...LOCK, timeoutMs: (input.timeoutMs ?? START_TIMEOUT_MS) * 2 }, + async () => { + const running = await healthy(files.runningFile, commit); + + const port = + running?.port ?? + (await startServer(root, files, input.env, input.timeoutMs)); + + return { + port, + connectionToken: (await readFile(files.tokenFile, "utf8")).trim(), + commit, + }; + }, + ); + + if (!outcome.acquired) + throw new Error("Another start of the VS Code server did not finish."); + + return { languageServer: outcome.result, languageGroups }; + } catch (error) { + return { + languageServer: null, + languageServerDetail: + error instanceof Error ? error.message : String(error), + languageGroups, + }; + } finally { + clearTimeout(cap); + } +} + +async function readInstalling( + files: ReturnType, +) { + return readFile(files.installingFile, "utf8") + .then((text) => installingSchema.safeParse(JSON.parse(text)).data) + .catch(() => undefined); +} + +async function installing(files: ReturnType) { + const running = await readInstalling(files); + + return running !== undefined && ours(running); +} + +export async function remoteLanguageServerGroups(env: NodeJS.ProcessEnv) { + const files = remoteLanguageServerFiles(env); + + const recorded = await Promise.all([ + readRunning(files.runningFile), + readInstalling(files), + ]); + + return recorded.flatMap((entry) => (entry && ours(entry) ? [entry.pid] : [])); +} + +export async function stopProcessGroup(leader: number) { + for (const signal of ["SIGTERM", "SIGKILL"] as const) { + try { + process.kill(-leader, signal); + } catch { + return; + } + + for (let i = 0; i < 50; i++) { + if (!groupAlive(leader)) return; + await delay(100); + } + } +} + +function groupAlive(leader: number) { + try { + process.kill(-leader, 0); + + return true; + } catch (error) { + return error instanceof Error && "code" in error && error.code === "EPERM"; + } +} + +async function installDetached( + files: ReturnType, + input: EnsureRemoteLanguageServerInput, +) { + const outcome = await withFileLock( + files.installLock, + { ...LOCK, timeoutMs: 10_000 }, + async () => { + if (await installing(files)) return; + const { command, args, env } = cliSpawn(input.cli, input.env); + + const child = await spawnDetached({ + command, + args: [ + ...args, + "remote", + "extensions", + "ensure", + "--json", + ...(input.groups?.length ? ["--groups", input.groups.join(",")] : []), + ], + cwd: files.serverDataDir, + env, + log: files.installLog, + }); + + if (child.pid !== undefined) + await writeFile( + files.installingFile, + `${JSON.stringify({ pid: child.pid, started: processStartIdentity(child.pid) })}\n`, + { mode: 0o600 }, + ); + }, + ); + + return outcome.acquired; +} + +async function readCommit(root: string) { + const file = path.join(root, "product.json"); + + const text = await readFile(file, "utf8").catch(() => { + throw new Error( + `This Whiteboard install has no VS Code server (${file} is missing).`, + ); + }); + + const { commit } = z + .object({ commit: z.string().regex(/^[0-9a-f]{40}$/) }) + .parse(JSON.parse(text)); + + return commit; +} + +async function readRunning(file: string) { + return readFile(file, "utf8") + .then((text) => runningSchema.safeParse(JSON.parse(text)).data) + .catch(() => undefined); +} + +const ours = (running: { pid: number; started: string | null }) => + running.started !== null && + processStartIdentity(running.pid) === running.started; + +async function healthy(file: string, commit: string) { + const running = await readRunning(file); + + if (!running) return undefined; + + if (ours(running) && (await version(running.port)) === commit) return running; + + await stop(running); + + return undefined; +} + +async function version(port: number) { + return fetch(`http://127.0.0.1:${port}/version`, { + signal: AbortSignal.timeout(VERSION_TIMEOUT_MS), + }) + .then((response) => (response.ok ? response.text() : undefined)) + .catch(() => undefined); +} + +async function stop(running: Running) { + if (!ours(running)) return; + + try { + process.kill(running.pid, "SIGTERM"); + } catch { + return; + } + + for (let i = 0; i < 50 && processStartIdentity(running.pid) !== null; i++) + await delay(100); +} + +async function startServer( + root: string, + files: ReturnType, + env: NodeJS.ProcessEnv, + timeoutMs = START_TIMEOUT_MS, +) { + const { extensionsDir, serverDataDir } = remoteServerPaths(env); + + const temporary = `${files.tokenFile}.${process.pid}.tmp`; + await writeFile(temporary, randomBytes(32).toString("hex"), { mode: 0o600 }); + await chmod(temporary, 0o600); + await rename(temporary, files.tokenFile); + + const logStart = (await stat(files.logFile).catch(() => null))?.size ?? 0; + + const child = await spawnDetached({ + command: process.execPath, + args: [ + path.join(root, "out", "server-main.js"), + "--host", + "127.0.0.1", + "--port", + "0", + "--connection-token-file", + files.tokenFile, + "--server-data-dir", + serverDataDir, + "--extensions-dir", + extensionsDir, + "--accept-server-license-terms", + ...idleArgs(env), + ...(env[SHUTDOWN_WITHOUT_DELAY_ENV] === "1" + ? ["--remote-auto-shutdown-without-delay"] + : []), + ], + cwd: serverDataDir, + env, + log: files.logFile, + }); + + const deadline = Date.now() + timeoutMs; + + while (Date.now() < deadline && !child.exited) { + const output = (await readFile(files.logFile)) + .subarray(logStart) + .toString("utf8"); + + const port = Number(LISTENING.exec(output)?.[1]); + + if (port && child.pid !== undefined) { + const running: Running = { + pid: child.pid, + started: processStartIdentity(child.pid), + port, + }; + + await writeFile(files.runningFile, `${JSON.stringify(running)}\n`, { + mode: 0o600, + }); + + return port; + } + + await delay(100); + } + + if (!child.exited && child.pid !== undefined) + process.kill(child.pid, "SIGTERM"); + + const tail = (await readFile(files.logFile)) + .subarray(logStart) + .toString("utf8") + .trimEnd() + .split("\n") + .slice(-10) + .join("\n"); + + throw new Error( + `The VS Code server did not start${child.error ? `: ${child.error.message}` : child.exited ? "" : ` within ${timeoutMs / 1_000} s`}. The end of ${files.logFile}:\n${tail}`, + ); +} diff --git a/packages/review/src/remote-toolchains.test.ts b/packages/review/src/remote-toolchains.test.ts new file mode 100644 index 000000000..40abdf11e --- /dev/null +++ b/packages/review/src/remote-toolchains.test.ts @@ -0,0 +1,73 @@ +import { chmod, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { missingToolchains } from "@review/remote-toolchains.js"; +import { afterEach, beforeEach, expect, it } from "vitest"; + +let home: string; + +let env: NodeJS.ProcessEnv; + +const toolchains = { + one: [["wbtest-one", "--version"]], + two: [ + ["wbtest-build", "--version"], + ["wbtest-compile", "--version"], + ], + slow: [["wbtest-slow", "--version"]], +}; + +async function tool(name: string, body: string) { + const file = path.join(home, "toolchain", name); + await writeFile(file, `#!/bin/sh\n${body}\n`); + await chmod(file, 0o755); +} + +beforeEach(async () => { + home = await mkdtemp(path.join(tmpdir(), "wb-toolchains-")); + await mkdir(path.join(home, "toolchain")); + await writeFile( + path.join(home, ".profile"), + `PATH="${home}/toolchain:$PATH"\nexport PATH\n`, + ); + env = { HOME: home, SHELL: "/bin/bash", PATH: "/usr/bin:/bin" }; +}); + +afterEach(async () => { + await rm(home, { recursive: true, force: true }); +}); + +it("finds a toolchain that only the login shell puts on PATH", async () => { + await tool("wbtest-one", "echo one 1.0"); + + expect(await missingToolchains(["one"], env, { toolchains })).toEqual( + new Map(), + ); +}); + +it("names each tool that the login shell cannot find, and skips an unknown group", async () => { + await tool("wbtest-compile", "echo compile 1.0"); + + expect( + await missingToolchains(["one", "two", "unknown"], env, { toolchains }), + ).toEqual( + new Map([ + ["one", "wbtest-one was not found on the login shell's PATH"], + ["two", "wbtest-build was not found on the login shell's PATH"], + ]), + ); +}); + +it("gives up on a tool that does not answer, and says so", async () => { + await tool("wbtest-slow", "sleep 30"); + + const started = Date.now(); + + expect( + await missingToolchains(["slow"], env, { toolchains, timeoutMs: 500 }), + ).toEqual( + new Map([["slow", "wbtest-slow --version did not answer within 0.5 s"]]), + ); + expect(Date.now() - started).toBeLessThan(5_000); +}); diff --git a/packages/review/src/remote-toolchains.ts b/packages/review/src/remote-toolchains.ts new file mode 100644 index 000000000..b0c41c5c5 --- /dev/null +++ b/packages/review/src/remote-toolchains.ts @@ -0,0 +1,107 @@ +import { spawn } from "node:child_process"; +import path from "node:path"; + +const PROBE_TIMEOUT_MS = 10_000; + +type Toolchains = Record; + +const TOOLCHAINS = { + rust: [ + ["cargo", "--version"], + ["rustc", "--version"], + ], + swift: [["swift", "--version"]], + csharp: [["dotnet", "--version"]], + go: [["go", "version"]], +} satisfies Toolchains; + +type Outcome = "found" | "missing" | string; + +function probe( + command: readonly string[], + env: NodeJS.ProcessEnv, + timeoutMs: number, +) { + const shell = env.SHELL || "/bin/sh"; + + const flags = ["csh", "tcsh"].includes(path.basename(shell)) + ? ["-ic"] + : ["-i", "-l", "-c"]; + + return new Promise((resolve) => { + const child = spawn(shell, [...flags, command.join(" ")], { + env, + detached: true, + stdio: "ignore", + }); + + const timer = setTimeout(() => { + child.removeAllListeners("exit"); + + try { + process.kill(-child.pid!, "SIGKILL"); + } catch {} + + resolve( + `${command.join(" ")} did not answer within ${timeoutMs / 1_000} s`, + ); + }, timeoutMs); + + child.once("error", (error) => { + clearTimeout(timer); + resolve(`the login shell ${shell} did not run: ${error.message}`); + }); + child.once("exit", (code) => { + clearTimeout(timer); + resolve( + code === 0 + ? "found" + : code === 127 + ? "missing" + : `${command.join(" ")} failed with exit code ${code ?? "none"}`, + ); + }); + }); +} + +export async function missingToolchains( + groups: readonly string[], + env: NodeJS.ProcessEnv, + { + timeoutMs = PROBE_TIMEOUT_MS, + toolchains = TOOLCHAINS, + }: { timeoutMs?: number; toolchains?: Toolchains } = {}, +) { + const entries = await Promise.all( + groups.map(async (group) => { + const commands = Object.hasOwn(toolchains, group) + ? toolchains[group] + : []; + + const outcomes = await Promise.all( + commands.map((command) => probe(command, env, timeoutMs)), + ); + + const missing = commands + .filter((_, i) => outcomes[i] === "missing") + .map(([tool]) => tool); + + const failures = outcomes.filter( + (outcome) => outcome !== "found" && outcome !== "missing", + ); + + const details = [ + ...(missing.length > 0 + ? [ + `${missing.join(" and ")} ${missing.length > 1 ? "were" : "was"} not found on the login shell's PATH`, + ] + : []), + ...failures, + ]; + + return details.length > 0 ? [[group, details.join("; ")] as const] : []; + }), + ); + + return new Map(entries.flat()); +} diff --git a/packages/review/src/remote-uninstall.test.ts b/packages/review/src/remote-uninstall.test.ts new file mode 100644 index 000000000..c50f221ae --- /dev/null +++ b/packages/review/src/remote-uninstall.test.ts @@ -0,0 +1,651 @@ +import { type ChildProcess, spawn, spawnSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { once } from "node:events"; +import { existsSync } from "node:fs"; +import { + mkdir, + mkdtemp, + readFile, + readdir, + realpath, + rm, + writeFile, +} from "node:fs/promises"; +import { type Server, createServer } from "node:http"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { PassThrough } from "node:stream"; + +import { REVIEW_REMOTE_WRAPPER_MARK } from "@dev.fast/review-protocol"; +import { processStartIdentity } from "@dev.fast/trace-core"; +import { runReviewCli } from "@review/cli-runner.js"; +import { remoteServerPaths } from "@review/remote-extensions.js"; +import { remoteUninstall, takeInstallLock } from "@review/remote-uninstall.js"; +import { reviewServerDiscoveryPath } from "@review/server-discovery.js"; +import { afterEach, beforeEach, expect, it } from "vitest"; + +let root: string; + +let home: string; + +let stateDir: string; + +let env: NodeJS.ProcessEnv; + +let install: string; + +let wrapper: string; + +const children: ChildProcess[] = []; + +const groups: number[] = []; + +const servers: Server[] = []; + +beforeEach(async () => { + root = await realpath(await mkdtemp(path.join(tmpdir(), "wb-uninstall-"))); + home = path.join(root, "home"); + stateDir = path.join(home, ".dev"); + env = { DEV_REVIEW_HOME: stateDir }; + install = path.join(stateDir, "whiteboard-remote"); + wrapper = path.join(home, ".local", "bin", "whiteboard"); + + const version = path.join(install, "versions", "0.1.6"); + await mkdir(path.join(version, "node_modules"), { recursive: true }); + await writeFile(path.join(version, ".whiteboard-install.json"), "{}\n"); + await mkdir(path.join(install, "node", "v24.18.0", "bin"), { + recursive: true, + }); + await mkdir(path.dirname(wrapper), { recursive: true }); + await writeFile( + wrapper, + `#!/bin/sh\n${REVIEW_REMOTE_WRAPPER_MARK}\nexec '${version}/whiteboard' "$@"\n`, + { mode: 0o755 }, + ); + + for (const name of [ + "review-api.db", + "review-api.db-wal", + "review-api.db-shm", + "review-api.db.workspaces", + ]) + await writeFile(path.join(stateDir, name), "reviews"); + await mkdir(path.join(stateDir, "review-tools"), { recursive: true }); + await writeFile(path.join(stateDir, "review-tools", "keep"), ""); +}); + +afterEach(async () => { + for (const child of children.splice(0)) child.kill("SIGKILL"); + + for (const leader of groups.splice(0)) + try { + process.kill(-leader, "SIGKILL"); + } catch {} + + for (const server of servers.splice(0)) server.close(); + await rm(root, { recursive: true, force: true }); +}); + +const homeEntries = async () => (await readdir(stateDir)).sort(); + +async function runningFrom(cli: string) { + const child = spawn( + process.execPath, + ["-e", "setInterval(() => {}, 1000)", cli], + { stdio: "ignore" }, + ); + + children.push(child); + await once(child, "spawn"); + + return child; +} + +async function groupFrom(file: string, child?: string) { + const forever = "setInterval(() => {}, 1000)"; + + const leader = spawn( + process.execPath, + [ + "-e", + child + ? `require("child_process").spawn(process.execPath, ["-e", ${JSON.stringify(forever)}, ${JSON.stringify(child)}], { stdio: "ignore" }); ${forever}` + : forever, + file, + ], + { stdio: "ignore", detached: true }, + ); + + groups.push(leader.pid!); + await once(leader, "spawn"); + + return leader.pid!; +} + +const runningWith = (text: string) => + spawnSync("pgrep", ["-f", text]).status === 0; + +async function languageServerRecords(server: number, ensure?: number) { + const { serverDataDir } = remoteServerPaths(env); + await mkdir(serverDataDir, { recursive: true }); + await writeFile( + path.join(serverDataDir, "server.json"), + JSON.stringify({ + pid: server, + started: processStartIdentity(server), + port: 1, + }), + ); + + if (ensure !== undefined) + await writeFile( + path.join(serverDataDir, "install.json"), + JSON.stringify({ pid: ensure, started: processStartIdentity(ensure) }), + ); +} + +async function serverRecord( + pid: number, + startedBy: "user" | "cli" | "desktop", +) { + const instanceId = randomUUID(); + const token = "token"; + + const server = createServer((request, response) => { + response.setHeader("content-type", "application/json"); + response.end( + request.headers["x-review-token"] === token + ? JSON.stringify({ + ok: true, + instanceId, + serverPid: pid, + version: "0.1.6", + }) + : "{}", + ); + }); + + servers.push(server); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const { port } = server.address() as { port: number }; + await mkdir(path.dirname(reviewServerDiscoveryPath(stateDir)), { + recursive: true, + }); + await writeFile( + reviewServerDiscoveryPath(stateDir), + JSON.stringify({ + version: 1, + instanceId, + url: `http://127.0.0.1:${port}`, + serverPid: pid, + token, + startedBy, + }), + ); +} + +const alive = (pid: number) => { + try { + process.kill(pid, 0); + + return true; + } catch { + return false; + } +}; + +it("with --keep-reviews removes the install and Desktop's script, and leaves the review store", async () => { + const result = await remoteUninstall({ + env, + home, + stateDir, + deleteReviews: false, + }); + + expect(result).toEqual({ + event: "remote.uninstall", + ok: true, + removed: [install, wrapper], + keptReviews: true, + }); + expect(existsSync(install)).toBe(false); + expect(existsSync(wrapper)).toBe(false); + expect(await homeEntries()).toEqual([ + "review-api.db", + "review-api.db-shm", + "review-api.db-wal", + "review-api.db.workspaces", + "review-tools", + ]); +}); + +it("with --delete-reviews also removes the review store and nothing else in the review home", async () => { + const result = await remoteUninstall({ + env, + home, + stateDir, + deleteReviews: true, + }); + + expect(result).toMatchObject({ + ok: true, + keptReviews: false, + removed: [ + install, + wrapper, + path.join(stateDir, "review-api.db"), + path.join(stateDir, "review-api.db-wal"), + path.join(stateDir, "review-api.db-shm"), + path.join(stateDir, "review-api.db.workspaces"), + ], + }); + expect(await homeEntries()).toEqual(["review-tools"]); +}); + +it("leaves a ~/.local/bin/whiteboard that Desktop did not write", async () => { + await writeFile(wrapper, "#!/bin/sh\necho mine\n"); + + const result = await remoteUninstall({ + env, + home, + stateDir, + deleteReviews: false, + }); + + expect(result).toMatchObject({ ok: true, removed: [install] }); + expect(await readFile(wrapper, "utf8")).toBe("#!/bin/sh\necho mine\n"); +}); + +it("stops a server Desktop started and reports it", async () => { + const server = await runningFrom( + path.join(install, "versions", "0.1.6", "cli.js"), + ); + + await serverRecord(server.pid!, "desktop"); + + const result = await remoteUninstall({ + env, + home, + stateDir, + deleteReviews: false, + }); + + expect(result).toMatchObject({ + ok: true, + stoppedServer: { pid: server.pid, version: "0.1.6" }, + }); + expect(alive(server.pid!)).toBe(false); + expect(existsSync(install)).toBe(false); +}); + +it("refuses while a server the user started runs from the install, and removes nothing", async () => { + const server = await runningFrom( + path.join(install, "versions", "0.1.6", "cli.js"), + ); + + await serverRecord(server.pid!, "user"); + + const result = await remoteUninstall({ + env, + home, + stateDir, + deleteReviews: false, + }); + + expect(result).toEqual({ + event: "remote.uninstall", + ok: false, + reason: expect.stringContaining(`process ${server.pid}`), + }); + expect(alive(server.pid!)).toBe(true); + expect(existsSync(install)).toBe(true); + expect(existsSync(wrapper)).toBe(true); + expect(existsSync(path.join(install, "install.lock"))).toBe(false); +}); + +it("refuses while any other process runs from the install, naming it, and stops nothing", async () => { + const desktopServer = await runningFrom( + path.join(install, "versions", "0.1.6", "cli.js"), + ); + + await serverRecord(desktopServer.pid!, "desktop"); + + const mcp = await runningFrom( + path.join(install, "versions", "0.1.6", "cli.js mcp"), + ); + + const result = await remoteUninstall({ + env, + home, + stateDir, + deleteReviews: false, + }); + + expect(result).toEqual({ + event: "remote.uninstall", + ok: false, + reason: `Process ${mcp.pid} runs from ${install}. Stop it, then run whiteboard remote uninstall again.`, + }); + expect(alive(mcp.pid!)).toBe(true); + expect(alive(desktopServer.pid!)).toBe(true); + expect(existsSync(install)).toBe(true); +}); + +it("takes over a stale install lock", async () => { + await mkdir(path.join(install, "install.lock")); + await writeFile( + path.join(install, "install.lock", "started"), + `${Math.floor(Date.now() / 1000) - 16 * 60}\n`, + ); + + const result = await remoteUninstall({ + env, + home, + stateDir, + deleteReviews: false, + }); + + expect(result).toMatchObject({ ok: true, removed: [install, wrapper] }); +}); + +async function installLock(token: string, ago: number) { + const lock = path.join(install, "install.lock"); + + await mkdir(lock, { recursive: true }); + await writeFile(path.join(lock, "token"), `${token}\n`); + await writeFile(path.join(lock, "owner"), "laptop\n"); + await writeFile( + path.join(lock, "started"), + `${Math.floor(Date.now() / 1000) - ago}\n`, + ); + + return lock; +} + +const lockEntries = async () => + (await readdir(install)).filter((name) => name.startsWith("install.lock")); + +it("leaves a stale lock its holder refreshes during the check, and refuses", async () => { + const lock = await installLock("installer", 16 * 60); + + const taken = await takeInstallLock(install, { + beforeMove: () => + writeFile( + path.join(lock, "started"), + `${Math.floor(Date.now() / 1000)}\n`, + ), + }); + + expect(taken).toEqual({ holder: "laptop" }); + expect(await readFile(path.join(lock, "token"), "utf8")).toBe("installer\n"); + expect(await lockEntries()).toEqual(["install.lock"]); +}); + +it("leaves a lock another install took over during the check", async () => { + const lock = await installLock("installer", 16 * 60); + + const taken = await takeInstallLock(install, { + beforeMove: async () => { + await rm(lock, { recursive: true }); + await installLock("winner", 0); + }, + }); + + expect(taken).toEqual({ holder: "laptop" }); + expect(await readFile(path.join(lock, "token"), "utf8")).toBe("winner\n"); + expect(await lockEntries()).toEqual(["install.lock"]); +}); + +it("releases the lock when writing it fails", async () => { + await expect( + takeInstallLock(install, { + afterMkdir: () => mkdir(path.join(install, "install.lock", "token")), + }), + ).rejects.toThrow(/EISDIR/); + expect(await lockEntries()).toEqual([]); +}); + +it("removes nothing without an absolute home", async () => { + for (const relative of ["", ".dev"]) { + const result = await remoteUninstall({ + env, + home: relative, + stateDir, + deleteReviews: true, + }); + + expect(result).toMatchObject({ ok: false }); + } + + expect( + await remoteUninstall({ + env, + home, + stateDir: ".dev", + deleteReviews: true, + }), + ).toMatchObject({ ok: false }); + expect(existsSync(install)).toBe(true); + expect(await homeEntries()).toContain("review-api.db"); +}); + +it("refuses while an install holds the lock", async () => { + await mkdir(path.join(install, "install.lock")); + await writeFile(path.join(install, "install.lock", "owner"), "laptop\n"); + await writeFile( + path.join(install, "install.lock", "started"), + `${Math.floor(Date.now() / 1000)}\n`, + ); + + const result = await remoteUninstall({ + env, + home, + stateDir, + deleteReviews: false, + }); + + expect(result).toMatchObject({ + ok: false, + reason: expect.stringContaining("laptop"), + }); + expect(existsSync(path.join(install, "versions"))).toBe(true); +}); + +async function cli(argv: string[], change: NodeJS.ProcessEnv = {}) { + const stdout = new PassThrough(); + const stderr = new PassThrough(); + let out = ""; + let err = ""; + stdout.on("data", (chunk) => (out += String(chunk))); + stderr.on("data", (chunk) => (err += String(chunk))); + + const code = await runReviewCli({ + argv, + stdout, + stderr, + env: { + PATH: process.env.PATH, + HOME: home, + DEV_REVIEW_HOME: stateDir, + DEV_FAST_REVIEW_TELEMETRY_DISABLED: "1", + DEV_FAST_REVIEW_CLI_NO_DELEGATE: "1", + ...change, + }, + }); + + return { code, stdout: out, stderr: err }; +} + +it("in --json mode with neither flag refuses and removes nothing", async () => { + const result = await cli(["remote", "uninstall", "--json"]); + + expect(result.code).toBe(1); + expect(JSON.parse(result.stdout)).toEqual({ + event: "remote.uninstall", + ok: false, + reason: expect.stringContaining("--keep-reviews or --delete-reviews"), + }); + expect(existsSync(install)).toBe(true); + expect(existsSync(wrapper)).toBe(true); +}); + +it("with HOME empty refuses and removes nothing", async () => { + const result = await cli( + ["remote", "uninstall", "--keep-reviews", "--json"], + { + HOME: "", + }, + ); + + expect(result.code).toBe(1); + expect(JSON.parse(result.stdout)).toEqual({ + event: "remote.uninstall", + ok: false, + reason: + "HOME is not set; Whiteboard removes nothing without an absolute home.", + }); + expect(existsSync(install)).toBe(true); +}); + +it("prints one JSON line on success", async () => { + const result = await cli(["remote", "uninstall", "--keep-reviews", "--json"]); + + expect(result).toMatchObject({ code: 0, stderr: "" }); + expect(JSON.parse(result.stdout)).toEqual({ + event: "remote.uninstall", + ok: true, + removed: [install, wrapper], + keptReviews: true, + }); +}); + +it("with DEV_REVIEW_HOME elsewhere removes the install there, with the VS Code server's data, and leaves ~/.dev alone", async () => { + const moved = path.join(root, "moved"); + const movedInstall = path.join(moved, "whiteboard-remote"); + const { serverDataDir } = remoteServerPaths({ DEV_REVIEW_HOME: moved }); + await mkdir(path.join(movedInstall, "versions", "0.1.6"), { + recursive: true, + }); + await mkdir(serverDataDir, { recursive: true }); + await writeFile(path.join(serverDataDir, "connection-token"), "token"); + + const result = await cli( + ["remote", "uninstall", "--keep-reviews", "--json"], + { + DEV_REVIEW_HOME: moved, + }, + ); + + expect(result).toMatchObject({ code: 0, stderr: "" }); + expect(JSON.parse(result.stdout)).toMatchObject({ + ok: true, + removed: [movedInstall, wrapper], + }); + expect(existsSync(serverDataDir)).toBe(false); + expect(existsSync(install)).toBe(true); +}); + +it("stops the VS Code server with its extension host, and a detached extension install, then removes the install", async () => { + const pkg = path.join( + install, + "versions", + "0.1.6", + "node_modules", + "@dev.fast", + "whiteboard", + ); + + const extensionHost = path.join( + install, + "extensions", + "ms-python.python", + "server.js", + ); + + const server = await groupFrom( + path.join(pkg, "vscode-server", "out", "server-main.js"), + extensionHost, + ); + + const ensure = await groupFrom(path.join(pkg, "dist", "cli.js")); + await languageServerRecords(server, ensure); + + for (let i = 0; i < 100 && !runningWith(extensionHost); i++) + await new Promise((resolve) => setTimeout(resolve, 50)); + + const result = await remoteUninstall({ + env, + home, + stateDir, + deleteReviews: false, + }); + + expect(result).toMatchObject({ ok: true, removed: [install, wrapper] }); + expect(runningWith(`${install}/`)).toBe(false); +}); + +it("refuses a server the user started before it stops anything, naming only that server", async () => { + const server = await groupFrom( + path.join( + install, + "versions", + "0.1.6", + "vscode-server", + "out", + "server-main.js", + ), + path.join(install, "extensions", "ms-python.python", "server.js"), + ); + + await languageServerRecords(server); + + const user = await runningFrom( + path.join(install, "versions", "0.1.6", "cli.js"), + ); + + await serverRecord(user.pid!, "user"); + + const result = await remoteUninstall({ + env, + home, + stateDir, + deleteReviews: false, + }); + + expect(result).toMatchObject({ + ok: false, + reason: expect.stringContaining( + `A Whiteboard server you started (process ${user.pid})`, + ), + }); + expect((result as { reason: string }).reason).not.toContain(`${server}`); + expect(alive(user.pid!)).toBe(true); + expect(alive(server)).toBe(true); + expect(existsSync(path.join(install, "versions"))).toBe(true); +}); + +it("removes nothing under a DEV_REVIEW_HOME it cannot normalise, as the probe refuses it", async () => { + for (const moved of [ + "relative/home", + `${root}/a/../b`, + `${root}//b`, + `//${root}`, + `${root}/a\nb`, + `${root}/a\u001bb`, + ]) { + const result = await remoteUninstall({ + env: { DEV_REVIEW_HOME: moved }, + home, + stateDir, + deleteReviews: false, + }); + + expect(result).toMatchObject({ + ok: false, + reason: expect.stringContaining("not an absolute, normalised path"), + }); + } + + expect(existsSync(install)).toBe(true); +}); diff --git a/packages/review/src/remote-uninstall.ts b/packages/review/src/remote-uninstall.ts new file mode 100644 index 000000000..cf318a40d --- /dev/null +++ b/packages/review/src/remote-uninstall.ts @@ -0,0 +1,361 @@ +import { execFile } from "node:child_process"; +import { randomBytes } from "node:crypto"; +import { existsSync } from "node:fs"; +import { + lstat, + mkdir, + readFile, + readdir, + rename, + rm, + stat, + writeFile, +} from "node:fs/promises"; +import path from "node:path"; +import { promisify } from "node:util"; + +import { + REVIEW_REMOTE_INSTALL_LOCK, + REVIEW_REMOTE_LOCK_STALE_SECONDS, + REVIEW_REMOTE_WRAPPER_MARK, +} from "@dev.fast/review-protocol"; + +import { whiteboardRemoteHome } from "./remote-extensions"; +import { + remoteLanguageServerGroups, + stopProcessGroup, +} from "./remote-language-server"; +import { DEV_REVIEW_HOME_ENV } from "./review-home-paths"; +import { + recordedBackgroundServer, + stopBackgroundServer, +} from "./server/background-server"; + +const LOCK_STALE_MS = REVIEW_REMOTE_LOCK_STALE_SECONDS * 1000; + +const REVIEW_STORE = [ + "review-api.db", + "review-api.db-wal", + "review-api.db-shm", + "review-api.db.workspaces", + "review-api.db.workspaces-wal", + "review-api.db.workspaces-shm", +]; + +export type RemoteUninstallResult = + | { + event: "remote.uninstall"; + ok: true; + removed: string[]; + keptReviews: boolean; + stoppedServer?: { pid: number; version: string | null }; + } + | { event: "remote.uninstall"; ok: false; reason: string }; + +export async function remoteUninstall(input: { + home: string; + env: NodeJS.ProcessEnv; + stateDir: string; + deleteReviews: boolean; +}): Promise { + const refuse = (reason: string): RemoteUninstallResult => ({ + event: "remote.uninstall", + ok: false, + reason, + }); + + // A relative home would aim every removal at the working directory. + if (!path.isAbsolute(input.home)) + return refuse( + `HOME is ${input.home ? JSON.stringify(input.home) : "not set"}; Whiteboard removes nothing without an absolute home.`, + ); + + if (!path.isAbsolute(input.stateDir)) + return refuse( + `The review home ${JSON.stringify(input.stateDir)} is not an absolute path.`, + ); + + const override = input.env[DEV_REVIEW_HOME_ENV]?.trim(); + + if ( + override && + (/[\x00-\x1f\x7f-\x9f]/.test(override) || + override.includes("//") || + path.resolve(override) !== (override.replace(/\/$/, "") || "/")) + ) + return refuse( + `DEV_REVIEW_HOME is ${JSON.stringify(override)}; Whiteboard removes nothing under a review home that is not an absolute, normalised path.`, + ); + + const install = whiteboardRemoteHome(input.env); + const wrapper = path.join(input.home, ".local", "bin", "whiteboard"); + const lock = path.join(install, REVIEW_REMOTE_INSTALL_LOCK); + + const taken = existsSync(install) + ? await takeInstallLock(install) + : undefined; + + if (taken && "holder" in taken) + return refuse( + `Desktop on ${taken.holder || "another computer"} is installing Whiteboard here. Try again when it is done.`, + ); + + const held = taken?.token; + let removedInstall = false; + + try { + const server = await recordedBackgroundServer(input.stateDir); + + const recorded = server && { + ...server.discovery, + version: server.health.version ?? null, + }; + + const stoppable = + recorded && recorded.startedBy !== "user" + ? recorded.serverPid + : undefined; + + const groups = await remoteLanguageServerGroups(input.env); + const running = await processesFrom(`${install}/`); + + if (running === undefined) + return refuse( + `Cannot list this host's processes to check that none runs from ${install}.`, + ); + + const others = running + .filter(({ pid, pgid }) => pid !== stoppable && !groups.includes(pgid)) + .map(({ pid }) => pid); + + if (others.length) + return refuse( + `${others.map((pid) => (pid === recorded?.serverPid ? `A Whiteboard server you started (process ${pid})` : `Process ${pid}`)).join(", ")} ${others.length > 1 ? "run" : "runs"} from ${install}. Stop ${others.length > 1 ? "them" : "it"}, then run whiteboard remote uninstall again.`, + ); + + if (recorded?.startedBy === "user" && input.deleteReviews) + return refuse( + `A Whiteboard server you started (process ${recorded.serverPid}) uses the reviews in ${input.stateDir}. Stop it, then run whiteboard remote uninstall again.`, + ); + + await Promise.all(groups.map(stopProcessGroup)); + let stoppedServer: { pid: number; version: string | null } | undefined; + + if (recorded && stoppable !== undefined) { + try { + await stopBackgroundServer({ serverPid: stoppable }); + } catch (error) { + return refuse(error instanceof Error ? error.message : String(error)); + } + + stoppedServer = { pid: stoppable, version: recorded.version }; + } + + const left = await processesFrom(`${install}/`); + + if (left?.length !== 0) + return refuse( + `${left ? `Process ${left.map(({ pid }) => pid).join(", ")}` : "A process"} still runs from ${install}. Stop it, then run whiteboard remote uninstall again.`, + ); + + const removed: string[] = []; + + if (held) { + await rm(install, { recursive: true, force: true }); + removedInstall = true; + removed.push(install); + } + + if (await desktopWrote(wrapper)) { + await rm(wrapper, { force: true }); + removed.push(wrapper); + } + + if (input.deleteReviews) + for (const name of REVIEW_STORE) { + const file = path.join(input.stateDir, name); + + if (existsSync(file)) { + await rm(file, { force: true }); + removed.push(file); + } + } + + const result: RemoteUninstallResult = { + event: "remote.uninstall", + ok: true, + removed, + keptReviews: !input.deleteReviews, + }; + + if (stoppedServer) result.stoppedServer = stoppedServer; + + return result; + } finally { + if (held && !removedInstall) await releaseLock(lock, held); + } +} + +export async function takeInstallLock( + install: string, + hooks: { + beforeMove?(): Promise; + afterMkdir?(): Promise; + } = {}, +): Promise<{ token: string } | { holder: string }> { + const lock = path.join(install, REVIEW_REMOTE_INSTALL_LOCK); + const token = randomBytes(8).toString("hex"); + + for (let attempt = 0; attempt < 5; attempt++) { + if (attempt) await new Promise((resolve) => setTimeout(resolve, 200)); + + if ( + await mkdir(lock).then( + () => true, + () => false, + ) + ) { + try { + await hooks.afterMkdir?.(); + await writeFile(path.join(lock, "token"), `${token}\n`); + await writeFile( + path.join(lock, "owner"), + "whiteboard-remote-uninstall\n", + ); + await stamp(lock, token); + } catch (error) { + await rm(lock, { recursive: true, force: true }); + throw error; + } + + return { token }; + } + + const judged = await readText(path.join(lock, "token")); + + if (!(await stale(lock))) + return { holder: await readText(path.join(lock, "owner")) }; + + await hooks.beforeMove?.(); + + const aside = path.join( + install, + `${REVIEW_REMOTE_INSTALL_LOCK}.${token}.stale`, + ); + + if ( + !(await rename(lock, aside).then( + () => true, + () => false, + )) + ) + continue; + + if ( + (await readText(path.join(aside, "token"))) === judged && + (await stale(aside)) + ) + await rm(aside, { recursive: true, force: true }); + else await rename(aside, lock).catch(() => undefined); + } + + return { holder: await readText(path.join(lock, "owner")) }; +} + +async function stale(lock: string) { + const started = await readText(path.join(lock, "started")); + + if (started !== "") + return ( + /^\d+$/.test(started) && + Date.now() - Number(started) * 1000 >= LOCK_STALE_MS + ); + + if (existsSync(path.join(lock, "started"))) return false; + + const since = (await stat(lock).catch(() => undefined))?.mtimeMs; + + return since !== undefined && Date.now() - since >= LOCK_STALE_MS; +} + +async function stamp(lock: string, token: string) { + const next = path.join(lock, `started.${token}`); + + await writeFile(next, `${Math.floor(Date.now() / 1000)}\n`); + await rename(next, path.join(lock, "started")); +} + +async function releaseLock(lock: string, token: string) { + if ((await readText(path.join(lock, "token"))) !== token) return; + const done = `${lock}.${token}.done`; + + if ( + await rename(lock, done).then( + () => true, + () => false, + ) + ) + await rm(done, { recursive: true, force: true }); +} + +const readText = (file: string) => + readFile(file, "utf8").then( + (text) => text.trim(), + () => "", + ); + +async function processesFrom(prefix: string) { + let lines: { pid: number; pgid: number; args: string }[]; + + try { + const pids = (await readdir("/proc")).filter((name) => /^\d+$/.test(name)); + + lines = await Promise.all( + pids.map(async (pid) => { + const [args = "", stat = ""] = await Promise.all( + [`/proc/${pid}/cmdline`, `/proc/${pid}/stat`].map((file) => + readFile(file, "utf8").catch(() => ""), + ), + ); + + // The group follows the state and the parent, after the command's parentheses. + const pgid = Number( + stat.slice(stat.lastIndexOf(")") + 2).split(" ")[2], + ); + + return { pid: Number(pid), pgid, args: args.replaceAll("\0", " ") }; + }), + ); + } catch { + try { + const { stdout } = await promisify(execFile)( + "ps", + ["-eo", "pid=,pgid=,args="], + { maxBuffer: 16 << 20 }, + ); + + lines = stdout.split("\n").map((line) => { + const [, pid = "", pgid = "", args = ""] = + /^\s*(\d+)\s+(\d+)\s(.*)$/.exec(line) ?? []; + + return { pid: Number(pid), pgid: Number(pgid), args }; + }); + } catch { + return undefined; + } + } + + return lines.filter( + ({ pid, args }) => pid && pid !== process.pid && args.includes(prefix), + ); +} + +async function desktopWrote(file: string) { + const entry = await lstat(file).catch(() => undefined); + + if (!entry?.isFile()) return false; + + return (await readFile(file, "utf8")) + .split("\n") + .includes(REVIEW_REMOTE_WRAPPER_MARK); +} diff --git a/packages/review/src/review-api/README.md b/packages/review/src/review-api/README.md index 6ca874dfb..60d633008 100644 --- a/packages/review/src/review-api/README.md +++ b/packages/review/src/review-api/README.md @@ -46,8 +46,7 @@ All paths below are relative to `/reviews-api`. | `GET /capabilities` | Desktop availability and permission for optional software-map generation, independent of opening a review | | `GET /:id/activity` | Currently reported authoring work, not stored in document history | | `POST /:id/activity/{begin,update,end} {activityId?,focus?}` | Begin a presence (returns its `activityId`), update its focus and expiry, or end it; each returns the review's live presences | -| `GET /watch` | NDJSON review summaries: initial list, then saved changes | -| `GET /watch?subscriptions=…` | One NDJSON connection for multiple `{reviewId}` subscriptions; `reviewId:null` selects the catalog. Each line is an ordered array of `{value}` or `{error}` results, with `null` where a subscription is unchanged since the previous line. | +| `GET /watch?subscriptions=[{reviewId,mode?}]` | NDJSON live updates. `reviewId:null` subscribes to the review list in `mode` (`structural` when absent); a string subscribes to that review. Each line names what it is about: `{kind:"list",mode,reviews}`, `{kind:"review",reviewId,value}` (the snapshot plus `activity` and `coverageRevision`), or `{kind:"review",reviewId,error}`. On connect there is one line per list mode and per review, however often each is subscribed; afterwards a line only for what changed (an edit changes the list entry's `version`, so it sends both; activity and coverage send only the review; an unchanged list is not resent). A missing or deleted review is an `error` line and the stream stays open; a reader that falls behind receives each item's latest state once. | | `GET /:id` | Compact outline | | `GET /:id?targetId=step-3` | Full block, sequence step, flow node or flow edge | | `GET /:id?full=true` | Full snapshot | @@ -55,7 +54,6 @@ All paths below are relative to `/reviews-api`. | `GET /:id/history` | Saved versions with titles and timestamps | | `GET /:id/inspect` | Agent reading view: nested text outline with IDs; `targetId` reads one component completely, `full=true` includes all content, `version` selects history. `format=json` returns raw data instead. | | `POST /:id/open` | Open the review in the attached Desktop; report an error when none is attached | -| `GET /:id/watch` | NDJSON snapshots: current state immediately, then committed updates | | `POST /commands` | Apply one command; return review ID, version, and for an edit the target ID, its `type`, and — after an insert or replace — `children`: its first-level children as `{id,type}` (a container's blocks; a diagram's steps, or nodes then edges), so new components are addressable without a read. A `create` with `pullRequestUrl` returns the newest existing review for that PR instead (owner/repository matched case-insensitively) unless `operation.reuseExisting` is `false`: `created:false`, a `note`, its stored `target`, `headMoved`, and `working`/`otherReviewIds` when they apply; its target is never moved. A new review reports `created:true`. Either way the result carries `review`, the review's `GET /` catalog entry (target, origin, repository name and path). An interactive `create` also opens the new review in an attached Desktop unless `operation.open` is `false`, and reports `opened` with the open result or an `openError`; the review is saved either way | | `POST /commands {operation:{type:"lens_edit",reviewId,edit}}` | Write one file lens, credited to `activityId` when given: `insert {title,targets,afterId?}` (host id `lens-N`), `update {targetId,title?,targets?}` or `remove {targetId}`. Returns `{targetId, type:"lens", uncategorized}`, where `uncategorized` lists changed files and ranges no lens covers yet (at most 50 files) | | `GET /:id/lenses` | The current version's file lenses with each one's file count, plus the same `uncategorized` report | diff --git a/packages/review/src/review-api/activity.test.ts b/packages/review/src/review-api/activity.test.ts index fd4fecbda..a23007f03 100644 --- a/packages/review/src/review-api/activity.test.ts +++ b/packages/review/src/review-api/activity.test.ts @@ -93,7 +93,7 @@ it("reports working transitions without heartbeats or focus changes", () => { activity.close(); }); -it("streams activity separately from document versions and closes the stream on deletion", async () => { +it("streams activity separately from document versions and reports deletion on the stream", async () => { const store = new ReviewStore(":memory:", { validatePins: async () => {}, validateSource: async () => {}, @@ -124,11 +124,11 @@ it("streams activity separately from document versions and closes the stream on const changed = vi.fn[0]>(); store.subscribe(changed); const abort = new AbortController(); - const stream = client.watch(reviewId, abort.signal); + const stream = client.watch([{ reviewId }], abort.signal); try { expect((await stream.next()).value).toMatchObject({ - activity: { workingCount: 0 }, + value: { activity: { workingCount: 0 } }, }); const input = { focus: { description: "Drafting outline" } }; @@ -139,30 +139,35 @@ it("streams activity separately from document versions and closes the stream on ); expect((await stream.next()).value).toMatchObject({ - activity: { - workingCount: 1, - activities: [{ activityId, slot: 0, focus: input.focus }], + value: { + activity: { + workingCount: 1, + activities: [{ activityId, slot: 0, focus: input.focus }], + }, }, }); expect(changed).not.toHaveBeenCalled(); expect(store.history(reviewId)).toHaveLength(1); - const reconnect = client.watch(reviewId, abort.signal); + const reconnect = client.watch([{ reviewId }], abort.signal); expect((await reconnect.next()).value).toMatchObject({ - activity: { workingCount: 1, activities: [{ focus: input.focus }] }, + value: { + activity: { workingCount: 1, activities: [{ focus: input.focus }] }, + }, }); await reconnect.return(undefined); await store.execute({ operation: { type: "delete", reviewId } }); - // A reader may already have buffered a pre-deletion snapshot. - await expect(async () => { - for await (const _snapshot of stream) { - } - }).rejects.toThrow(Error); + expect((await stream.next()).value).toEqual({ + kind: "review", + reviewId, + error: expect.stringMatching(/not found/i), + }); await expect( client.post(`/${reviewId}/activity/begin`, input), ).rejects.toThrow(/not found/i); expect(store.activity.read(reviewId).workingCount).toBe(0); } finally { abort.abort(); + await stream.return(undefined); await store.close(); } }); @@ -211,85 +216,79 @@ it("keeps each agent's focus until it changes, clears or expires", () => { activity.close(); }); -it.each([false, true])( - "streams completion and expiry to the catalog without an open canvas (multiplexed=%s)", - async (multiplexed) => { - vi.useFakeTimers(); +it("streams completion and expiry to the catalog without an open canvas", async () => { + vi.useFakeTimers(); - const store = new ReviewStore(":memory:", { - validatePins: async () => {}, - validateSource: async () => {}, - validateResource: async () => {}, - }); + const store = new ReviewStore(":memory:", { + validatePins: async () => {}, + validateSource: async () => {}, + validateResource: async () => {}, + }); - const api = createReviewApi(store); - - const { reviewId } = await store.execute({ - operation: { - type: "create", - title: "Background review", - target: { - kind: "commits", - repositoryId: "repo", - base: "base", - head: "head", - }, + const api = createReviewApi(store); + + const { reviewId } = await store.execute({ + operation: { + type: "create", + title: "Background review", + target: { + kind: "commits", + repositoryId: "repo", + base: "base", + head: "head", }, - }); + }, + }); - const client = new ReviewApiClient( - { serverUrl: "http://review.test", token: "test" }, - async (url, init) => api.request(url.replace("/reviews-api", ""), init), - ); + const client = new ReviewApiClient( + { serverUrl: "http://review.test", token: "test" }, + async (url, init) => api.request(url.replace("/reviews-api", ""), init), + ); - const abort = new AbortController(); + const abort = new AbortController(); - const stream = client.watch( - multiplexed ? [{ reviewId: null }] : null, - abort.signal, - ); + const stream = client.watch([{ reviewId: null }], abort.signal); + + const next = async () => { + const line = (await stream.next()).value; - const next = async () => { - const value = (await stream.next()).value; - - return multiplexed ? (value as { value: unknown }[])[0]!.value : value; - }; - - const expected = (working: boolean) => [ - expect.objectContaining({ reviewId, working }), - ]; - - try { - expect(await next()).toEqual(expected(false)); - - const { activityId } = store.activity.update(reviewId, { - action: "begin", - }); - - expect(await next()).toEqual(expected(true)); - // One line per transition: no repeat, nothing for renewals or focus. - store.activity.update(reviewId, { - action: "update", - activityId: activityId!, - }); - store.activity.update(reviewId, { - action: "update", - activityId: activityId!, - focus: { description: "Reading the diff" }, - }); - store.activity.update(reviewId, { - action: "end", - activityId: activityId!, - }); - expect(await next()).toEqual(expected(false)); - store.activity.update(reviewId, { action: "begin" }); - expect(await next()).toEqual(expected(true)); - await vi.advanceTimersByTimeAsync(ACTIVITY_TTL_MS); - expect(await next()).toEqual(expected(false)); - } finally { - abort.abort(); - await stream.return(undefined); - await store.close(); - } - }, -); + return line?.kind === "list" ? line.reviews : line; + }; + + const expected = (working: boolean) => [ + expect.objectContaining({ reviewId, working }), + ]; + + try { + expect(await next()).toEqual(expected(false)); + + const { activityId } = store.activity.update(reviewId, { + action: "begin", + }); + + expect(await next()).toEqual(expected(true)); + // One line per transition: no repeat, nothing for renewals or focus. + store.activity.update(reviewId, { + action: "update", + activityId: activityId!, + }); + store.activity.update(reviewId, { + action: "update", + activityId: activityId!, + focus: { description: "Reading the diff" }, + }); + store.activity.update(reviewId, { + action: "end", + activityId: activityId!, + }); + expect(await next()).toEqual(expected(false)); + store.activity.update(reviewId, { action: "begin" }); + expect(await next()).toEqual(expected(true)); + await vi.advanceTimersByTimeAsync(ACTIVITY_TTL_MS); + expect(await next()).toEqual(expected(false)); + } finally { + abort.abort(); + await stream.return(undefined); + await store.close(); + } +}); diff --git a/packages/review/src/review-api/agent-client.test.ts b/packages/review/src/review-api/agent-client.test.ts index 26a40d330..475d3b7c0 100644 --- a/packages/review/src/review-api/agent-client.test.ts +++ b/packages/review/src/review-api/agent-client.test.ts @@ -1,14 +1,29 @@ import { randomUUID } from "node:crypto"; import { once } from "node:events"; -import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises"; import { createServer } from "node:http"; import { tmpdir } from "node:os"; import path from "node:path"; import { PassThrough, Readable, Writable } from "node:stream"; import type { JsonObject } from "@dev.fast/json"; +import { REVIEW_DESKTOP_DISCOVERY_VERSION } from "@dev.fast/review-protocol"; import { ListToolsResultSchema } from "@modelcontextprotocol/sdk/types.js"; -import { afterAll, afterEach, expect, it, vi } from "vitest"; +import { readReviewServerDiscovery } from "@review/server-discovery.js"; +import { + isolatedEnv, + sourceCli, + stopServersUnder, +} from "@review/server/background-server-test-utils.js"; +import { + afterAll, + afterEach, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; import { z } from "zod"; import { runReviewAgentCli } from "./agent-cli.js"; @@ -17,6 +32,7 @@ import { type AuthoringTool, callAuthoringTool } from "./agent-client.js"; import { ReviewApiClient } from "./client.js"; import { createReviewApi } from "./http.js"; import { serveReviewMcp } from "./mcp.js"; +import { callPublicTool, publicTool } from "./public-tools.js"; import { ReviewStore } from "./store.js"; const store = new ReviewStore(":memory:", { @@ -676,3 +692,144 @@ it("names the agent from the MCP handshake, falling back to the session environm connection.mockRestore(); } }); + +describe("with no Desktop running", () => { + let root: string; + let env: NodeJS.ProcessEnv; + + const home = () => env.DEV_REVIEW_HOME!; + + beforeEach(async () => { + root = await realpath(await mkdtemp(path.join(tmpdir(), "wb-connect-"))); + env = isolatedEnv(root); + }); + + afterEach(async () => { + await stopServersUnder(root); + await rm(root, { recursive: true, force: true }); + }); + + it("starts the default headless server for a tool call when no Desktop is installed", async () => { + const connected = await agentClient.connectReviewInstance( + env, + {}, + { desktopInstalled: () => false, cli: sourceCli }, + ); + + const tools = ( + await connected.client.read("/authoring") + ).map(publicTool); + + const list = tools.find((tool) => tool.name === "session_list")!; + + expect(await callPublicTool(connected.client, list, {})).toEqual( + expect.any(Array), + ); + expect(connected.instance).toBeUndefined(); + expect(await readReviewServerDiscovery(home())).toMatchObject({ + startedBy: "cli", + }); + }, 60_000); + + it.each([ + ["an explicit selection that is not running", "selected"], + ["a broken instance record", "broken"], + ["several running and none selected", "several"], + ] as const)( + "keeps the old diagnosis for %s, even with no Desktop installed", + async (_case, kind) => { + const desktops = await fakeDesktops(kind, env); + + try { + const connect = (installed: boolean) => + agentClient + .connectReviewInstance( + env, + {}, + { desktopInstalled: () => installed, cli: sourceCli }, + ) + .then( + () => "connected", + (error: Error) => error.message, + ); + + const before = await connect(true); + + expect(before).not.toBe("connected"); + expect(await connect(false)).toBe(before); + expect(await readReviewServerDiscovery(home())).toBeNull(); + } finally { + for (const server of desktops) server.close(); + } + }, + 30_000, + ); + + it("fails as before when a Desktop is installed but not running", async () => { + await expect( + agentClient.connectReviewInstance( + env, + {}, + { desktopInstalled: () => true, cli: sourceCli }, + ), + ).rejects.toThrow( + "Whiteboard `stable` is not running. Start it with `whiteboard app launch`, or pick another instance with `whiteboard instances`. No Whiteboard is running. For headless authoring, select a running server with --state-dir or DEV_REVIEW_SERVER_DIR.", + ); + expect(await readReviewServerDiscovery(home())).toBeNull(); + }); +}); + +async function fakeDesktops( + kind: "selected" | "broken" | "several", + env: NodeJS.ProcessEnv, +) { + const instances = path.join( + env.DEV_REVIEW_HOME!, + "review-desktop", + "instances", + ); + + await mkdir(instances, { recursive: true }); + + if (kind === "selected") { + env.DEV_REVIEW_INSTANCE = "preview"; + + return []; + } + + if (kind === "broken") { + await writeFile(path.join(instances, "stable.json"), "{"); + + return []; + } + + return Promise.all( + ["preview", "dev-other-0123456789ab"].map(async (key) => { + const instanceId = randomUUID(); + + const server = createServer((_request, response) => + response.end( + JSON.stringify({ ok: true, instanceId, desktopAttached: true }), + ), + ).listen(0, "127.0.0.1"); + + await once(server, "listening"); + const { port } = z.object({ port: z.number() }).parse(server.address()); + await writeFile( + path.join(instances, `${key}.json`), + JSON.stringify({ + version: REVIEW_DESKTOP_DISCOVERY_VERSION, + instanceId, + url: `http://127.0.0.1:${port}`, + appPid: process.pid, + serverPid: process.pid, + token: "token", + startedAt: 1, + key, + }), + ); + + return server; + }), + ); +} diff --git a/packages/review/src/review-api/agent-client.ts b/packages/review/src/review-api/agent-client.ts index 658b1b2e8..2184cdbdb 100644 --- a/packages/review/src/review-api/agent-client.ts +++ b/packages/review/src/review-api/agent-client.ts @@ -43,9 +43,15 @@ export async function connectReviewApi( return (await connectReviewInstance(env, headers)).client; } +export interface ConnectReviewOptions { + desktopInstalled?: () => boolean; + cli?: readonly string[]; +} + export async function connectReviewInstance( env = process.env, headers: Record = {}, + options: ConnectReviewOptions = {}, ): Promise { const request: ConstructorParameters[1] = ( url, @@ -77,6 +83,23 @@ export async function connectReviewInstance( const discovery = healthyReviewInstance(selection); if (!discovery) { + const { ensureServerWithoutDesktop } = + await import("@review/server/background-server.js"); + + const server = await ensureServerWithoutDesktop({ + selection, + env, + ...options, + }); + + if (server) + return { + client: new ReviewApiClient( + { serverUrl: server.url, token: server.token }, + request, + ), + }; + const unavailable = reviewInstanceUnavailable(selection); const message = `${unavailable.message} For headless authoring, select a running server with --state-dir or DEV_REVIEW_SERVER_DIR.`; diff --git a/packages/review/src/review-api/http.ts b/packages/review/src/review-api/http.ts index 6b597b9b3..96e4306b7 100644 --- a/packages/review/src/review-api/http.ts +++ b/packages/review/src/review-api/http.ts @@ -4,6 +4,8 @@ import { type JsonObject, isJsonObject } from "@dev.fast/json"; import { REVIEW_CLIENT_HEADER, REVIEW_CLIENT_REMOTE, + type ReviewRemoteLanguageEnvironment, + type ReviewStreamLine, type ReviewStructuralDiffEvent, } from "@dev.fast/review-protocol"; import { errorMessage } from "@dev.fast/trace-core"; @@ -75,6 +77,7 @@ import { inspectSnapshot, } from "./store.js"; import { listPinnedTraces, readStoredTrace } from "./traces.js"; +import type { WorkspaceStatus } from "./workspaces.js"; export interface AskHost { threads: AskThreads; @@ -155,6 +158,24 @@ export interface ReviewApiHooks { const remoteCaller = (context: Context) => context.req.header(REVIEW_CLIENT_HEADER) === REVIEW_CLIENT_REMOTE; +// Acquisition errors and preparation logs can quote local paths. +export const REMOTE_CHECKOUT_ISSUE = + "The checkout for language features is not available on the remote machine."; + +export const REMOTE_STRUCTURAL_DIFF_ERROR = + "The structural diff failed on the remote machine."; + +const workspaceFor = (context: Context, status: WorkspaceStatus) => + remoteCaller(context) + ? { + id: status.id, + commit: status.commit, + generation: status.generation, + state: status.state, + ...(status.issue && { issue: REMOTE_CHECKOUT_ISSUE }), + } + : status; + /** Both hosts mount this behind their token authentication. */ export function createReviewApi( store: ReviewStore, @@ -480,121 +501,123 @@ export function createReviewApi( ), ); }); - app.get("/watch", async (context) => { - const query = context.req.query("subscriptions"); + app.get("/watch", (context) => { + let input: unknown; + + try { + input = JSON.parse(context.req.query("subscriptions") ?? ""); + } catch { + throw new ReviewInputError("Invalid subscriptions."); + } + + const subscriptions = z + .array( + z.strictObject({ + reviewId: z.string().min(1).nullable(), + mode: coverageModeSchema, + }), + ) + .parse(input); + + const modes = new Set( + subscriptions.flatMap((item) => + item.reviewId === null ? [item.mode] : [], + ), + ); + + const reviewIds = new Set( + subscriptions.flatMap((item) => item.reviewId ?? []), + ); - if (query !== undefined) { - let input: unknown; + const dirtyModes = new Set(modes); + const dirtyReviews = new Set(reviewIds); + const sentLists = new Map(); + const readReviewLine = (reviewId: string): ReviewStreamLine => { try { - input = JSON.parse(query); - } catch { - throw new ReviewInputError("Invalid subscriptions."); + return { + kind: "review", + reviewId, + value: { + ...readReview(reviewId), + activity: store.activity.read(reviewId), + coverageRevision: data?.coverageRevision ?? 0, + }, + }; + } catch (error) { + return { + kind: "review", + reviewId, + error: + error instanceof ReviewInputError + ? error.message + : "Could not read review.", + }; } + }; - const subscriptions = z - .array( - z.strictObject({ - reviewId: z.string().min(1).nullable(), - mode: coverageModeSchema, - }), - ) - .parse(input); - - // Only entries whose review (or the catalog) changed are re-read and re-sent. - const dirty = new Set(subscriptions.keys()); - - const mark = (id: string | null) => { - let marked = false; - - subscriptions.forEach((item, index) => { - if (item.reviewId === id) { - dirty.add(index); - marked = true; - } - }); + const lines = () => { + const pending: string[] = []; - return marked; - }; + for (const mode of dirtyModes) { + const line = JSON.stringify({ + kind: "list", + mode, + reviews: catalog(mode), + } satisfies ReviewStreamLine); - return watch( - () => - subscriptions.map(({ reviewId, mode }, index) => { - if (!dirty.delete(index)) return null; - - try { - return { - value: - reviewId === null - ? catalog(mode) - : { - ...readReview(reviewId), - activity: store.activity.read(reviewId), - coverageRevision: data?.coverageRevision ?? 0, - }, - }; - } catch (error) { - return { - error: - error instanceof ReviewInputError - ? error.message - : "Could not read review.", - }; - } - }), - (notify) => { - const stopRefresh = store.watchWorktrees(); - - const stops = [ - stopRefresh, - data?.subscribeCoverage(() => { - subscriptions.forEach((item, index) => { - if (item.reviewId !== null) dirty.add(index); - }); - - // Coverage never changes the catalog; don't send an all-null line. - if (dirty.size > 0) notify(); - }) ?? (() => {}), - store.subscribe((result) => { - if (mark(result.reviewId)) notify(); - }), - store.activity.subscribe((id) => { - if (mark(id)) notify(); - }), - store.activity.subscribeWorking(() => { - if (mark(null)) notify(); - }), - shared?.subscribe(() => { - if (mark(null)) notify(); - }) ?? (() => {}), - store.subscribeCatalog(() => { - if (mark(null)) notify(); - }), - ]; - - return () => stops.forEach((stop) => stop()); - }, - // A missing review is an {error} entry here, never a 404. - () => {}, - ); - } + if (sentLists.get(mode) !== line) pending.push(line); + sentLists.set(mode, line); + } - await ensureScratchpad(); + for (const reviewId of dirtyReviews) + pending.push(JSON.stringify(readReviewLine(reviewId))); - return watch( - () => catalog(coverageModeSchema.parse(context.req.query("mode"))), - (notify) => { - const local = store.subscribeCatalog(notify); - const activity = store.activity.subscribeWorking(notify); - const imported = shared?.subscribe(notify); - - return () => { - local(); - activity(); - imported?.(); - }; - }, - ); + dirtyModes.clear(); + dirtyReviews.clear(); + + return pending; + }; + + const markReview = (reviewId: string) => { + if (!reviewIds.has(reviewId)) return false; + dirtyReviews.add(reviewId); + + return true; + }; + + const markLists = () => { + modes.forEach((mode) => dirtyModes.add(mode)); + + return modes.size > 0; + }; + + return watch(lines, (notify) => { + const stops = [ + store.watchWorktrees(), + data?.subscribeCoverage(() => { + reviewIds.forEach((id) => dirtyReviews.add(id)); + notify(); + }) ?? (() => {}), + store.subscribe((result) => { + if (markReview(result.reviewId)) notify(); + }), + store.activity.subscribe((id) => { + if (markReview(id)) notify(); + }), + store.activity.subscribeWorking(() => { + if (markLists()) notify(); + }), + shared?.subscribe(() => { + if (markLists()) notify(); + }) ?? (() => {}), + store.subscribeCatalog(() => { + if (markLists()) notify(); + }), + ]; + + return () => stops.forEach((stop) => stop()); + }); }); /** Show a review in Desktop and start preparing its pinned checkouts. */ @@ -655,41 +678,6 @@ export function createReviewApi( return context.json({ ok: true, ...(await openReview(readReview(id))) }); }); - app.get("/:id/watch", (context) => { - const id = context.req.param("id"); - - // Activity changes every renewal; reload the document only when it changed. - let document: Snapshot | undefined; - - return watch( - () => ({ - ...(document ??= readReview(id)), - activity: isShared(id) - ? { workingCount: 0, expiresAt: null } - : store.activity.read(id), - }), - (notify) => { - const stopRefresh = store.watchWorktrees(); - - const stopDocument = store.subscribe((result) => { - if (result.reviewId === id) { - document = undefined; - notify(); - } - }); - - const stopActivity = store.activity.subscribe((changed) => { - if (changed === id) notify(); - }); - - return () => { - stopRefresh(); - stopDocument(); - stopActivity(); - }; - }, - ); - }); if (data) { const traceQuery = readQuerySchemas.maps.extend({ @@ -739,12 +727,6 @@ export function createReviewApi( return context.json(result); }); app.post("/:id/navigator", async (context) => { - if (remoteCaller(context)) - throw new ReviewInputError( - "Source windows are not available for a review on another machine.", - 409, - ); - const input = readQuerySchemas.file .extend({ side: z.enum(["base", "head"]).default("head"), @@ -753,15 +735,19 @@ export function createReviewApi( }) .parse(context.req.query()); + const answer = await data.navigatorWorkspace( + readReview(context.req.param("id"), input.version), + { + ...input, + empty: input.empty === "true", + anchor: queryAnchor(input), + }, + ); + return context.json( - await data.navigatorWorkspace( - readReview(context.req.param("id"), input.version), - { - ...input, - empty: input.empty === "true", - anchor: queryAnchor(input), - }, - ), + remoteCaller(context) && input.empty === "true" + ? { workspacePath: answer.workspacePath, emptySide: true } + : answer, ); }); app.get("/:id/tree", async (context) => { @@ -855,32 +841,34 @@ export function createReviewApi( queryAnchor(input), ); - return context.json( - remoteCaller(context) - ? { - // A live checkout's identity names its path; keep only its equality. - identity: createHash("sha256") - .update(environment.identity) - .digest("hex"), - // An acquisition error can quote local paths. - ...(environment.issue && { - issue: - "The checkout for language features is not available on the remote machine.", - }), - } - : environment, - ); + if (!remoteCaller(context)) return context.json(environment); + + const remote: ReviewRemoteLanguageEnvironment = { + remoteRootPath: environment.rootPath ?? null, + // A live checkout's identity names its path; keep only its equality. + identity: createHash("sha256") + .update(environment.identity) + .digest("hex"), + serverId: store.serverId(), + ...(environment.issue && { issue: REMOTE_CHECKOUT_ISSUE }), + }; + + return context.json(remote); }); app.post("/:id/environment", async (context) => { const input = z .strictObject({ retry: z.boolean().optional() }) .parse(await readBoundedRequestJson(context.req.raw)); + const issues = await data.environmentIssues( + readReview(context.req.param("id")), + input.retry, + ); + return context.json({ - issues: await data.environmentIssues( - readReview(context.req.param("id")), - input.retry, - ), + issues: remoteCaller(context) + ? issues.map(({ side }) => ({ side, message: REMOTE_CHECKOUT_ISSUE })) + : issues, }); }); app.post("/workspace-cleanup", async (context) => { @@ -891,18 +879,29 @@ export function createReviewApi( if (input.workspaceId) await data.workspaces.retryCleanup(input.workspaceId); - return context.json({ failures: data.workspaces.failures() }); + return context.json({ + failures: data.workspaces + .failures() + .map((status) => workspaceFor(context, status)), + }); }); app.get("/:id/workspaces", (context) => { readReview(context.req.param("id")); - return context.json(data.workspaces.list(context.req.param("id"))); + return context.json( + data.workspaces + .list(context.req.param("id")) + .map((status) => workspaceFor(context, status)), + ); }); app.post("/:id/workspaces/:workspaceId/retry", async (context) => { return context.json( - await data.workspaces.retry( - context.req.param("id"), - context.req.param("workspaceId"), + workspaceFor( + context, + await data.workspaces.retry( + context.req.param("id"), + context.req.param("workspaceId"), + ), ), ); }); @@ -980,7 +979,10 @@ export function createReviewApi( } catch (error) { send({ type: "error", - message: error instanceof Error ? error.message : String(error), + // Checkout and diffr errors can quote local paths. + message: remoteCaller(context) + ? REMOTE_STRUCTURAL_DIFF_ERROR + : errorMessage(error), }); } finally { if (!abort.signal.aborted) controller.close(); @@ -1847,35 +1849,23 @@ async function locateRepositories( }; } -/** Send committed state, coalescing updates when the reader falls behind. */ -function watch( - read: () => T, +function watch( + lines: () => string[], subscribe: (notify: () => void) => () => void, - probe: () => void = read, ) { - probe(); // Return a normal 404 before opening the response. let stop = () => {}; - let dirty = true; const encoder = new TextEncoder(); const send = (controller: ReadableStreamDefaultController) => { - if ( - !dirty || - controller.desiredSize === null || - controller.desiredSize <= 0 - ) - return; + if (controller.desiredSize === null || controller.desiredSize <= 0) return; try { - const line = JSON.stringify(read()) + "\n"; + const pending = lines(); - // enqueue can pull synchronously; clear first so it doesn't resend. - dirty = false; - controller.enqueue(encoder.encode(line)); + if (pending.length) + controller.enqueue(encoder.encode(pending.join("\n") + "\n")); } catch (error) { - // A review can be deleted while this stream is open. Do not throw into - // the already-committed writer; close this reader and unsubscribe it. stop(); controller.error(error); } @@ -1883,9 +1873,15 @@ function watch( const body = new ReadableStream({ start(controller) { + let scheduled = false; + stop = subscribe(() => { - dirty = true; - send(controller); + if (scheduled) return; + scheduled = true; + queueMicrotask(() => { + scheduled = false; + send(controller); + }); }); send(controller); }, diff --git a/packages/review/src/review-api/live-tabs.test.ts b/packages/review/src/review-api/live-tabs.test.ts index fc7c8acab..5632c70c7 100644 --- a/packages/review/src/review-api/live-tabs.test.ts +++ b/packages/review/src/review-api/live-tabs.test.ts @@ -165,9 +165,15 @@ it("finishes an in-flight render before another tab replaces the shared stream", start(controller) { controller.enqueue( new TextEncoder().encode( - JSON.stringify( - subscriptions.map(() => ({ value: streams.length })), - ) + "\n", + subscriptions + .map(({ reviewId }: { reviewId: string }) => + JSON.stringify({ + kind: "review", + reviewId, + value: streams.length, + }), + ) + .join("\n") + "\n", ), ); }, @@ -190,6 +196,7 @@ it("finishes an in-flight render before another tab replaces the shared stream", }); const rendered: unknown[] = []; + const disconnects: unknown[] = []; let started = false; const first = client.follow( @@ -200,9 +207,7 @@ it("finishes an in-flight render before another tab replaces the shared stream", await rendering; rendered.push(value); }, - (error) => { - throw error; - }, + (error) => disconnects.push(error), ); let second: Promise | undefined; @@ -213,14 +218,13 @@ it("finishes an in-flight render before another tab replaces the shared stream", "b", b.signal, () => {}, - (error) => { - throw error; - }, + (error) => disconnects.push(error), ); await new Promise((resolve) => setImmediate(resolve)); expect(streams).toHaveLength(1); release(); await vi.waitFor(() => expect(rendered).toEqual([1, 2])); + expect(disconnects).toEqual([]); } finally { release(); a.abort(); @@ -228,3 +232,77 @@ it("finishes an in-flight render before another tab replaces the shared stream", await Promise.all([first, second]); } }); + +it("delivers each line only to the listeners it is about", async () => { + const store = new ReviewStore(":memory:", { + validatePins: async () => {}, + validateSource: async () => {}, + validateResource: async () => {}, + }); + + const app = createReviewApi(store); + + const client = new ReviewApiClient( + { serverUrl: "http://review.test", token: "token" }, + async (url, init) => + app.request(url.replace("http://review.test/reviews-api", ""), init), + ); + + const command = (operation: Operation) => + store.execute({ operation }); + + const target = { + kind: "commits" as const, + repositoryId: "repo", + base: "base", + head: "head", + }; + + const a = (await command({ type: "create", title: "A", target })).reviewId; + const b = (await command({ type: "create", title: "B", target })).reviewId; + const abort = new AbortController(); + + const seen = { + a: [] as { reviewId: string; title: string }[], + b: [] as { reviewId: string; title: string }[], + }; + + const disconnects: unknown[] = []; + + const follow = (key: "a" | "b", id: string) => + client.follow<{ reviewId: string; title: string }>( + id, + abort.signal, + (value) => { + seen[key].push(value); + }, + (error) => disconnects.push(error), + ); + + const following = [follow("a", a)]; + + try { + await vi.waitFor(() => expect(seen.a).toHaveLength(1)); + following.push(follow("b", b)); + await vi.waitFor(() => expect(seen.b).toHaveLength(1)); + await vi.waitFor(() => expect(seen.a).toHaveLength(2)); + const delivered = seen.a.length; + + for (const title of ["B1", "B2", "B3"]) { + await command({ type: "rename", reviewId: b, title }); + await vi.waitFor(() => expect(seen.b.at(-1)?.title).toBe(title)); + } + + expect(seen.a).toHaveLength(delivered); + await command({ type: "rename", reviewId: a, title: "A1" }); + await vi.waitFor(() => expect(seen.a.at(-1)?.title).toBe("A1")); + expect(seen.a).toHaveLength(delivered + 1); + expect(seen.a.every((value) => value.reviewId === a)).toBe(true); + expect(seen.b.every((value) => value.reviewId === b)).toBe(true); + expect(disconnects).toEqual([]); + } finally { + abort.abort(); + await Promise.all(following); + await store.close(); + } +}); diff --git a/packages/review/src/review-api/local-data.test.ts b/packages/review/src/review-api/local-data.test.ts index c2322da97..d6d9b54df 100644 --- a/packages/review/src/review-api/local-data.test.ts +++ b/packages/review/src/review-api/local-data.test.ts @@ -17,12 +17,19 @@ import { tmpdir } from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { + type ClientConnection, + agent, + methods, +} from "@agentclientprotocol/sdk"; import { setLocalVcsCommandObserver } from "@dev.fast/local-vcs"; import { type JsonValue, REVIEW_CLIENT_HEADER, REVIEW_CLIENT_REMOTE, } from "@dev.fast/review-protocol"; +import { type AskAgentLauncher, detectAskAgents } from "@review/ask/agents.js"; +import { AskThreads } from "@review/ask/threads.js"; import { rangeAnchor } from "@review/lens-selection"; import { createGlobalReviewServer } from "@review/server/desktop-server.js"; import { Hono } from "hono"; @@ -978,7 +985,7 @@ it("serves a historical version's file at the pins that version was saved with", }); }); -it("gives a remote caller a commit review's language context without its checkout path", async () => { +it("gives a remote caller a commit review's checkout as remoteRootPath, with the server's id", async () => { const { reviewId } = await local.store.execute( command({ type: "create", @@ -1000,8 +1007,11 @@ it("gives a remote caller a commit review's language context without its checkou expect(ownMachine.rootPath).toEqual(expect.any(String)); const remote = await read({ [REVIEW_CLIENT_HEADER]: REVIEW_CLIENT_REMOTE }); - expect(remote).toEqual({ identity: expect.any(String) }); - expect(JSON.stringify(remote)).not.toContain(directory); + expect(remote).toEqual({ + remoteRootPath: ownMachine.rootPath, + identity: expect.stringMatching(/^[0-9a-f]{64}$/), + serverId: local.store.serverId(), + }); expect(await read({ [REVIEW_CLIENT_HEADER]: "local" })).toEqual(ownMachine); }); @@ -1031,12 +1041,185 @@ it("gives a remote caller a fixed issue when a commit review's checkout fails", const remote = await read({ [REVIEW_CLIENT_HEADER]: REVIEW_CLIENT_REMOTE }); expect(JSON.stringify(remote)).not.toContain(directory); expect(remote).toEqual({ + remoteRootPath: null, identity: expect.any(String), + serverId: local.store.serverId(), issue: "The checkout for language features is not available on the remote machine.", }); }); +it("gives a remote caller Ask answers with no local-path fields and checkout-relative mentions", async () => { + const { reviewId } = await local.store.execute( + command({ + type: "create", + title: "Ask", + target: { kind: "commits", ...pins }, + }), + ); + + const { rootPath } = await local.data.agentCheckout( + local.store.read(reviewId), + ); + + const fake = agent({ name: "fake" }) + .onRequest(methods.agent.initialize, () => ({ + protocolVersion: 1, + agentCapabilities: { loadSession: true }, + authMethods: [], + })) + .onRequest(methods.agent.session.new, () => ({ sessionId: "session" })) + .onRequest(methods.agent.session.load, () => ({})) + .onRequest(methods.agent.session.prompt, () => ({ + stopReason: "end_turn" as const, + })); + + const launch: AskAgentLauncher = async () => { + let connection: ClientConnection | undefined; + + return { + connect: (client) => (connection = client.connect(fake)), + diagnostics: () => "", + stop: () => connection?.close(), + }; + }; + + const threads = new AskThreads(launch); + + const app = createReviewApi( + local.store, + local.data, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + { threads, agents: () => detectAskAgents({}) }, + ); + + const remote = { [REVIEW_CLIENT_HEADER]: REVIEW_CLIENT_REMOTE }; + + const request = async (route: string, body?: JsonValue) => { + const response = await app.request(`/${reviewId}/${route}`, { + headers: { ...remote, "content-type": "application/json" }, + ...(body !== undefined && { + method: "POST", + body: JSON.stringify(body), + }), + }); + + expect(response.status).toBe(200); + + return response; + }; + + const read = async (route: string, body?: JsonValue) => + (await request(route, body)).json(); + + const snapshot = async (threadId: string) => { + const reader = (await request(`ask/${threadId}/watch`)) + .body!.pipeThrough(new TextDecoderStream()) + .getReader(); + + const { value = "" } = await reader.read(); + await reader.cancel(); + + return JSON.parse(value.split("\n")[0]!).snapshot; + }; + + const noLocalPaths = (answer: JsonValue) => { + for (const key of [ + "localPath", + "rootPath", + "workspacePath", + "filePath", + "localRoot", + ]) + expect(JSON.stringify(answer)).not.toContain(`"${key}"`); + }; + + const agents = await read("ask/agents"); + expect(agents.agents.length).toBeGreaterThan(0); + noLocalPaths(agents); + + const { threadId } = await read("ask", { + agent: "claude", + question: { text: "Why?" }, + selection: { target: { kind: "text", quote: "value" }, title: "value" }, + }); + + const asked = await snapshot(threadId); + expect(asked.cwd).toBe(rootPath); + noLocalPaths(asked); + + await vi.waitFor(async () => + expect((await read("ask/threads")).threads).toHaveLength(1), + ); + noLocalPaths(await read("ask/threads")); + + for (const route of [ + "ask/mentions?query=example", + `ask/mentions?query=example&thread=${threadId}`, + ]) { + const mentions = await read(route); + expect(mentions.paths).toContain(source.file); + expect(mentions.paths.every((file: string) => !file.startsWith("/"))).toBe( + true, + ); + noLocalPaths(mentions); + } + + threads.close(threadId); + await read(`ask/${threadId}/open`, {}); + const reopened = await snapshot(threadId); + expect(reopened.cwd).toBe(rootPath); + noLocalPaths(reopened); + threads.close(threadId); +}); + +it("answers a remote caller's navigator with the host paths a local caller gets", async () => { + const { reviewId } = await local.store.execute( + command({ + type: "create", + title: "Remote navigator", + target: { kind: "commits", ...pins }, + }), + ); + + const app = createReviewApi(local.store, local.data); + + const open = async ( + query: Record, + headers: Record = {}, + ) => { + const response = await app.request( + `/${reviewId}/navigator?${new URLSearchParams(query)}`, + { method: "POST", headers }, + ); + + expect(response.status).toBe(200); + + return response.json(); + }; + + const remote = { [REVIEW_CLIENT_HEADER]: REVIEW_CLIENT_REMOTE }; + const file = { version: "0", side: "base", file: source.file }; + const answer = await open(file, remote); + const { folders } = JSON.parse(readFileSync(answer.workspacePath, "utf8")); + expect(answer).toEqual(await open(file)); + expect(answer.filePath).toBe(path.join(folders[0].path, source.file)); + expect(readFileSync(answer.filePath, "utf8")).toContain("value = 1"); + + const added = { ...file, file: "added.ts", empty: "true" }; + expect(await open(added, remote)).toEqual({ + workspacePath: answer.workspacePath, + emptySide: true, + }); + expect((await open(added)).filePath).toEqual(expect.any(String)); +}); + it("opens a stable native workspace on the Review's pinned checkout at the selected version", async () => { const { reviewId } = await local.store.execute( command({ diff --git a/packages/review/src/review-api/origin.ts b/packages/review/src/review-api/origin.ts index 39164dd92..564fa6425 100644 --- a/packages/review/src/review-api/origin.ts +++ b/packages/review/src/review-api/origin.ts @@ -2,10 +2,13 @@ import { z } from "zod"; import type { Snapshot } from "./store.js"; +export const PULL_REQUEST_URL = + /^https:\/\/[a-z0-9.-]+\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+\/pull\/[1-9]\d*$/; + export const pullRequestUrl = z .string() .regex( - /^https:\/\/[a-z0-9.-]+\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+\/pull\/[1-9]\d*$/, + PULL_REQUEST_URL, "Use a canonical GitHub PR URL: https://github.com/owner/repository/pull/123, or the same path on a GitHub Enterprise host.", ) .refine( diff --git a/packages/review/src/review-api/review-api.test.ts b/packages/review/src/review-api/review-api.test.ts index 64eb82a94..f80049ea3 100644 --- a/packages/review/src/review-api/review-api.test.ts +++ b/packages/review/src/review-api/review-api.test.ts @@ -1863,20 +1863,25 @@ it("serves the experiment through the real desktop HTTP server and existing auth }), ).rejects.toThrow(/lines count from 1/); const abort = new AbortController(); - const catalog = client.watch(null, abort.signal); - expect(reviewsOnly((await catalog.next()).value)).toMatchObject([ - { reviewId, dismissedAt: null }, - ]); + const catalog = client.watch([{ reviewId: null }], abort.signal); + + const nextList = async () => { + const line = (await catalog.next()).value; + + return reviewsOnly(line?.kind === "list" ? line.reviews : []); + }; + + expect(await nextList()).toMatchObject([{ reviewId, dismissedAt: null }]); await post({ type: "attention", reviewId, action: "dismiss" }); - expect(reviewsOnly((await catalog.next()).value)).toMatchObject([ + expect(await nextList()).toMatchObject([ { reviewId, dismissedAt: expect.any(String) }, ]); await catalog.return(undefined); - const live = client.watch(reviewId, abort.signal); + const live = client.watch([{ reviewId }], abort.signal); expect((await live.next()).value).toMatchObject({ + kind: "review", reviewId, - version: 0, - document: [], + value: { reviewId, version: 0, document: [] }, }); expect( ( @@ -1889,13 +1894,19 @@ it("serves the experiment through the real desktop HTTP server and existing auth ).toBe(200); const read = await fetch(url + "/" + reviewId + "?full=true", { headers }); expect((await live.next()).value).toMatchObject({ - version: 1, - document: [{ type: "sequence" }], + value: { version: 1, document: [{ type: "sequence" }] }, }); await live.return(undefined); abort.abort(); - const reconnect = client.watch(reviewId, new AbortController().signal); - expect((await reconnect.next()).value).toMatchObject({ version: 1 }); + + const reconnect = client.watch( + [{ reviewId }], + new AbortController().signal, + ); + + expect((await reconnect.next()).value).toMatchObject({ + value: { version: 1 }, + }); await reconnect.return(undefined); expect(await read.json()).toMatchObject({ title: "HTTP review", @@ -1922,15 +1933,15 @@ it("serves the experiment through the real desktop HTTP server and existing auth }) ).status, ).toBe(413); - const watching = client.watch(reviewId, new AbortController().signal); + const watching = client.watch([{ reviewId }], new AbortController().signal); await watching.next(); - - await Promise.all([ - expect(watching.next()).rejects.toThrow(Error), - post({ type: "delete", reviewId }).then((response) => { - expect(response.status).toBe(200); - }), - ]); + expect((await post({ type: "delete", reviewId })).status).toBe(200); + expect((await watching.next()).value).toEqual({ + kind: "review", + reviewId, + error: expect.stringMatching(/not found/i), + }); + await watching.return(undefined); expect( (await fetch(`${url}/${reviewId}?full=true`, { headers })).status, ).toBe(404); @@ -2770,7 +2781,7 @@ it("returns pending progress without waiting for coverage and signals completion new TextDecoder().decode((await reader.read()).value), ); - expect(initial[0].value.coverageRevision).toBeGreaterThan(0); + expect(initial.value.coverageRevision).toBeGreaterThan(0); await reader.cancel(); const ready = await api.request(route); expect(ready.status).toBe(200); diff --git a/packages/review/src/review-api/store.ts b/packages/review/src/review-api/store.ts index acb64293b..7bfdb4f22 100644 --- a/packages/review/src/review-api/store.ts +++ b/packages/review/src/review-api/store.ts @@ -1433,6 +1433,12 @@ export class ReviewStore { .all() .map((row) => String(row.review_id)); } + reviewIds(): string[] { + return this.db + .prepare("SELECT id FROM reviews") + .all() + .map((row) => String(row.id)); + } has(reviewId: string): boolean { return ( this.db.prepare("SELECT 1 FROM reviews WHERE id=?").get(reviewId) !== diff --git a/packages/review/src/review-api/watch.test.ts b/packages/review/src/review-api/watch.test.ts new file mode 100644 index 000000000..e80960993 --- /dev/null +++ b/packages/review/src/review-api/watch.test.ts @@ -0,0 +1,287 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { type IncomingMessage, createServer, get } from "node:http"; +import type { AddressInfo } from "node:net"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import type { ReviewStreamLine } from "@dev.fast/review-protocol"; +import { getRequestListener } from "@hono/node-server"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; + +import { createReviewApi } from "./http.js"; +import { LocalReviewData } from "./local-data.js"; +import { ReviewStore, SCRATCHPAD_ID } from "./store.js"; + +const pins = { repositoryId: "repo", base: "base", head: "head" }; + +let directory: string, store: ReviewStore; + +const cleanup: (() => void | Promise)[] = []; + +beforeEach(() => { + directory = mkdtempSync(path.join(tmpdir(), "review-watch-")); + store = new ReviewStore(path.join(directory, "review.db"), { + validatePins: async () => {}, + validateSource: async () => {}, + validateResource: async () => {}, + }); +}); + +afterEach(async () => { + for (const step of cleanup.splice(0).reverse()) await step(); + await store.close(); + rmSync(directory, { recursive: true, force: true }); +}); + +const command = (operation: Operation) => + store.execute({ operation }); + +const create = async (title: string) => + ( + await command({ + type: "create", + title, + target: { kind: "commits", ...pins }, + }) + ).reviewId; + +const rename = (reviewId: string, title: string) => + command({ type: "rename", reviewId, title }); + +async function open( + subscriptions: { reviewId: string | null; mode?: string }[], + data?: LocalReviewData, +) { + const app = createReviewApi(store, data); + const server = createServer(getRequestListener(app.fetch)); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address() as AddressInfo; + + const response = await new Promise((resolve, reject) => + get( + `http://127.0.0.1:${port}/watch?subscriptions=${encodeURIComponent(JSON.stringify(subscriptions))}`, + resolve, + ).on("error", reject), + ); + + cleanup.push( + () => + new Promise((resolve) => { + response.destroy(); + server.closeAllConnections(); + server.close(() => resolve()); + }), + ); + expect(response.statusCode).toBe(200); + response.setEncoding("utf8"); + const lines: ReviewStreamLine[] = []; + let pending = ""; + response.on("data", (chunk: string) => { + pending += chunk; + let end: number; + + while ((end = pending.indexOf("\n")) !== -1) { + lines.push(JSON.parse(pending.slice(0, end))); + pending = pending.slice(end + 1); + } + }); + + const take = async (count: number) => { + await vi.waitFor(() => expect(lines.length).toBeGreaterThanOrEqual(count)); + + return lines.splice(0, count); + }; + + return { response, lines, take }; +} + +const label = (line: ReviewStreamLine) => + line.kind === "list" + ? `list:${line.mode}` + : `review:${line.reviewId}:${"error" in line ? "error" : "value"}`; + +it("sends one line per subscribed item on connect, however often it is subscribed", async () => { + const a = await create("A"); + const b = await create("B"); + + const stream = await open([ + { reviewId: a }, + { reviewId: null, mode: "structural" }, + { reviewId: b, mode: "textual" }, + { reviewId: a, mode: "textual" }, + { reviewId: null }, + ]); + + const first = await stream.take(3); + expect(first.map(label).sort()).toEqual( + ["list:structural", `review:${a}:value`, `review:${b}:value`].sort(), + ); + expect(first.find((line) => line.kind === "list")).toMatchObject({ + reviews: expect.arrayContaining( + ["A", "B"].map((title) => expect.objectContaining({ title })), + ), + }); + expect( + first.find((line) => line.kind === "review" && line.reviewId === a), + ).toMatchObject({ + value: { reviewId: a, title: "A", activity: { workingCount: 0 } }, + }); + + store.activity.update(b, { action: "begin" }); + expect((await stream.take(2)).map(label)).toEqual([ + "list:structural", + `review:${b}:value`, + ]); +}); + +it("sends a list line only for what changed in the list", async () => { + const a = await create("A"); + const b = await create("B"); + const data = new LocalReviewData(store); + cleanup.push(() => data.close()); + vi.spyOn(data, "resolveSource").mockImplementation(async (snapshot) => ({ + snapshot, + pins: snapshot.pins!, + })); + vi.spyOn(data, "changes").mockImplementation((async ( + _pins: typeof pins, + file?: string, + ) => (file ? "" : [])) as typeof data.changes); + + const stream = await open( + [{ reviewId: null }, { reviewId: a }, { reviewId: b }], + data, + ); + + await stream.take(3); + + store.activity.update(a, { action: "begin" }); + expect((await stream.take(2)).map(label)).toEqual([ + "list:structural", + `review:${a}:value`, + ]); + + await data.coverage(a, pins, "textual"); + expect((await stream.take(2)).map(label).sort()).toEqual( + [`review:${a}:value`, `review:${b}:value`].sort(), + ); + store.invalidateCatalog(); + + await command({ + type: "edit", + reviewId: b, + edit: { type: "insert", content: { type: "markdown", markdown: "Hi" } }, + }); + expect((await stream.take(2)).map(label).sort()).toEqual( + ["list:structural", `review:${b}:value`].sort(), + ); + + await rename(b, "B renamed"); + const renamed = await stream.take(2); + expect(renamed.map(label).sort()).toEqual( + ["list:structural", `review:${b}:value`].sort(), + ); + expect(renamed.find((line) => line.kind === "list")).toMatchObject({ + reviews: expect.arrayContaining([ + expect.objectContaining({ title: "B renamed" }), + ]), + }); + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(stream.lines).toEqual([]); +}); + +it("reports a missing or deleted review as an error line and keeps the stream open", async () => { + const a = await create("A"); + const b = await create("B"); + + const stream = await open([ + { reviewId: a }, + { reviewId: b }, + { reviewId: SCRATCHPAD_ID }, + ]); + + const first = await stream.take(3); + expect(first.map(label).sort()).toEqual( + [ + `review:${a}:value`, + `review:${b}:value`, + `review:${SCRATCHPAD_ID}:error`, + ].sort(), + ); + expect( + first.find( + (line) => line.kind === "review" && line.reviewId === SCRATCHPAD_ID, + ), + ).toMatchObject({ error: expect.stringMatching(/not found/i) }); + + await command({ type: "delete", reviewId: a }); + expect(await stream.take(1)).toEqual([ + { kind: "review", reviewId: a, error: expect.stringMatching(/not found/i) }, + ]); + + await rename(b, "B still live"); + expect(await stream.take(1)).toMatchObject([ + { kind: "review", reviewId: b, value: { title: "B still live" } }, + ]); + + await store.ensureScratchpad(); + expect(await stream.take(1)).toMatchObject([ + { + kind: "review", + reviewId: SCRATCHPAD_ID, + value: { reviewId: SCRATCHPAD_ID, kind: "scratchpad" }, + }, + ]); +}); + +it("gives a reader that stops reading the latest state once, not a backlog", async () => { + const a = await create("A"); + await command({ + type: "edit", + reviewId: a, + edit: { + type: "insert", + content: { type: "markdown", markdown: "x".repeat(1024 * 1024) }, + }, + }); + + const stream = await open([{ reviewId: a }]); + await stream.take(1); + stream.response.pause(); + const updates = 40; + + const { activityId } = store.activity.update(a, { + action: "begin", + focus: { description: "Step 1" }, + }); + + for (let index = 2; index <= updates; index++) { + store.activity.update(a, { + action: "update", + activityId: activityId!, + focus: { description: `Step ${index}` }, + }); + await new Promise((resolve) => setImmediate(resolve)); + } + + stream.response.resume(); + const steps: string[] = []; + + await vi.waitFor( + () => { + for (const line of stream.lines.splice(0)) + if (line.kind === "review" && "value" in line) + steps.push( + ( + line.value.activity as { + activities?: { focus?: { description: string } }[]; + } + ).activities?.[0]?.focus?.description ?? "", + ); + expect(steps.at(-1)).toBe(`Step ${updates}`); + }, + { timeout: 10_000 }, + ); + expect(steps.filter((step) => step === `Step ${updates}`)).toHaveLength(1); + expect(steps.length).toBeLessThan(updates / 2); +}); diff --git a/packages/review/src/review-api/workspaces.test.ts b/packages/review/src/review-api/workspaces.test.ts index 912dfcc55..2b86364a5 100644 --- a/packages/review/src/review-api/workspaces.test.ts +++ b/packages/review/src/review-api/workspaces.test.ts @@ -14,6 +14,7 @@ import { import { tmpdir } from "node:os"; import path from "node:path"; +import { reviewManagedCheckoutRoot } from "@review/review-checkout-paths.js"; import { runPrepareCommand } from "@review/review-prepare.js"; import { afterEach, beforeEach, expect, it, vi } from "vitest"; import type { z } from "zod"; @@ -110,11 +111,11 @@ it("lets a second Desktop share the profile without preparing a review the first preparing, ); await expect(second.data.workspaces.remove(reviewId)).rejects.toThrow( - /Another Desktop/, + /Another Whiteboard server/, ); await expect( second.data.workspaces.retry(reviewId, preparing.id), - ).rejects.toThrow(/Another Desktop/); + ).rejects.toThrow(/Another Whiteboard server/); await local.data.close(); await local.store.close(); local = second; @@ -311,7 +312,7 @@ it("claims unowned workspaces before removing them", async () => { await local.data.close(); const removal = second.data.workspaces.remove(reviewId); await expect(third.data.workspaces.remove(reviewId)).rejects.toThrow( - /Another Desktop/, + /Another Whiteboard server/, ); await removal; expect(third.data.workspaces.list(reviewId)).toEqual([]); @@ -398,6 +399,28 @@ it("dismissing a review frees only its own managed checkout, leaving a user's si expect(git("status", "--porcelain")).toBe(mainStatusBefore); }); +it("removes a deleted review's whole checkout directory", async () => { + const { workspacePath } = await local.data.navigatorWorkspace( + local.store.read(reviewId), + ); + + await local.data.workspaces.open(reviewId, pins); + + const managed = reviewManagedCheckoutRoot( + path.join(repository, ".git"), + reviewId, + ); + + expect(existsSync(workspacePath)).toBe(true); + await command({ type: "delete", reviewId }); + await local.data.workspaces.idle(); + expect(local.data.workspaces.list(reviewId)).toEqual([]); + expect(existsSync(managed)).toBe(false); + expect( + git("worktree", "list", "--porcelain").match(/^worktree /gm), + ).toHaveLength(1); +}); + it("removes checkouts left by reviews dismissed while Desktop was closed", async () => { const { rootPath } = await local.data.workspaces.source( reviewId, diff --git a/packages/review/src/review-api/workspaces.ts b/packages/review/src/review-api/workspaces.ts index 2de13857b..c4978374e 100644 --- a/packages/review/src/review-api/workspaces.ts +++ b/packages/review/src/review-api/workspaces.ts @@ -48,9 +48,8 @@ interface Environment extends WorkspaceStatus { role: "base" | "head"; } -// A lease lasts until the owning Desktop exits; closing the review does not release it. const OWNED_ELSEWHERE = - "Another Desktop owns this review's language workspaces. Quit that Desktop, then retry."; + "Another Whiteboard server is managing this review's language workspaces. Retry once it stops."; /** Local lifecycle only: source and authored history never depend on preparation. * Status/queue/process lifecycle follows #334, retaining the legacy prepare config @@ -104,9 +103,27 @@ export class ReviewWorkspaces { this.stop = store.subscribeCatalog(() => { this.collect(); this.releaseDismissed(); + this.releaseDeleted(); }); this.collect(); this.releaseDismissed(); + this.releaseDeleted(); + } + + private saved?: Set; + + private releaseDeleted() { + const saved = new Set(this.store.reviewIds()); + const deleted = [...(this.saved ?? [])].filter((id) => !saved.has(id)); + this.saved = saved; + + if (!deleted.length) return; + + const repositories = this.repositoryDirs(); + repositories.catch(() => undefined); + + for (const reviewId of deleted) + this.release(reviewId, repositories, () => !this.hasReview(reviewId)); } // Unset until the first scan, so startup also frees reviews dismissed @@ -131,7 +148,13 @@ export class ReviewWorkspaces { // Each release awaits this only after earlier cleanup and logs its failure. repositories.catch(() => undefined); - for (const reviewId of released) this.release(reviewId, repositories); + for (const reviewId of released) + this.release(reviewId, repositories, () => this.stillDismissed(reviewId)); + } + + /** Restored since the dismissal: its checkouts may be in use. */ + private stillDismissed(reviewId: string) { + return this.store.dismissedIds().includes(reviewId); } private async repositoryDirs(): Promise { @@ -151,18 +174,26 @@ export class ReviewWorkspaces { return this.releasing.get(reviewId) ?? Promise.resolve(); } - private release(reviewId: string, repositories = this.repositoryDirs()) { + private release( + reviewId: string, + repositories: Promise, + stillReleased: () => boolean, + ) { // Requests started later wait for this release; earlier ones finish first. const requests = [...this.requests.values()]; const done = this.cleanup .then(async () => { await Promise.allSettled(requests); - await this.releaseCheckouts(reviewId, await repositories); + await this.releaseCheckouts( + reviewId, + await repositories, + stillReleased, + ); }) .catch((error) => { console.error( - `Could not free the pinned checkouts of dismissed review ${reviewId}:`, + `Could not free the pinned checkouts of review ${reviewId}:`, error, ); }) @@ -175,9 +206,12 @@ export class ReviewWorkspaces { this.cleanup = done; } - private async releaseCheckouts(reviewId: string, repositories: string[]) { - // Restored since the dismissal: its checkouts may be in use. - if (!this.store.dismissedIds().includes(reviewId)) return; + private async releaseCheckouts( + reviewId: string, + repositories: string[], + stillReleased: () => boolean, + ) { + if (!stillReleased()) return; const environments = this.all().filter( (item) => item.reviewId === reviewId, @@ -365,7 +399,9 @@ export class ReviewWorkspaces { // A dismissed review keeps its record, so collection would skip it. if (this.store.has(environment.reviewId)) - this.release(environment.reviewId); + this.release(environment.reviewId, this.repositoryDirs(), () => + this.stillDismissed(environment.reviewId), + ); else this.collect(id); await this.cleanup; } @@ -647,6 +683,20 @@ export class ReviewWorkspaces { if (environment.rootPath) await removeReviewPrepareArtifacts(environment.rootPath); + if ( + environment.repository && + !this.all().some( + (other) => + other.id !== environment.id && + other.reviewId === environment.reviewId && + other.repository === environment.repository, + ) + ) + await removeReviewManagedCheckouts( + environment.repository, + environment.reviewId, + ); + this.db .prepare("DELETE FROM pinned_environments WHERE id=?") .run(environment.id); diff --git a/packages/review/src/review-app-launcher.test.ts b/packages/review/src/review-app-launcher.test.ts index bfe76134f..5944ad421 100644 --- a/packages/review/src/review-app-launcher.test.ts +++ b/packages/review/src/review-app-launcher.test.ts @@ -20,6 +20,7 @@ import { describe, expect, it, vi } from "vitest"; import type { ReviewInstanceSelection } from "./desktop-discovery"; import { type LaunchDesktopApplicationInput, + desktopApplicationInstalled, launchDesktopApplication, runReviewAppLaunch, } from "./review-app-launcher"; @@ -699,6 +700,44 @@ describe("Review Desktop launcher", () => { child.emit("error", new Error("spawn /usr/bin/review-desktop ENOENT")); await expect(attempt.completion).rejects.toThrow("ENOENT"); }); + + it("finds an installed Desktop only where a launcher could start one", () => { + const linux = { platform: "linux" as const, electron: false, env: {} }; + + const installed = (paths: string[]) => (file: string) => + paths.includes(file); + + expect( + desktopApplicationInstalled({ ...linux, exists: installed([]) }), + ).toBe(false); + expect( + desktopApplicationInstalled({ + ...linux, + exists: installed(["/usr/bin/review-preview-desktop"]), + }), + ).toBe(true); + expect( + desktopApplicationInstalled({ + ...linux, + env: { DEV_FAST_REVIEW_DESKTOP_COMMAND: "/opt/wb/desktop" }, + exists: installed([]), + }), + ).toBe(true); + expect( + desktopApplicationInstalled({ + ...linux, + electron: true, + exists: installed([]), + }), + ).toBe(true); + expect( + desktopApplicationInstalled({ + platform: "darwin", + env: {}, + exists: installed([]), + }), + ).toBe(true); + }); }); function launcherRuntime( diff --git a/packages/review/src/review-app-launcher.ts b/packages/review/src/review-app-launcher.ts index 5c0b1905d..b12527001 100644 --- a/packages/review/src/review-app-launcher.ts +++ b/packages/review/src/review-app-launcher.ts @@ -1,5 +1,12 @@ import { type SpawnOptions, spawn } from "node:child_process"; -import { closeSync, fstatSync, mkdtempSync, openSync, readSync } from "node:fs"; +import { + closeSync, + existsSync, + fstatSync, + mkdtempSync, + openSync, + readSync, +} from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; @@ -227,6 +234,28 @@ export async function focusReviewDesktop( } } +export function desktopApplicationInstalled( + input: Pick< + LaunchDesktopApplicationInput, + "platform" | "electron" | "env" + > & { + exists?: (path: string) => boolean; + } = {}, +): boolean { + const env = input.env ?? process.env; + + if ( + (input.platform ?? process.platform) !== "linux" || + (input.electron ?? Boolean(process.versions.electron)) || + env.DEV_FAST_REVIEW_DESKTOP_COMMAND?.trim() + ) + return true; + + return Object.values(RELEASE_APPS).some(({ linuxLauncher }) => + (input.exists ?? existsSync)(linuxLauncher), + ); +} + export function launchDesktopApplication( input: LaunchDesktopApplicationInput = {}, ): DesktopLaunchAttempt { diff --git a/packages/review/src/server-discovery.ts b/packages/review/src/server-discovery.ts index 4ef177911..5b2477830 100644 --- a/packages/review/src/server-discovery.ts +++ b/packages/review/src/server-discovery.ts @@ -15,6 +15,8 @@ const discoverySchema = z.object({ }), serverPid: z.number().int().positive(), token: z.string().min(1), + // Older servers wrote no startedBy; only a person ran those. + startedBy: z.enum(["user", "cli", "desktop"]).default("user"), }); export type ReviewServerDiscovery = z.infer; @@ -76,7 +78,9 @@ export async function readReviewServerHealth( ok: z.literal(true), instanceId: z.string(), serverId: z.string().optional(), + serverPid: z.number().optional(), version: z.string().optional(), + commit: z.string().nullable().optional(), }) .safeParse(await response.json()); diff --git a/packages/review/src/server/background-server-test-utils.ts b/packages/review/src/server/background-server-test-utils.ts new file mode 100644 index 000000000..2bcc0dcab --- /dev/null +++ b/packages/review/src/server/background-server-test-utils.ts @@ -0,0 +1,77 @@ +import { spawnSync } from "node:child_process"; +import { createRequire } from "node:module"; +import path from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +export const packageRoot = fileURLToPath(new URL("../..", import.meta.url)); + +export const sourceCli = [ + process.execPath, + "--import", + pathToFileURL(createRequire(import.meta.url).resolve("tsx")).href, + path.join(packageRoot, "src", "cli.ts"), +]; + +export function isolatedEnv(root: string): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = {}; + + for (const [key, value] of Object.entries(process.env)) + if ( + !/^(DEV_REVIEW_|DEV_FAST_REVIEW_|REVIEW_DIFFR_|TRACE_|TSX_|XDG_CONFIG_HOME$|CODEX_HOME$)/.test( + key, + ) && + !/^(https?|no|all)_proxy$/i.test(key) + ) + env[key] = value; + + const home = path.join(root, "home"); + roots.add(root); + + return { + ...env, + HOME: home, + TRACE_HOME_DIR: home, + DEV_REVIEW_HOME: path.join(home, ".dev"), + DEV_FAST_REVIEW_TELEMETRY_DISABLED: "1", + DEV_FAST_REVIEW_CLI_NO_DELEGATE: "1", + NODE_USE_ENV_PROXY: "1", + HTTPS_PROXY: "http://127.0.0.1:9", + NO_PROXY: "127.0.0.1,localhost", + }; +} + +const roots = new Set(); + +process.on("exit", () => { + for (const root of roots) + spawnSync("pkill", ["-KILL", "-f", path.basename(root)]); +}); + +export async function stopServersUnder(root: string) { + const pattern = path.basename(root); + + for (const signal of ["TERM", "TERM", "KILL"]) { + checked(spawnSync("pkill", [`-${signal}`, "-f", pattern])); + + for (let i = 0; i < 30 && running(pattern); i++) await delay(100); + + if (!running(pattern)) { + roots.delete(root); + + return; + } + } + + throw new Error(`Processes under ${root} survived SIGKILL`); +} + +function running(pattern: string) { + return checked(spawnSync("pgrep", ["-f", pattern])).status === 0; +} + +function checked(result: ReturnType) { + if (result.error) throw result.error; + + return result; +} diff --git a/packages/review/src/server/background-server.test.ts b/packages/review/src/server/background-server.test.ts new file mode 100644 index 000000000..98ceee1a7 --- /dev/null +++ b/packages/review/src/server/background-server.test.ts @@ -0,0 +1,637 @@ +import { type ChildProcess, spawn, spawnSync } from "node:child_process"; +import { once } from "node:events"; +import { + mkdir, + mkdtemp, + readFile, + realpath, + rm, + writeFile, +} from "node:fs/promises"; +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { + REVIEW_REMOTE_ATTACH_BEGIN, + REVIEW_REMOTE_ATTACH_END, +} from "@dev.fast/review-protocol"; +import { processStartIdentity } from "@dev.fast/trace-core"; +import { remoteAttach } from "@review/remote-attach.js"; +import { + headlessServerLockPath, + readReviewServerDiscovery, + readReviewServerHealth, + reviewServerDiscoveryPath, +} from "@review/server-discovery.js"; +import { afterEach, beforeEach, expect, it } from "vitest"; + +import { + isolatedEnv, + packageRoot, + sourceCli, + stopServersUnder, +} from "./background-server-test-utils.js"; +import { + backgroundServerLogPath, + ensureBackgroundServer, +} from "./background-server.js"; + +let root: string; + +let stateDir: string; + +let env: NodeJS.ProcessEnv; + +beforeEach(async () => { + root = await realpath(await mkdtemp(path.join(tmpdir(), "wb-bg-"))); + stateDir = path.join(root, "state"); + env = isolatedEnv(root); +}); + +afterEach(async () => { + await stopServersUnder(root); + await rm(root, { recursive: true, force: true }); +}); + +it("leaves a detached server running, reports it again, and stops it", async () => { + const first = await cli(["server", "start", "--detach", "--json"]); + + expect(first).toMatchObject({ code: 0, stderr: "" }); + const started = JSON.parse(first.stdout); + expect(started).toMatchObject({ event: "server.status", started: true }); + + const discovery = (await readReviewServerDiscovery(stateDir))!; + expect(discovery.startedBy).toBe("cli"); + expect(discovery.serverPid).toBe(started.serverPid); + expect(await readReviewServerHealth(discovery)).toMatchObject({ ok: true }); + + const second = await cli(["server", "start", "--detach", "--json"]); + expect(JSON.parse(second.stdout)).toMatchObject({ + started: false, + serverPid: started.serverPid, + url: started.url, + }); + + expect((await cli(["server", "reset-id", "--json"])).code).toBe(1); + + const stopped = await cli(["server", "stop", "--json"]); + expect(JSON.parse(stopped.stdout)).toMatchObject({ + event: "server.stop", + stopped: true, + serverPid: started.serverPid, + }); + expect(alive(started.serverPid)).toBe(false); + expect(await readReviewServerDiscovery(stateDir)).toBeNull(); +}, 60_000); + +it("prints the new lines of the log when a detached server never becomes ready", async () => { + const busy = createServer().listen(0, "127.0.0.1"); + await once(busy, "listening"); + const { port } = busy.address() as { port: number }; + + try { + await mkdir(path.dirname(backgroundServerLogPath(stateDir)), { + recursive: true, + }); + await writeFile(backgroundServerLogPath(stateDir), "earlier run\n"); + + const result = await cli([ + "server", + "start", + "--detach", + "--port", + `${port}`, + ]); + + expect(result.code).toBe(1); + expect(result.stderr).toContain("EADDRINUSE"); + expect(result.stderr).not.toContain("earlier run"); + } finally { + busy.close(); + } +}, 60_000); + +it("refuses to stop a server the user started in the foreground", async () => { + const server = spawn( + sourceCli[0]!, + [ + ...sourceCli.slice(1), + "server", + "start", + "--json", + "--state-dir", + stateDir, + ], + { env, stdio: ["ignore", "pipe", "pipe"] }, + ); + + await readyLine(server); + + const refused = await cli(["server", "stop", "--json"]); + + expect(refused.code).toBe(1); + expect(JSON.parse(refused.stdout).error.message).toMatch(/foreground/); + + const discovery = (await readReviewServerDiscovery(stateDir))!; + expect(discovery.startedBy).toBe("user"); + expect(await readReviewServerHealth(discovery)).toMatchObject({ ok: true }); +}, 60_000); + +it("a paused server still owns its state directory: start and reset-id refuse, and stop ends it", async () => { + const { discovery } = await ensureBackgroundServer({ + stateDir, + env, + cli: sourceCli, + }); + + const pid = discovery.serverPid; + process.kill(pid, "SIGSTOP"); + + try { + expect(await readReviewServerHealth(discovery)).toBeNull(); + + const start = spawnSync( + sourceCli[0]!, + [...sourceCli.slice(1), "server", "start", "--state-dir", stateDir], + { env, encoding: "utf8", timeout: 30_000 }, + ); + + expect(start.status).toBe(1); + expect(start.stderr).toMatch(/^A Whiteboard server already owns [^\n]+\n$/); + expect(await cli(["server", "reset-id"])).toMatchObject({ + code: 1, + stderr: expect.stringMatching(/^A Whiteboard server is using [^\n]+\n$/), + }); + expect((await cli(["server", "reset-id", "--json"])).code).toBe(1); + expect(alive(pid)).toBe(true); + + const stopped = await cli(["server", "stop", "--json"]); + + expect(JSON.parse(stopped.stdout)).toMatchObject({ + event: "server.stop", + stopped: true, + serverPid: pid, + }); + expect(alive(pid)).toBe(false); + } finally { + try { + process.kill(pid, "SIGCONT"); + } catch {} + } +}, 90_000); + +it("never signals the pid of a stale discovery file", async () => { + const bystander = spawn(process.execPath, [ + "-e", + "setTimeout(() => {}, 60000)", + root, + ]); + + try { + const file = reviewServerDiscoveryPath(stateDir); + await mkdir(path.dirname(file), { recursive: true }); + await writeFile( + file, + JSON.stringify({ + version: 1, + instanceId: crypto.randomUUID(), + url: "http://127.0.0.1:9", + serverPid: bystander.pid, + token: "stale", + startedBy: "cli", + }), + ); + + const result = await cli(["server", "stop", "--json"]); + + expect(result.code).toBe(0); + expect(JSON.parse(result.stdout)).toMatchObject({ stopped: false }); + expect(alive(bystander.pid!)).toBe(true); + } finally { + bystander.kill("SIGKILL"); + } +}, 30_000); + +it("ends two simultaneous starts with one server that both callers reach", async () => { + const [a, b] = await Promise.all([ + ensureBackgroundServer({ stateDir, env, cli: sourceCli }), + ensureBackgroundServer({ stateDir, env, cli: sourceCli }), + ]); + + expect(a.discovery.instanceId).toBe(b.discovery.instanceId); + expect([a.started, b.started].filter(Boolean)).toHaveLength(1); + expect(await readReviewServerHealth(a.discovery)).toMatchObject({ ok: true }); + expect( + spawnSync("pgrep", ["-f", `server start --state-dir ${stateDir}`], { + encoding: "utf8", + }) + .stdout.trim() + .split("\n"), + ).toEqual([`${a.discovery.serverPid}`]); +}, 60_000); + +it("attaches with one JSON line between the sentinels, and its token reaches the server", async () => { + const first = await cli(["remote", "attach", "--json"]); + + expect(first.code).toBe(0); + const lines = first.stdout.split("\n"); + expect(lines).toEqual([ + REVIEW_REMOTE_ATTACH_BEGIN, + expect.any(String), + REVIEW_REMOTE_ATTACH_END, + "", + ]); + + const attach = JSON.parse(lines[1]!); + expect(attach).toEqual({ + event: "remote.attach", + version: expect.any(String), + commit: null, + serverId: expect.any(String), + url: expect.stringMatching(/^http:\/\/127\.0\.0\.1:\d+$/), + token: expect.any(String), + startedServer: true, + languageServer: null, + languageServerDetail: expect.stringContaining("has no VS Code server"), + languageGroups: [], + }); + + const reviews = (token?: string) => + fetch(`${attach.url}/reviews-api`, { + headers: token ? { "x-review-token": token } : {}, + }).then((response) => response.status); + + expect(await reviews(attach.token)).toBe(200); + expect(await reviews()).toBe(401); + expect((await readReviewServerDiscovery(stateDir))!.startedBy).toBe( + "desktop", + ); + + const second = await cli(["remote", "attach", "--json"]); + expect(JSON.parse(second.stdout.split("\n")[1]!)).toMatchObject({ + startedServer: false, + serverId: attach.serverId, + url: attach.url, + token: attach.token, + }); + + expect( + await readFile(backgroundServerLogPath(stateDir), "utf8"), + ).not.toContain(attach.token); + expect( + spawnSync("ps", ["-eo", "args"], { encoding: "utf8" }).stdout, + ).not.toContain(attach.token); +}, 60_000); + +it("with --replace, stops a server of another version the CLI started and starts this one", async () => { + const old = await ensureBackgroundServer({ stateDir, env, cli: sourceCli }); + const oldVersion = (await readReviewServerHealth(old.discovery))!.version; + + const attach = await remoteAttach({ + stateDir, + env, + cli: sourceCli, + replace: true, + version: "9.9.9", + }); + + expect(attach).toMatchObject({ + startedServer: true, + replaced: true, + previousVersion: oldVersion, + }); + expect(attach).not.toHaveProperty("incompatibleRunning"); + expect(alive(old.discovery.serverPid)).toBe(false); + const discovery = (await readReviewServerDiscovery(stateDir))!; + expect(discovery.serverPid).not.toBe(old.discovery.serverPid); + expect(discovery.startedBy).toBe("desktop"); +}, 60_000); + +it("with --replace and --groups, replaces the older server and reports the language groups in one line", async () => { + const old = await ensureBackgroundServer({ stateDir, env, cli: sourceCli }); + const oldVersion = (await readReviewServerHealth(old.discovery))!.version; + + const attach = await remoteAttach({ + stateDir, + env, + cli: sourceCli, + replace: true, + version: "9.9.9", + groups: ["go"], + }); + + expect(attach).toMatchObject({ + startedServer: true, + replaced: true, + previousVersion: oldVersion, + languageServer: null, + languageGroups: [{ group: "go", installed: false }], + }); + expect(alive(old.discovery.serverPid)).toBe(false); + + const both = await cli([ + "remote", + "attach", + "--json", + "--replace", + "--groups", + "go", + ]); + + expect(both.code).toBe(0); + expect(JSON.parse(both.stdout.split("\n")[1]!)).toMatchObject({ + startedServer: false, + languageServer: null, + languageGroups: [{ group: "go", installed: false }], + }); +}, 60_000); + +it("with --replace, leaves a server of another version a user started, and reports it", async () => { + const server = spawn( + sourceCli[0]!, + [ + ...sourceCli.slice(1), + "server", + "start", + "--json", + "--state-dir", + stateDir, + ], + { env, stdio: ["ignore", "pipe", "pipe"] }, + ); + + await readyLine(server); + const running = (await readReviewServerDiscovery(stateDir))!; + + const attach = await remoteAttach({ + stateDir, + env, + cli: sourceCli, + replace: true, + version: "9.9.9", + }); + + expect(attach).toMatchObject({ + startedServer: false, + incompatibleRunning: { + version: (await readReviewServerHealth(running))!.version, + pid: running.serverPid, + startedBy: "user", + }, + }); + expect(attach).not.toHaveProperty("replaced"); + expect(alive(running.serverPid)).toBe(true); +}, 60_000); + +it("with --replace, leaves a newer server another Desktop started, and reports it", async () => { + const newer = await ensureBackgroundServer({ + stateDir, + env, + cli: sourceCli, + startedBy: "desktop", + }); + + const attach = await remoteAttach({ + stateDir, + env, + cli: sourceCli, + replace: true, + version: "0.0.0", + }); + + expect(attach).toMatchObject({ + startedServer: false, + incompatibleRunning: { + version: (await readReviewServerHealth(newer.discovery))!.version, + pid: newer.discovery.serverPid, + startedBy: "desktop", + }, + }); + expect(attach).not.toHaveProperty("replaced"); + expect(alive(newer.discovery.serverPid)).toBe(true); +}, 60_000); + +it("with --replace, keeps a server of the same version", async () => { + const first = await cli(["remote", "attach", "--json", "--replace"]); + const second = await cli(["remote", "attach", "--json", "--replace"]); + + expect(first.code).toBe(0); + + const [started, kept] = [first, second].map((result) => + JSON.parse(result.stdout.split("\n")[1]!), + ); + + expect(started).toMatchObject({ startedServer: true }); + expect(kept).toMatchObject({ startedServer: false, token: started.token }); + expect(kept).not.toHaveProperty("replaced"); +}, 60_000); + +it("prints a failed attach between the sentinels and exits non-zero", async () => { + await writeFile(path.join(root, "file"), ""); + + const failed = await cli( + ["remote", "attach", "--json"], + {}, + path.join(root, "file", "state"), + ); + + expect(failed.code).toBe(1); + const lines = failed.stdout.split("\n"); + expect(lines).toEqual([ + REVIEW_REMOTE_ATTACH_BEGIN, + expect.any(String), + REVIEW_REMOTE_ATTACH_END, + "", + ]); + expect(JSON.parse(lines[1]!)).toMatchObject({ + event: "error", + error: { message: expect.stringContaining("ENOTDIR") }, + }); +}, 30_000); + +it("attaches the review server when the language extensions cannot be installed", async () => { + const packageRoot = path.join(root, "package"); + await mkdir(path.join(packageRoot, "vscode-server"), { recursive: true }); + await writeFile( + path.join(packageRoot, "vscode-server", "product.json"), + JSON.stringify({ commit: "f".repeat(40) }), + ); + + const attach = await remoteAttach({ + stateDir, + env: { ...env, PATH: path.dirname(process.execPath) }, + packageRoot, + cli: sourceCli, + groups: ["go"], + ensureExtensions: async ({ groups }) => ({ + failed: [{ id: "golang.go", error: `groups ${groups?.join(",")}` }], + }), + }); + + expect(attach).toMatchObject({ + startedServer: true, + languageServer: null, + languageServerDetail: + "Could not install the language extensions: golang.go: groups go", + languageGroups: [{ group: "go", installed: false }], + }); + + const reviews = await fetch(`${attach.url}/reviews-api`, { + headers: { "x-review-token": attach.token }, + }); + + expect(reviews.status).toBe(200); +}, 60_000); + +it("runs the detached server in its state directory, not the caller's", async () => { + const caller = path.join(root, "caller"); + await mkdir(caller); + + const started = await cli( + ["server", "start", "--detach", "--json"], + {}, + stateDir, + caller, + ); + + expect(started.code).toBe(0); + const { serverPid, url } = JSON.parse(started.stdout); + await rm(caller, { recursive: true }); + + const cwd = spawnSync( + "lsof", + ["-a", "-p", `${serverPid}`, "-d", "cwd", "-Fn"], + { + encoding: "utf8", + }, + ).stdout; + + expect(cwd).toContain(`n${stateDir}\n`); + + const discovery = (await readReviewServerDiscovery(stateDir))!; + expect(await readReviewServerHealth(discovery)).toMatchObject({ ok: true }); + + const response = await fetch(`${url}/reviews-api`, { + headers: { "x-review-token": discovery.token }, + }); + + expect(response.status).toBe(200); +}, 60_000); + +it("starts over a lock whose pid now belongs to an unrelated live process", async () => { + await writeLock({ pid: process.pid, started: "a process before a reboot" }); + + const { started } = await ensureBackgroundServer({ + stateDir, + env, + cli: sourceCli, + }); + + expect(started).toBe(true); +}, 60_000); + +it("never starts beside a live lock holder", async () => { + await writeLock({ + pid: process.pid, + started: processStartIdentity(process.pid), + }); + + await expect( + ensureBackgroundServer({ stateDir, env, cli: sourceCli, timeoutMs: 3_000 }), + ).rejects.toThrow(/did not become ready/); + expect(await readReviewServerDiscovery(stateDir)).toBeNull(); +}, 30_000); + +it("tries once more when the lock holder was shutting down", async () => { + await writeLock({ + pid: process.pid, + started: processStartIdentity(process.pid), + }); + + const starting = ensureBackgroundServer({ stateDir, env, cli: sourceCli }); + + let log = ""; + + for (let i = 0; i < 200 && !log.includes("already owns"); i++) { + await new Promise((resolve) => setTimeout(resolve, 50)); + log = await readFile(backgroundServerLogPath(stateDir), "utf8").catch( + () => "", + ); + } + + expect(log).toContain("already owns"); + await rm(headlessServerLockPath(stateDir), { recursive: true }); + + expect(await starting).toMatchObject({ started: true }); +}, 60_000); + +it("reports a server command that cannot be spawned", async () => { + await expect( + ensureBackgroundServer({ + stateDir, + env, + cli: [path.join(root, "missing")], + }), + ).rejects.toThrow(/Could not start the Whiteboard server: .*ENOENT/); +}); + +async function cli( + args: string[], + extraEnv: NodeJS.ProcessEnv = {}, + dir = stateDir, + cwd?: string, +) { + const child = spawn( + sourceCli[0]!, + [...sourceCli.slice(1), ...args, "--state-dir", dir], + { + cwd, + env: { + ...env, + ...(cwd && { + TSX_TSCONFIG_PATH: path.join(packageRoot, "tsconfig.json"), + }), + ...extraEnv, + }, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => (stdout += chunk)); + child.stderr.on("data", (chunk) => (stderr += chunk)); + // "close", not "exit": a detached server holding our pipes would hang here. + const [code] = await once(child, "close"); + + return { code, stdout, stderr }; +} + +async function readyLine(child: ChildProcess) { + let output = ""; + + for await (const chunk of child.stdout!) { + output += chunk; + + if (output.includes("server.ready")) return; + } + + throw new Error(`The server exited before it was ready: ${output}`); +} + +function alive(pid: number) { + try { + process.kill(pid, 0); + + return true; + } catch { + return false; + } +} + +async function writeLock(owner: { pid: number; started: string | null }) { + const lock = headlessServerLockPath(stateDir); + await mkdir(lock, { recursive: true }); + await writeFile(path.join(lock, "owner.json"), JSON.stringify(owner)); +} diff --git a/packages/review/src/server/background-server.ts b/packages/review/src/server/background-server.ts new file mode 100644 index 000000000..478cdc827 --- /dev/null +++ b/packages/review/src/server/background-server.ts @@ -0,0 +1,242 @@ +import { spawn } from "node:child_process"; +import { mkdir, open, readFile, realpath, stat } from "node:fs/promises"; +import path from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; + +import { liveLockOwner, processIsAlive } from "@dev.fast/trace-core"; +import type { ReviewInstanceSelection } from "@review/desktop-discovery.js"; +import { findReviewPackageRoot } from "@review/package-paths.js"; +import { desktopApplicationInstalled } from "@review/review-app-launcher.js"; +import { + type ReviewServerDiscovery, + headlessServerLockPath, + readReviewServerDiscovery, + readReviewServerHealth, + reviewServerStateDir, +} from "@review/server-discovery.js"; + +export interface EnsureBackgroundServerInput { + stateDir: string; + env?: NodeJS.ProcessEnv; + timeoutMs?: number; + startedBy?: "cli" | "desktop"; + args?: readonly string[]; + cli?: readonly string[]; +} + +export function backgroundServerLogPath(stateDir: string) { + return path.join(stateDir, "review-server", "server.log"); +} + +export async function ensureBackgroundServer( + input: EnsureBackgroundServerInput, +): Promise<{ discovery: ReviewServerDiscovery; started: boolean }> { + const stateDir = path.resolve(input.stateDir); + const running = await healthyDiscovery(stateDir); + + if (running) return { discovery: running, started: false }; + + const logPath = backgroundServerLogPath(stateDir); + await mkdir(path.dirname(logPath), { recursive: true, mode: 0o700 }); + const logStart = (await stat(logPath).catch(() => null))?.size ?? 0; + const deadline = Date.now() + (input.timeoutMs ?? 15_000); + let child = await spawnServer(stateDir, logPath, input); + let respawned = false; + + while (Date.now() < deadline) { + const discovery = await healthyDiscovery(stateDir); + + if (discovery) + return { discovery, started: discovery.serverPid === child.pid }; + + if (child.error) + throw new Error( + `Could not start the Whiteboard server: ${child.error.message}`, + ); + + if (child.exited && (await headlessServerOwner(stateDir)) === undefined) { + if (respawned) break; + respawned = true; + child = await spawnServer(stateDir, logPath, input); + } + + await delay(100); + } + + const owner = await headlessServerOwner(stateDir); + + throw new Error( + `The Whiteboard server did not become ready${child.exited ? "" : ` within ${Math.round((input.timeoutMs ?? 15_000) / 1_000)} s; process ${child.pid} is still starting`}.${owner !== undefined && owner !== child.pid ? ` Process ${owner} holds its state directory without answering; \`whiteboard server stop\` ends it.` : ""} The end of ${logPath}:\n${await logTail(logPath, logStart)}`, + ); +} + +export async function ensureServerWithoutDesktop(input: { + selection: ReviewInstanceSelection; + env: NodeJS.ProcessEnv; + desktopInstalled?: () => boolean; + cli?: readonly string[]; +}) { + const { selection, env } = input; + + if ( + selection.source !== "fallback" || + selection.instances.length !== 0 || + selection.problem || + (input.desktopInstalled ?? (() => desktopApplicationInstalled({ env })))() + ) + return undefined; + + const { discovery } = await ensureBackgroundServer({ + stateDir: reviewServerStateDir(env), + env, + cli: input.cli, + }); + + return discovery; +} + +export async function headlessServerOwner(stateDir: string) { + const resolved = await realpath(stateDir).catch(() => path.resolve(stateDir)); + + return liveLockOwner(headlessServerLockPath(resolved)); +} + +export async function recordedBackgroundServer(stateDir: string) { + const discovery = await readReviewServerDiscovery(stateDir).catch(() => null); + const health = discovery && (await readReviewServerHealth(discovery)); + + return discovery && health?.serverPid === discovery.serverPid + ? { discovery, health } + : undefined; +} + +export async function stopBackgroundServer( + discovery: Pick, +) { + const { serverPid } = discovery; + + for (const [signal, waitMs] of [ + ["SIGTERM", 10_000], + ["SIGKILL", 2_000], + ] as const) { + try { + process.kill(serverPid, signal); + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ESRCH") + return; + throw error; + } + + for (let waited = 0; waited < waitMs; waited += 100) { + if (!processIsAlive(serverPid)) return; + await delay(100); + } + } + + throw new Error(`The Whiteboard server (process ${serverPid}) did not stop.`); +} + +interface DetachedChild { + pid?: number; + exited: boolean; + error?: Error; +} + +export async function spawnDetached(input: { + command: string; + args: readonly string[]; + cwd: string; + env: NodeJS.ProcessEnv; + log: string; +}) { + const log = await open(input.log, "a", 0o600); + + const child = spawn(input.command, input.args, { + cwd: input.cwd, + detached: true, + env: input.env, + stdio: ["ignore", log.fd, log.fd], + }); + + const state: DetachedChild = { pid: child.pid, exited: false }; + + child.once("exit", () => (state.exited = true)); + child.once("error", (error) => { + state.exited = true; + state.error = error; + }); + child.unref(); + await log.close(); + + return state; +} + +function spawnServer( + stateDir: string, + logPath: string, + input: EnsureBackgroundServerInput, +) { + const { command, args, env } = cliSpawn(input.cli, input.env ?? process.env); + + return spawnDetached({ + command, + args: [ + ...args, + "server", + "start", + "--state-dir", + stateDir, + "--started-by", + input.startedBy ?? "cli", + ...(input.args ?? []), + ], + cwd: stateDir, + env, + log: logPath, + }); +} + +export function cliSpawn( + cli: readonly string[] | undefined, + env: NodeJS.ProcessEnv, +) { + const [command, ...args] = cli ?? currentCli(); + + return { + command: command!, + args, + // Run from source, tsx finds the path aliases only through this. + env: + args.at(-1)?.endsWith(".ts") && !env.TSX_TSCONFIG_PATH + ? { + ...env, + TSX_TSCONFIG_PATH: path.join( + findReviewPackageRoot(import.meta.url), + "tsconfig.json", + ), + } + : env, + }; +} + +function currentCli() { + return [ + process.execPath, + ...process.execArgv, + path.resolve(process.argv[1]!), + ]; +} + +async function healthyDiscovery(stateDir: string) { + const discovery = await readReviewServerDiscovery(stateDir).catch(() => null); + + return discovery && (await readReviewServerHealth(discovery)) + ? discovery + : null; +} + +async function logTail(logPath: string, from: number) { + const text = (await readFile(logPath)).subarray(from).toString("utf8"); + + return text.trimEnd().split("\n").slice(-20).join("\n"); +} diff --git a/packages/review/src/server/desktop-host-shutdown.test.ts b/packages/review/src/server/desktop-host-shutdown.test.ts index 556b710cc..b628db4c4 100644 --- a/packages/review/src/server/desktop-host-shutdown.test.ts +++ b/packages/review/src/server/desktop-host-shutdown.test.ts @@ -1,5 +1,6 @@ import { EventEmitter } from "node:events"; +import type { ReviewGatewayHost } from "@dev.fast/review-protocol"; import { describe, expect, it, vi } from "vitest"; import { listenForDesktopHostShutdown } from "./desktop-host-shutdown"; @@ -41,6 +42,37 @@ describe("listenForDesktopHostShutdown", () => { ); }); + it("hands a valid remote host list to the gateway", () => { + const parentPort = new EventEmitter(); + const onRemoteHosts = vi.fn<(hosts: ReviewGatewayHost[]) => void>(); + + listenForDesktopHostShutdown( + Object.assign(new EventEmitter(), { parentPort }), + () => {}, + undefined, + undefined, + onRemoteHosts, + ); + + const hosts = [ + { + alias: "devbox", + endpoint: { url: "http://127.0.0.1:41234", token: "secret" }, + }, + { + alias: "other", + problem: { state: "auth-failed", detail: "Permission denied." }, + }, + ]; + + parentPort.emit("message", { data: { type: "remote-hosts", hosts } }); + parentPort.emit("message", { + data: { type: "remote-hosts", hosts: [{ alias: "" }] }, + }); + + expect(onRemoteHosts.mock.calls).toEqual([[hosts]]); + }); + it("accepts direct Node child-process messages", () => { const processEvents = new EventEmitter(); const onShutdown = vi.fn<() => void>(); diff --git a/packages/review/src/server/desktop-host-shutdown.ts b/packages/review/src/server/desktop-host-shutdown.ts index ab33c676f..16089d2fb 100644 --- a/packages/review/src/server/desktop-host-shutdown.ts +++ b/packages/review/src/server/desktop-host-shutdown.ts @@ -1,5 +1,7 @@ import { type JsonValue, + type ReviewGatewayHost, + ReviewGatewayHostSchema, isJsonObject, jsonBoolean, jsonProperty, @@ -9,12 +11,23 @@ import { interface DesktopHostMessagePort { on(event: "message", listener: (event: { data: JsonValue }) => void): void; off(event: "message", listener: (event: { data: JsonValue }) => void): void; + postMessage?(message: JsonValue): void; } interface DesktopHostProcess { parentPort?: DesktopHostMessagePort; on(event: "message", listener: (message: JsonValue) => void): void; off(event: "message", listener: (message: JsonValue) => void): void; + send?(message: JsonValue): void; +} + +export function postToDesktop( + hostProcess: DesktopHostProcess, + message: JsonValue, +): void { + if (hostProcess.parentPort?.postMessage) + hostProcess.parentPort.postMessage(message); + else hostProcess.send?.(message); } export function listenForDesktopHostShutdown( @@ -22,6 +35,7 @@ export function listenForDesktopHostShutdown( onShutdown: () => void, onTelemetrySetting?: (enabled: boolean) => void, onStageRustAnalyzer?: (path: string) => void, + onRemoteHosts?: (hosts: ReviewGatewayHost[]) => void, ): () => void { const handleMessage = (message: JsonValue) => { if (!isJsonObject(message)) return; @@ -45,6 +59,16 @@ export function listenForDesktopHostShutdown( const path = jsonString(jsonProperty(message, "path")); if (path) onStageRustAnalyzer?.(path); + + return; + } + + if (type === "remote-hosts") { + const hosts = ReviewGatewayHostSchema.array().safeParse( + jsonProperty(message, "hosts"), + ); + + if (hosts.success) onRemoteHosts?.(hosts.data); } }; diff --git a/packages/review/src/server/desktop-host.ts b/packages/review/src/server/desktop-host.ts index 54a6d0393..4518aad10 100644 --- a/packages/review/src/server/desktop-host.ts +++ b/packages/review/src/server/desktop-host.ts @@ -12,7 +12,10 @@ import { ReviewTelemetry } from "@review/review-telemetry"; import { SharedReviewStore } from "@review/sharing/import.js"; import { reviewTelemetryChannel } from "@review/telemetry-config"; -import { listenForDesktopHostShutdown } from "./desktop-host-shutdown"; +import { + listenForDesktopHostShutdown, + postToDesktop, +} from "./desktop-host-shutdown"; import { createGlobalReviewServer } from "./desktop-server"; import { drainServerCrashReport, @@ -85,13 +88,19 @@ export async function runDesktopHost( shared.connect(local.store, local.data); await shared.load(); + const log = (message: string) => + process.stdout.write(`[Whiteboard gateway] ${message}\n`); + const server = createGlobalReviewServer({ ...serverInput, + log, sharedReviews: shared, reviewStore: local.store, reviewData: local.data, cliRuntimePath: env.DEV_FAST_REVIEW_CLI_RUNTIME, crashDumpsDir: env.DEV_FAST_REVIEW_CRASH_DUMPS_DIR, + onRemoteHostRestarted: (alias) => + postToDesktop(process, { type: "remote-host-restarted", alias }), }); try { @@ -142,6 +151,7 @@ export async function runDesktopHost( env.DEV_FAST_REVIEW_RUST_ANALYZER = sourcePath; void stageRustAnalyzer(); }, + (hosts) => server.setRemoteHosts(hosts), ); process.once("SIGINT", () => { void stop().then(() => process.exit(0)); diff --git a/packages/review/src/server/desktop-server.ts b/packages/review/src/server/desktop-server.ts index 70dcdd4ab..3c3f30218 100644 --- a/packages/review/src/server/desktop-server.ts +++ b/packages/review/src/server/desktop-server.ts @@ -13,14 +13,15 @@ import { REVIEW_DESKTOP_DISCOVERY_VERSION, type ReviewCliInstallApplyResponse, type ReviewDesktopDiscovery, + type ReviewGatewayHost, type ReviewTutorialOpenResponse, isJsonObject, isObjectValue, parseReviewCliInstallApplyRequest, reviewDiffrSummarizerInputSchema, } from "@dev.fast/review-protocol"; -import { shellQuote, writePrivateJsonAtomic } from "@dev.fast/trace-core"; -import type { AskTools } from "@review/ask/threads.js"; +import { writePrivateJsonAtomic } from "@dev.fast/trace-core"; +import { cliAskTools } from "@review/ask/threads.js"; import { applyCliInstall, declineCliInstall, @@ -52,6 +53,7 @@ import { type ReviewTelemetryContext, } from "@review/review-telemetry"; import type { SharedReviewStore } from "@review/sharing/import.js"; +import { Hono } from "hono"; import { z } from "zod"; import { aliasInstallationToAccount } from "./account-alias"; @@ -67,12 +69,21 @@ import { GlobalReviewDesktopVerbRelay, type ReviewDesktopVerbRelay, } from "./global-verb-relay"; -import { createNodeRequestListener, readBoundedRequestJson } from "./hono-http"; +import { + type ReviewHonoEnv, + createNodeRequestListener, + readBoundedRequestJson, +} from "./hono-http"; import { ReviewServerError } from "./http-json"; import { createJsonReviewReporting } from "./json-review-reporting"; +import { createReviewGateway } from "./review-gateway"; import { reviewLifecycleTelemetry } from "./review-lifecycle-telemetry"; import { ReviewOpenWatchdog } from "./review-open-watchdog"; -import { createWhiteboardCore, serverJson } from "./review-server-core"; +import { + answerErrorsAsJson, + createWhiteboardCore, + serverJson, +} from "./review-server-core"; import { invalidateStructuralComparisons } from "./structural-comparisons.js"; import { createTutorialService } from "./tutorial-service"; import { captureSanitizedUiTelemetry } from "./ui-telemetry"; @@ -100,6 +111,8 @@ export interface GlobalReviewServerInput { relay?: ReviewDesktopVerbRelay; /** Electron's Review crash dump directory; `/crash-reports` reads only inside it. */ crashDumpsDir?: string; + onRemoteHostRestarted?(alias: string): void; + log?(message: string): void; } export interface GlobalReviewServer { @@ -107,6 +120,7 @@ export interface GlobalReviewServer { readonly url: string; listen(): Promise; close(reason?: "app-exit"): Promise; + setRemoteHosts(hosts: ReviewGatewayHost[]): void; } export function createGlobalReviewServer( @@ -135,40 +149,13 @@ export function createGlobalReviewServer( const relay = input.relay ?? new GlobalReviewDesktopVerbRelay({ maxClients: 1 }); - // This Desktop's own CLI, pinned to this instance so another running - // Whiteboard never answers it. - const askCliEnv = () => [ - { name: REVIEW_INSTANCE_ENV, value: identity.key }, - ...(process.versions.electron - ? [{ name: "ELECTRON_RUN_AS_NODE", value: "1" }] - : []), - ...(process.env.DEV_REVIEW_HOME - ? [{ name: "DEV_REVIEW_HOME", value: process.env.DEV_REVIEW_HOME }] - : []), - ]; - - // Ask sessions get its MCP server, `whiteboard mcp`; an agent whose model - // would not get it uses `whiteboard api` from its shell instead. - const askTools: AskTools = { - mcpServers: () => - discovery.cliPath - ? [ - { - name: "whiteboard", - command: process.execPath, - args: [discovery.cliPath, "mcp"], - env: askCliEnv(), - }, - ] - : [], - cli: () => - discovery.cliPath && - [ - ...askCliEnv().map(({ name, value }) => `${name}=${shellQuote(value)}`), - shellQuote(process.execPath), - shellQuote(discovery.cliPath), - ].join(" "), - }; + // Ask sessions get this Desktop's own CLI, pinned to this instance so + // another running Whiteboard never answers it: its MCP server, or + // `whiteboard api` from the shell for an agent whose model would not get it. + const askTools = cliAskTools(() => discovery.cliPath, { + name: REVIEW_INSTANCE_ENV, + value: identity.key, + }); const reviewStore = input.reviewStore; @@ -258,14 +245,30 @@ export function createGlobalReviewServer( const { app } = core; - app.route( + const laptopApi = new Hono(); + answerErrorsAsJson(laptopApi); + + laptopApi.route( "/reviews-api", createJsonReviewReporting(input.reviewStore, telemetry, { shared: input.sharedReviews, }), ); - app.route("/reviews-api", core.api); + laptopApi.route("/reviews-api", core.api); + + const gateway = createReviewGateway({ + local: (request) => laptopApi.fetch(request), + version: readReviewPackageVersion(import.meta.url), + home: devReviewHome(), + relay, + restarted: (alias) => input.onRemoteHostRestarted?.(alias), + log: input.log, + }); + + app.all("/reviews-api", (context) => gateway.fetch(context.req.raw)); + app.all("/reviews-api/*", (context) => gateway.fetch(context.req.raw)); + app.get("/remote-hosts", () => serverJson(200, gateway.hosts())); app.get("/preferences/scratchpad", () => serverJson(200, { enabled: scratchpadEnabled }), ); @@ -587,8 +590,10 @@ export function createGlobalReviewServer( openWatchdog.dispose(); await closeHttpServer(httpServer); + await gateway.close(); await telemetry.shutdown(1_500); }, + setRemoteHosts: (hosts) => gateway.setHosts(hosts), }; } diff --git a/packages/review/src/server/global-verb-relay.test.ts b/packages/review/src/server/global-verb-relay.test.ts index 3c9d42e54..ea56d3445 100644 --- a/packages/review/src/server/global-verb-relay.test.ts +++ b/packages/review/src/server/global-verb-relay.test.ts @@ -426,3 +426,29 @@ function createWriter() { }, }; } + +it("says when the first client attaches and when the last one goes", () => { + const relay = new GlobalReviewDesktopVerbRelay(); + const heard: boolean[] = []; + const stop = relay.onAttachedChange((attached) => heard.push(attached)); + const first = new AbortController(); + const second = new AbortController(); + + const writer = (signal: AbortSignal) => ({ + signal, + write() {}, + close() {}, + }); + + relay.attach(writer(first.signal)); + relay.attach(writer(second.signal)); + first.abort(); + expect(heard).toEqual([true]); + + second.abort(); + expect(heard).toEqual([true, false]); + + stop(); + relay.attach(writer(new AbortController().signal)); + expect(heard).toEqual([true, false]); +}); diff --git a/packages/review/src/server/global-verb-relay.ts b/packages/review/src/server/global-verb-relay.ts index 5a2b993c5..5f5b1e7b4 100644 --- a/packages/review/src/server/global-verb-relay.ts +++ b/packages/review/src/server/global-verb-relay.ts @@ -35,6 +35,7 @@ export interface ReviewDesktopVerbRelay { dispatch(value: JsonValue): Promise; acceptResult(value: JsonValue): boolean; close(): void; + onAttachedChange?(listener: (attached: boolean) => void): () => void; } /** @@ -51,6 +52,7 @@ export class GlobalReviewDesktopVerbRelay implements ReviewDesktopVerbRelay { private readonly pending = new Map(); private readonly timeoutMs: number; private readonly maxClients: number; + private readonly attachedListeners = new Set<(attached: boolean) => void>(); constructor(options: { timeoutMs?: number; maxClients?: number } = {}) { this.timeoutMs = options.timeoutMs ?? DEFAULT_VERB_TIMEOUT_MS; @@ -73,9 +75,21 @@ export class GlobalReviewDesktopVerbRelay implements ReviewDesktopVerbRelay { this.clients.set(writer, detach); writer.signal.addEventListener("abort", detach, { once: true }); + if (this.clients.size === 1) this.notifyAttached(true); + return true; } + onAttachedChange(listener: (attached: boolean) => void): () => void { + this.attachedListeners.add(listener); + + return () => this.attachedListeners.delete(listener); + } + + private notifyAttached(attached: boolean): void { + for (const listener of [...this.attachedListeners]) listener(attached); + } + dispatch(value: JsonValue): Promise { const request: ReviewVerbRequest = parseReviewVerbRequest(value); @@ -170,6 +184,8 @@ export class GlobalReviewDesktopVerbRelay implements ReviewDesktopVerbRelay { for (const [id, verb] of [...this.pending]) { if (verb.waiting.get(id) === writer) this.stopWaiting(verb, id); } + + if (this.clients.size === 0) this.notifyAttached(false); } /** Drops one client's id; with nobody left to answer, the verb fails. */ diff --git a/packages/review/src/server/headless-host.test.ts b/packages/review/src/server/headless-host.test.ts index 84bac8cdd..dd78a4494 100644 --- a/packages/review/src/server/headless-host.test.ts +++ b/packages/review/src/server/headless-host.test.ts @@ -1,5 +1,7 @@ -import { execFileSync, spawnSync } from "node:child_process"; +import { execFileSync, spawn, spawnSync } from "node:child_process"; import { randomUUID } from "node:crypto"; +import { once } from "node:events"; +import { chmodSync, existsSync } from "node:fs"; import { access, mkdir, @@ -13,14 +15,24 @@ import { import { request as httpRequest } from "node:http"; import { tmpdir } from "node:os"; import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; import { PassThrough } from "node:stream"; +import { + type ClientConnection, + agent, + methods, +} from "@agentclientprotocol/sdk"; +import * as diffr from "@dev.fast/diffr"; import { type JsonValue, REVIEW_CLIENT_HEADER, REVIEW_CLIENT_REMOTE, + type ReviewStreamLine, + STRUCTURAL_DIFF_WIRE_VERSION, parseReviewDesktopVerbFrame, } from "@dev.fast/review-protocol"; +import type { AskAgentLauncher } from "@review/ask/agents.js"; import { runReviewCli } from "@review/cli-runner.js"; import { connectReviewApi, @@ -28,10 +40,19 @@ import { } from "@review/review-api/agent-client.js"; import { ReviewApiClient } from "@review/review-api/client.js"; import type { Pins } from "@review/review-api/document.js"; -import { createReviewApi } from "@review/review-api/http.js"; +import { + REMOTE_CHECKOUT_ISSUE, + REMOTE_STRUCTURAL_DIFF_ERROR, + createReviewApi, +} from "@review/review-api/http.js"; import { serveReviewMcp } from "@review/review-api/mcp.js"; import { openReviewProfile } from "@review/review-api/profile.js"; import type { Result, Snapshot } from "@review/review-api/store.js"; +import type { WorkspaceStatus } from "@review/review-api/workspaces.js"; +import { + reviewManagedCheckoutDir, + reviewManagedCheckoutRoot, +} from "@review/review-checkout-paths.js"; import { writeScratchpadEnabled } from "@review/review-preferences.js"; import { ReviewTelemetry } from "@review/review-telemetry.js"; import { @@ -42,7 +63,7 @@ import { reviewServerIsHealthy, } from "@review/server-discovery.js"; import sharp from "sharp"; -import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, expect, it, onTestFinished, vi } from "vitest"; import { z } from "zod"; import { createGlobalReviewServer } from "./desktop-server.js"; @@ -56,6 +77,7 @@ beforeEach(async () => { root = await mkdtemp(path.join(tmpdir(), "review-headless-")); vi.stubEnv("DEV_REVIEW_HOME", root); vi.stubEnv("DEV_FAST_REVIEW_TELEMETRY_DISABLED", "1"); + vi.stubEnv("SHELL", ""); }); afterEach(async () => { @@ -67,6 +89,7 @@ afterEach(async () => { async function start( stateDir = path.join(root, "server"), softwareMapEnabled = false, + launchAskAgent?: AskAgentLauncher, ) { const controller = new AbortController(); const ready = Promise.withResolvers(); @@ -76,6 +99,7 @@ async function start( softwareMapEnabled, signal: controller.signal, onReady: ready.resolve, + launchAskAgent, }); const stop = async () => { @@ -172,12 +196,12 @@ it("shares review identity, resources, sessions and live changes with Desktop in ); const abort = new AbortController(); - const catalog = desktop.watch(null, abort.signal); + const catalog = desktop.watch([{ reviewId: null }], abort.signal); let reviewStream: ReturnType | undefined; try { // The scratchpad is off by default, so there is nothing to list yet. - expect((await catalog.next()).value).toEqual([]); + expect((await catalog.next()).value).toMatchObject({ reviews: [] }); const registered = await server.client.post<{ id: string }>( "/repositories", @@ -199,18 +223,20 @@ it("shares review identity, resources, sessions and live changes with Desktop in }); // Catalog refreshes can also report repository registration before creation. - for await (const value of catalog) { + for await (const line of catalog) { if ( - Array.isArray(value) && - value.some((item) => item.reviewId === created.reviewId) + line.kind === "list" && + line.reviews.some((item) => item.reviewId === created.reviewId) ) break; } - reviewStream = desktop.watch(created.reviewId, abort.signal); + reviewStream = desktop.watch( + [{ reviewId: created.reviewId }], + abort.signal, + ); expect((await reviewStream.next()).value).toMatchObject({ - reviewId: created.reviewId, - version: 0, + value: { reviewId: created.reviewId, version: 0 }, }); const { activityId } = await server.client.post<{ activityId: string }>( @@ -219,14 +245,9 @@ it("shares review identity, resources, sessions and live changes with Desktop in ); for (;;) { - const value = (await reviewStream.next()).value; + const line = (await reviewStream.next()).value; - if ( - value && - !Array.isArray(value) && - "activity" in value && - value.activity?.workingCount === 1 - ) + if (line && "value" in line && line.value.activity.workingCount === 1) break; } @@ -263,15 +284,9 @@ it("shares review identity, resources, sessions and live changes with Desktop in }); for (;;) { - const value = (await reviewStream.next()).value; + const line = (await reviewStream.next()).value; - if ( - value && - !Array.isArray(value) && - "version" in value && - value.version === 1 - ) - break; + if (line && "value" in line && line.value.version === 1) break; } await desktop.post(`/${created.reviewId}/open`, {}); @@ -297,10 +312,18 @@ it("shares review identity, resources, sessions and live changes with Desktop in await server.client.post("/commands", { operation: { type: "delete", reviewId: created.reviewId }, }); - await expect(async () => { - for await (const _value of reviewStream!) { - } - }).rejects.toThrow(/not found/i); + let deleted: ReviewStreamLine | void = undefined; + + while (!(deleted && "error" in deleted)) { + const next = await reviewStream.next(); + + if (next.done) throw new Error("The stream ended before the deletion."); + deleted = next.value; + } + + expect(deleted).toMatchObject({ + error: expect.stringMatching(/not found/i), + }); } finally { abort.abort(); await local.data.close(); @@ -784,7 +807,7 @@ const workspaceFiles = async () => entry.endsWith(".code-workspace"), ); -it("gives a remote caller no local paths and no source window", async () => { +it("gives a remote caller no local paths beyond the navigator's host paths", async () => { const server = await start(); const desktop = await attachDesktop(server.discovery); @@ -838,33 +861,58 @@ it("gives a remote caller no local paths and no source window", async () => { expect(localContext.rootPath).toBe(checkout); expect(localContext.identity).not.toMatch(hash); - // A headless server prepares no commit checkouts, so only the worktree - // review has a language context here. - const remoteContext = await read(worktree, context, true); - expect(remoteContext).toEqual({ identity: expect.stringMatching(hash) }); - expect(JSON.stringify(remoteContext)).not.toContain(home); + const serverId = await serverIdOf(server.discovery); + + expect(await read(worktree, context, true)).toEqual({ + remoteRootPath: checkout, + identity: expect.stringMatching(hash), + serverId, + }); + + await expect + .poll(async () => (await read(commits, context, true)).remoteRootPath, { + timeout: 20_000, + }) + .toEqual(expect.any(String)); + + const pinned = await read(commits, context, true); + expect(pinned).toEqual({ + remoteRootPath: expect.any(String), + identity: expect.stringMatching(hash), + serverId, + }); + expect( + await readFile(path.join(pinned.remoteRootPath, "example.ts"), "utf8"), + ).toBe("export const value = 2;\n"); for (const reviewId of [worktree, commits]) { const remoteFile = await read(reviewId, file, true); expect(remoteFile).toMatchObject({ text: expect.any(String) }); expect(JSON.stringify(remoteFile)).not.toContain(home); - const navigator = await call(reviewId, "/navigator", true, "POST"); - expect(navigator.status).toBe(409); - expect(await navigator.json()).toEqual({ - error: - "Source windows are not available for a review on another machine.", - }); + const navigator = await call( + reviewId, + "/navigator?file=example.ts", + true, + "POST", + ); + + expect(navigator.status).toBe(200); + const answer = await navigator.json(); + expect(answer.filePath).toEqual(expect.stringMatching(/example\.ts$/)); + + const own = await call( + reviewId, + "/navigator?file=example.ts", + false, + "POST", + ); + + expect(await own.json()).toEqual(answer); } - expect(await workspaceFiles()).toEqual([]); + expect(await workspaceFiles()).toHaveLength(2); expect(desktop.verbs).toEqual([]); - - // The same search finds the file an unmarked call writes. - const navigator = await call(worktree, "/navigator", false, "POST"); - expect(navigator.status).toBe(200); - expect(await navigator.json()).toHaveProperty("workspacePath"); - expect(await workspaceFiles()).toHaveLength(1); } finally { desktop.detach(); } @@ -1121,3 +1169,570 @@ it("refuses the removed batch authoring mode instead of ignoring it", async () = expect(result.exitCode).not.toBe(0); expect(result.errors).toContain("--authoring-mode was removed"); }); + +async function stubDiffr() { + const executable = path.join(root, "diffr"); + await writeFile( + executable, + `#!${process.execPath} +const side = (rev) => ({ type: "revision", rev }); +if (process.argv[2] === "config") console.log(JSON.stringify({ changed: false })); +else { + console.log(JSON.stringify({ type: "start", version: ${STRUCTURAL_DIFF_WIRE_VERSION}, lhs: side("base"), rhs: side("head"), files: [] })); + console.log(JSON.stringify({ type: "complete", succeeded: 0, failed: 0 })); +} +`, + { mode: 0o755 }, + ); + + const binary = vi.spyOn(diffr, "diffrBinaryPath").mockReturnValue(executable); + + const info = vi.spyOn(console, "info").mockImplementation(() => {}); + stops.push(async () => { + binary.mockRestore(); + info.mockRestore(); + }); +} + +async function commitsReviews(client: ReviewApiClient) { + const repo = await repository(); + + const { id: repositoryId } = await client.post<{ id: string }>( + "/repositories", + { path: repo.directory }, + ); + + const create = async (title: string) => + ( + await client.post("/commands", { + operation: { + type: "create", + title, + target: { + kind: "commits", + repositoryId, + base: repo.base, + head: repo.head, + }, + open: false, + }, + }) + ).reviewId; + + const worktrees = () => + execFileSync("git", ["worktree", "list", "--porcelain"], { + cwd: repo.directory, + encoding: "utf8", + }).match(/^worktree /gm)?.length; + + return { + repo, + create, + worktrees, + commonDir: path.join(repo.directory, ".git"), + }; +} + +async function structuralDiffEvents( + discovery: Pick, + reviewId: string, + remote = false, +): Promise<{ type: string; message?: string }[]> { + const response = await fetch( + `${discovery.url}/reviews-api/${reviewId}/structural-diff`, + { + headers: { + "x-review-token": discovery.token, + ...(remote && { [REVIEW_CLIENT_HEADER]: REVIEW_CLIENT_REMOTE }), + }, + }, + ); + + expect(response.status).toBe(200); + + return (await response.text()) + .trim() + .split("\n") + .map((line) => JSON.parse(line)); +} + +async function structuralDiff( + discovery: Pick, + reviewId: string, +) { + const events = await structuralDiffEvents(discovery, reviewId); + expect(events.filter((event) => event.type === "error")).toEqual([]); +} + +const cleanupTimeout = { timeout: 10_000 }; + +const attention = ( + client: ReviewApiClient, + reviewId: string, + action: "dismiss" | "restore", +) => + client.post("/commands", { + operation: { type: "attention", reviewId, action }, + }); + +it("frees the checkouts a structural diff made when the review is dismissed or deleted", async () => { + await stubDiffr(); + const server = await start(); + + const { repo, create, worktrees, commonDir } = await commitsReviews( + server.client, + ); + + const dismissed = await create("Dismissed"); + const deleted = await create("Deleted"); + + for (const reviewId of [dismissed, deleted]) { + await structuralDiff(server.discovery, reviewId); + expect( + existsSync( + reviewManagedCheckoutDir(commonDir, reviewId, "head", repo.head), + ), + ).toBe(true); + } + + await attention(server.client, dismissed, "dismiss"); + await expect + .poll( + () => existsSync(reviewManagedCheckoutRoot(commonDir, dismissed)), + cleanupTimeout, + ) + .toBe(false); + + await server.client.post("/commands", { + operation: { type: "delete", reviewId: deleted }, + }); + await expect + .poll( + () => existsSync(reviewManagedCheckoutRoot(commonDir, deleted)), + cleanupTimeout, + ) + .toBe(false); + expect(worktrees()).toBe(1); +}); + +it("leaves a review's checkouts to a live lease owner and takes over a dead owner's", async () => { + await stubDiffr(); + const first = await start(); + + const { repo, create, worktrees, commonDir } = await commitsReviews( + first.client, + ); + + const reviewId = await create("Leased"); + const deleted = await create("Leased and deleted"); + + for (const id of [reviewId, deleted]) + await structuralDiff(first.discovery, id); + await first.stop(); + + const checkout = (id: string) => + reviewManagedCheckoutDir(commonDir, id, "head", repo.head); + + expect(existsSync(checkout(reviewId))).toBe(true); + + const owner = spawn(process.execPath, ["-e", "setInterval(() => {}, 1e9)"], { + stdio: "ignore", + }); + + const exited = once(owner, "exit"); + + const stopOwner = async () => { + if (owner.exitCode === null && owner.signalCode === null) owner.kill(); + await exited; + }; + + stops.push(stopOwner); + + const leases = new DatabaseSync( + path.join(first.stateDir, "review-api.db.workspaces"), + ); + + try { + for (const id of [reviewId, deleted]) + leases + .prepare("INSERT OR REPLACE INTO workspace_leases VALUES(?,?,?)") + .run(id, randomUUID(), owner.pid!); + } finally { + leases.close(); + } + + const leased = await start(first.stateDir); + await attention(leased.client, reviewId, "dismiss"); + await leased.client.post("/commands", { + operation: { type: "delete", reviewId: deleted }, + }); + await leased.stop(); + expect(existsSync(checkout(reviewId))).toBe(true); + expect(existsSync(checkout(deleted))).toBe(true); + + await stopOwner(); + await (await start(first.stateDir)).stop(); + expect(existsSync(reviewManagedCheckoutRoot(commonDir, reviewId))).toBe( + false, + ); + expect(worktrees()).toBe(2); +}); + +function caller(discovery: Pick) { + return async ( + route: string, + remote: boolean, + body?: JsonValue, + ): Promise => { + const response = await fetch(`${discovery.url}/reviews-api${route}`, { + method: body === undefined ? "GET" : "POST", + headers: { + "x-review-token": discovery.token, + "content-type": "application/json", + ...(remote && { [REVIEW_CLIENT_HEADER]: REVIEW_CLIENT_REMOTE }), + }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + + expect(response.status).toBe(200); + + return response.json() as Promise; + }; +} + +type Workspace = Partial; + +it("gives a remote caller no checkout paths or logs from workspace management", async () => { + const server = await start(); + const call = caller(server.discovery); + const { create, commonDir } = await commitsReviews(server.client); + const reviewId = await create("Workspaces"); + const marker = path.basename(root); + + await mkdir(path.join(commonDir, "dev-fast"), { recursive: true }); + const blocker = path.join(commonDir, "dev-fast", "reviews"); + await writeFile(blocker, ""); + + const environment = `/${reviewId}/environment`; + const localIssues = await call(environment, false, {}); + expect(JSON.stringify(localIssues)).toContain(marker); + expect(await call(environment, true, {})).toEqual({ + issues: [ + { side: "head", message: REMOTE_CHECKOUT_ISSUE }, + { side: "base", message: REMOTE_CHECKOUT_ISSUE }, + ], + }); + + const workspaces = `/${reviewId}/workspaces`; + const failed = await call(workspaces, false); + expect(failed).toHaveLength(2); + + for (const workspace of failed) { + expect(workspace).toMatchObject({ rootPath: null, state: "failed" }); + expect(workspace.log).toContain(marker); + } + + const remoteFailed = await call(workspaces, true); + expect(remoteFailed).toEqual( + failed.map(({ rootPath: _, log: __, ...status }) => ({ + ...status, + issue: REMOTE_CHECKOUT_ISSUE, + })), + ); + + const retry = `${workspaces}/${failed[0]!.id}/retry`; + expect((await call(retry, false, {})).log).toContain(marker); + const remoteRetry = await call(retry, true, {}); + expect(remoteRetry).toMatchObject({ issue: REMOTE_CHECKOUT_ISSUE }); + expect(remoteRetry).not.toHaveProperty("rootPath"); + expect(remoteRetry).not.toHaveProperty("log"); + expect(JSON.stringify(remoteRetry)).not.toContain(marker); + + await rm(blocker); + expect(await call(environment, false, { retry: true })).toEqual({ + issues: [], + }); + + for (const workspace of await call(workspaces, false)) + expect(workspace.rootPath).toContain(marker); + + for (const workspace of await call(workspaces, true)) { + expect(workspace).not.toHaveProperty("rootPath"); + expect(workspace).not.toHaveProperty("log"); + expect(JSON.stringify(workspace)).not.toContain(marker); + } +}); + +it.skipIf(process.getuid?.() === 0)( + "gives a remote caller no paths from a failed checkout cleanup", + async () => { + const server = await start(); + const call = caller(server.discovery); + const { create } = await commitsReviews(server.client); + const reviewId = await create("Cleanup"); + const marker = path.basename(root); + await call(`/${reviewId}/environment`, false, {}); + + const [head] = await call(`/${reviewId}/workspaces`, false); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + chmodSync(head!.rootPath!, 0o500); + stops.push(async () => { + chmodSync(head!.rootPath!, 0o700); + error.mockRestore(); + }); + await attention(server.client, reviewId, "dismiss"); + + const cleanup = "/workspace-cleanup"; + + const failures = async (remote: boolean) => + (await call<{ failures: Workspace[] }>(cleanup, remote, {})).failures; + + await expect.poll(() => failures(false), cleanupTimeout).not.toEqual([]); + const local = await failures(false); + expect(JSON.stringify(local)).toContain(marker); + + expect(await failures(true)).toEqual( + local.map(({ rootPath: _, log: __, ...status }) => status), + ); + }, +); + +it("gives a remote caller no paths from a failed structural diff", async () => { + await stubDiffr(); + const server = await start(); + const { create, commonDir } = await commitsReviews(server.client); + const reviewId = await create("Blocked"); + const marker = path.basename(root); + + await mkdir(path.join(commonDir, "dev-fast"), { recursive: true }); + await writeFile(path.join(commonDir, "dev-fast", "reviews"), ""); + + const local = await structuralDiffEvents(server.discovery, reviewId); + expect(local).toEqual([{ type: "error", message: expect.any(String) }]); + expect(local[0]!.message).toContain(marker); + expect(await structuralDiffEvents(server.discovery, reviewId, true)).toEqual([ + { type: "error", message: REMOTE_STRUCTURAL_DIFF_ERROR }, + ]); +}); + +async function askCall( + server: Awaited>, + route: string, + body?: JsonValue, +) { + return fetch(`${server.discovery.url}/reviews-api/${route}`, { + headers: { + "x-review-token": server.discovery.token, + "content-type": "application/json", + }, + ...(body !== undefined && { method: "POST", body: JSON.stringify(body) }), + }); +} + +async function executable(file: string, body: string) { + await mkdir(path.dirname(file), { recursive: true }); + await writeFile(file, `#!/bin/sh\n${body}\n`, { mode: 0o755 }); +} + +async function askAgentsAndEnv() { + const launched = Promise.withResolvers(); + const started = Date.now(); + + const server = await start( + undefined, + false, + async (_agent, _cwd, options) => { + launched.resolve(options?.env); + throw new Error("No agent here."); + }, + ); + + const startMs = Date.now() - started; + const { worktree } = await reviewsOfBothKinds(server.client); + + const { agents } = await ( + await askCall(server, `${worktree}/ask/agents`) + ).json(); + + await askCall(server, `${worktree}/ask`, { + agent: "claude", + question: { text: "Why?" }, + selection: { target: { kind: "text", quote: "value" }, title: "value" }, + }); + + return { + claude: agents.find((entry: { id: string }) => entry.id === "claude"), + env: await launched.promise, + startMs, + }; +} + +it("finds and launches Ask agents on the login shell's PATH, with its own node last", async () => { + const login = path.join(root, "login-bin"); + await executable(path.join(login, "claude"), "exit 0"); + await executable( + path.join(root, "shell"), + `echo Welcome\nprintf '\\nWHITEBOARD-PATH=%s\\n' '${login}'\necho Bye`, + ); + vi.stubEnv("SHELL", path.join(root, "shell")); + vi.stubEnv("PATH", "/usr/bin:/bin"); + + const { claude, env } = await askAgentsAndEnv(); + + expect(claude).toMatchObject({ available: true }); + expect(env?.PATH).toBe( + [login, "/usr/bin:/bin", path.dirname(process.execPath)].join( + path.delimiter, + ), + ); +}); + +it("starts with its own PATH when the login shell hangs", async () => { + const bin = path.join(root, "bin"); + await executable(path.join(bin, "claude"), "exit 0"); + await executable(path.join(root, "shell"), "sleep 60"); + vi.stubEnv("SHELL", path.join(root, "shell")); + vi.stubEnv("PATH", `${bin}:/usr/bin:/bin`); + + const { claude, env, startMs } = await askAgentsAndEnv(); + + expect(startMs).toBeLessThan(8_000); + expect(claude).toMatchObject({ available: true }); + expect(env?.PATH).toBe( + [bin, "/usr/bin:/bin", path.dirname(process.execPath)].join(path.delimiter), + ); +}, 20_000); + +it("opens an Ask thread in the review's checkout and closes it on stop", async () => { + const launched = Promise.withResolvers(); + let stopped = 0; + + const fake = agent({ name: "fake" }) + .onRequest(methods.agent.initialize, () => ({ + protocolVersion: 1, + agentCapabilities: {}, + authMethods: [], + })) + .onRequest(methods.agent.session.new, () => ({ sessionId: "session" })) + .onRequest(methods.agent.session.prompt, () => ({ + stopReason: "end_turn" as const, + })); + + const server = await start(undefined, false, async () => { + let connection: ClientConnection | undefined; + launched.resolve(); + + return { + connect: (client) => (connection = client.connect(fake)), + diagnostics: () => "", + stop: () => { + stopped++; + connection?.close(); + }, + }; + }); + + const { root: checkout, worktree } = await reviewsOfBothKinds(server.client); + + const opened = await askCall(server, `${worktree}/ask`, { + agent: "claude", + question: { text: "Why?" }, + selection: { target: { kind: "text", quote: "value" }, title: "value" }, + }); + + expect(opened.status).toBe(200); + + const { threadId } = await opened.json(); + + const reader = (await askCall(server, `${worktree}/ask/${threadId}/watch`)) + .body!.pipeThrough(new TextDecoderStream()) + .getReader(); + + const { value = "" } = await reader.read(); + expect(JSON.parse(value.split("\n")[0]!).snapshot.cwd).toBe(checkout); + + await launched.promise; + expect(stopped).toBe(0); + await server.stop(); + expect(stopped).toBe(1); + + while (!(await reader.read()).done); +}); + +it("closes an Ask thread half an hour after its last watcher leaves and reopens it from history", async () => { + let launches = 0, + stopped = 0; + + const fake = agent({ name: "fake" }) + .onRequest(methods.agent.initialize, () => ({ + protocolVersion: 1, + agentCapabilities: { loadSession: true }, + authMethods: [], + })) + .onRequest(methods.agent.session.new, () => ({ sessionId: "session" })) + .onRequest(methods.agent.session.load, () => ({})) + .onRequest(methods.agent.session.prompt, () => ({ + stopReason: "end_turn" as const, + })); + + const server = await start(undefined, false, async () => { + let connection: ClientConnection | undefined; + launches++; + + return { + connect: (client) => (connection = client.connect(fake)), + diagnostics: () => "", + stop: () => { + stopped++; + connection?.close(); + }, + }; + }); + + const { worktree } = await reviewsOfBothKinds(server.client); + + vi.useFakeTimers({ + shouldAdvanceTime: true, + toFake: ["setTimeout", "clearTimeout"], + }); + onTestFinished(() => { + vi.useRealTimers(); + }); + + const { threadId } = await ( + await askCall(server, `${worktree}/ask`, { + agent: "claude", + question: { text: "Why?" }, + selection: { target: { kind: "text", quote: "value" }, title: "value" }, + }) + ).json(); + + const reader = (await askCall(server, `${worktree}/ask/${threadId}/watch`)) + .body!.pipeThrough(new TextDecoderStream()) + .getReader(); + + let seen = ""; + + while (!seen.includes('"status":"idle"')) + seen += (await reader.read()).value ?? ""; + + await vi.advanceTimersByTimeAsync(60 * 60_000); + expect(stopped).toBe(0); + + await reader.cancel(); + await vi.waitFor(async () => { + await vi.advanceTimersByTimeAsync(30 * 60_000); + expect(stopped).toBe(1); + }); + + const { threads } = await ( + await askCall(server, `${worktree}/ask/threads`) + ).json(); + + expect(threads).toMatchObject([{ id: threadId }]); + + expect( + (await askCall(server, `${worktree}/ask/${threadId}/open`, {})).status, + ).toBe(200); + await vi.waitFor(() => expect(launches).toBe(2)); +}); diff --git a/packages/review/src/server/headless-host.ts b/packages/review/src/server/headless-host.ts index ba98d88ff..96c984556 100644 --- a/packages/review/src/server/headless-host.ts +++ b/packages/review/src/server/headless-host.ts @@ -1,10 +1,16 @@ +import { spawn } from "node:child_process"; import { randomBytes, randomUUID } from "node:crypto"; import { once } from "node:events"; +import { existsSync } from "node:fs"; import { mkdir, realpath, rm } from "node:fs/promises"; import { createServer } from "node:http"; +import path from "node:path"; import { isObjectValue } from "@dev.fast/json"; import { withFileLock, writePrivateJsonAtomic } from "@dev.fast/trace-core"; +import type { AskAgentLauncher } from "@review/ask/agents.js"; +import { cliAskTools } from "@review/ask/threads.js"; +import { findReviewPackageRoot } from "@review/package-paths.js"; import { openReviewProfile } from "@review/review-api/profile.js"; import { type ReviewServerDiscovery, @@ -26,12 +32,16 @@ interface HeadlessServerInput { stateDir: string; port?: number; softwareMapEnabled?: boolean; + startedBy?: ReviewServerDiscovery["startedBy"]; signal: AbortSignal; /** The CLI's instance, already on the `headless` surface. */ telemetry?: Pick; onReady(discovery: ReviewServerDiscovery): void; + launchAskAgent?: AskAgentLauncher; } +export class HeadlessServerBusyError extends Error {} + /** One foreground headless endpoint per profile; Desktop shares its database. */ export async function runHeadlessServer(input: HeadlessServerInput) { await mkdir(input.stateDir, { recursive: true, mode: 0o700 }); @@ -45,7 +55,7 @@ export async function runHeadlessServer(input: HeadlessServerInput) { ).finally(() => stopErrorTelemetry?.()); if (!outcome.acquired) - throw new Error( + throw new HeadlessServerBusyError( `A Whiteboard server already owns ${stateDir}. Stop it first, or choose another --state-dir.`, ); } @@ -70,6 +80,61 @@ export function withHeadlessServerLock( ); } +const LOGIN_PATH_TIMEOUT_MS = 4_000; + +/** Started over ssh, this server has sshd's PATH; agents are on the login + * shell's. Its own node comes last. */ +async function agentEnv(): Promise { + const login = process.env.SHELL && (await loginPath(process.env.SHELL)); + + return { + ...process.env, + PATH: [login, process.env.PATH, path.dirname(process.execPath)] + .filter(Boolean) + .join(path.delimiter), + }; +} + +/** Only its marked line is read, so start-up output is ignored. */ +function loginPath(shell: string) { + return new Promise((resolve) => { + const child = spawn( + shell, + ["-lic", 'printf "\\nWHITEBOARD-PATH=%s\\n" "$PATH"'], + { detached: true, stdio: ["ignore", "pipe", "ignore"] }, + ); + + let output = ""; + + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => (output += chunk)); + + const timer = setTimeout(() => { + try { + process.kill(-child.pid!, "SIGKILL"); + } catch {} + + resolve(undefined); + }, LOGIN_PATH_TIMEOUT_MS); + + child.once("error", () => { + clearTimeout(timer); + resolve(undefined); + }); + child.once("close", () => { + clearTimeout(timer); + + const found = output + .replaceAll("\r", "") + .split("\n") + .findLast((line) => line.startsWith("WHITEBOARD-PATH=")) + ?.slice("WHITEBOARD-PATH=".length); + + resolve(found?.startsWith("/") ? found : undefined); + }); + }); +} + async function serve(input: HeadlessServerInput) { if (input.signal.aborted) return; @@ -78,7 +143,7 @@ async function serve(input: HeadlessServerInput) { if (input.telemetry) await drainServerCrashReport(input.telemetry); const local = await openReviewProfile(input.stateDir, { - manageWorkspaces: false, + manageWorkspaces: true, }); const discovery: ReviewServerDiscovery = { @@ -87,11 +152,20 @@ async function serve(input: HeadlessServerInput) { url: "http://127.0.0.1:0", serverPid: process.pid, token: randomBytes(32).toString("base64url"), + startedBy: input.startedBy ?? "user", }; const relay = new GlobalReviewDesktopVerbRelay(); - const { app, api } = createWhiteboardCore({ + const cliPath = path.join( + findReviewPackageRoot(import.meta.url), + "dist", + "cli.js", + ); + + const env = await agentEnv(); + + const { app, api, close } = createWhiteboardCore({ profile: local, relay, token: discovery.token, @@ -100,6 +174,15 @@ async function serve(input: HeadlessServerInput) { // The scratchpad is the laptop's alone, even with a Desktop attached. scratchpad: () => false, status: () => ({ key: "headless", home: input.stateDir }), + ask: { + tools: cliAskTools(() => (existsSync(cliPath) ? cliPath : undefined), { + name: "DEV_REVIEW_SERVER_DIR", + value: input.stateDir, + }), + launch: input.launchAskAgent, + env, + idleCloseMs: 30 * 60_000, + }, }); app.route("/reviews-api", api); @@ -129,6 +212,8 @@ async function serve(input: HeadlessServerInput) { input.signal.addEventListener("abort", () => resolve(), { once: true }); }); } finally { + close(); + // Watch streams may live forever. Drain ordinary requests, then bound shutdown. const forceClose = setTimeout(() => server.closeAllConnections(), 5_000); forceClose.unref(); diff --git a/packages/review/src/server/headless-sharing.test.ts b/packages/review/src/server/headless-sharing.test.ts index 010ab7816..525f6b4c1 100644 --- a/packages/review/src/server/headless-sharing.test.ts +++ b/packages/review/src/server/headless-sharing.test.ts @@ -37,6 +37,7 @@ it("commits and uploads through a real headless server and CLI without Desktop, vi.stubEnv("DEV_REVIEW_SHARE_ORIGIN", "https://sharing.test"); vi.stubEnv("DEV_REVIEW_HOME", stateDir); vi.stubEnv("DEV_FAST_REVIEW_TELEMETRY_DISABLED", "1"); + vi.stubEnv("SHELL", ""); const realFetch = globalThis.fetch; const requests: { url: string; headers: Headers; body?: string }[] = []; const blobs = new Map(); diff --git a/packages/review/src/server/hono-http.ts b/packages/review/src/server/hono-http.ts index ef915df5e..133bef662 100644 --- a/packages/review/src/server/hono-http.ts +++ b/packages/review/src/server/hono-http.ts @@ -1,7 +1,11 @@ import crypto from "node:crypto"; import type { IncomingMessage, ServerResponse } from "node:http"; -import { type JsonValue, parseJsonText } from "@dev.fast/review-protocol"; +import { + type JsonValue, + REVIEW_HOST_HEADER, + parseJsonText, +} from "@dev.fast/review-protocol"; import { type HttpBindings, getRequestListener } from "@hono/node-server"; import { REVIEW_APP_SESSION_ID_HEADER } from "@review/ui-telemetry-events"; import type { Hono } from "hono"; @@ -76,6 +80,7 @@ export function applyCorsHeaders( "GET, HEAD, POST, PUT, PATCH, DELETE, OPTIONS", ); response.headers.set("access-control-allow-private-network", "true"); + response.headers.set("access-control-expose-headers", REVIEW_HOST_HEADER); return response; } diff --git a/packages/review/src/server/review-gateway-hosts.test.ts b/packages/review/src/server/review-gateway-hosts.test.ts new file mode 100644 index 000000000..3c19c534f --- /dev/null +++ b/packages/review/src/server/review-gateway-hosts.test.ts @@ -0,0 +1,682 @@ +import { once } from "node:events"; +import { cp, mkdtemp, rm } from "node:fs/promises"; +import net, { type AddressInfo } from "node:net"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { readReviewPackageVersion } from "@review/package-paths.js"; +import { Agent } from "undici"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; + +import { createGatewayHosts, readBody, send } from "./review-gateway-hosts.js"; +import { + startFake, + startRemote, + stopAll, +} from "./review-gateway-test-utils.js"; + +const version = readReviewPackageVersion(import.meta.url); + +let root: string; + +const closes: (() => void)[] = []; + +beforeEach(async () => { + root = await mkdtemp(path.join(tmpdir(), "review-gateway-hosts-")); + vi.stubEnv("DEV_REVIEW_HOME", root); + vi.stubEnv("DEV_FAST_REVIEW_TELEMETRY_DISABLED", "1"); +}); + +afterEach(async () => { + for (const close of closes.splice(0)) close(); + await stopAll(); + vi.unstubAllEnvs(); + await rm(root, { recursive: true, force: true }); +}); + +function hosts(laptopVersion = version) { + const created = createGatewayHosts({ version: laptopVersion }); + closes.push(() => created.close()); + + return created; +} + +it("never contacts a host that arrives with a problem", async () => { + const fake = await startFake({ version }); + const gateway = hosts(); + + gateway.set([ + { + alias: "devbox", + endpoint: fake.endpoint, + problem: { state: "auth-failed", detail: "Permission denied." }, + }, + { alias: "later" }, + ]); + await new Promise((resolve) => setTimeout(resolve, 200)); + + expect(gateway.states()).toEqual([ + { alias: "devbox", state: "auth-failed", detail: "Permission denied." }, + { alias: "later", state: "connecting", detail: expect.any(String) }, + ]); + expect(fake.requests).toEqual([]); +}); + +it("refuses a host on another version, naming both versions, with the install command beside the detail", async () => { + const remote = await startRemote(path.join(root, "a")); + const gateway = hosts("0.0.0-other"); + + gateway.set([{ alias: "devbox", endpoint: remote.endpoint }]); + + await expect.poll(() => gateway.states()[0]?.state).toBe("incompatible"); + const [state] = gateway.states(); + expect(state?.serverId).toBe((await remote.health()).serverId); + expect(state?.detail).toContain(version); + expect(state?.detail).toContain("0.0.0-other"); + expect(state?.detail).toContain("Install Whiteboard 0.0.0-other on devbox."); + expect(state?.detail).not.toContain("npm install"); + expect(state?.installCommand).toBe( + "npm install -g @dev.fast/whiteboard@0.0.0-other", + ); + expect(gateway.online()).toEqual([]); +}); + +it("refuses a host whose version is not a version, without repeating it", async () => { + const fake = await startFake({ version: "x npm install -g evil" }); + const gateway = hosts("0.1.6"); + + gateway.set([{ alias: "devbox", endpoint: fake.endpoint }]); + + await expect.poll(() => gateway.states()[0]?.state).toBe("incompatible"); + const [state] = gateway.states(); + expect(state?.detail).toBe("devbox reports an invalid version."); + expect(state?.installCommand).toBe( + "npm install -g @dev.fast/whiteboard@0.1.6", + ); + expect(gateway.online()).toEqual([]); +}); + +it("names the laptop's install command for a host without Whiteboard", () => { + const gateway = hosts("0.1.6"); + + gateway.set([ + { + alias: "devbox", + problem: { + state: "not-installed", + detail: "Whiteboard is not installed on devbox.", + }, + }, + { + alias: "other", + problem: { state: "auth-failed", detail: "Permission denied." }, + }, + ]); + + expect(gateway.states()).toEqual([ + { + alias: "devbox", + state: "not-installed", + detail: "Whiteboard is not installed on devbox.", + installCommand: "npm install -g @dev.fast/whiteboard@0.1.6", + }, + { alias: "other", state: "auth-failed", detail: "Permission denied." }, + ]); +}); + +it("an unsupported host keeps its reason and gets no install command", () => { + const gateway = hosts("0.1.6"); + const detail = "This host runs glibc 2.31; Whiteboard needs 2.34 or newer."; + + gateway.set([{ alias: "old", problem: { state: "unsupported", detail } }]); + + expect(gateway.states()).toEqual([ + { alias: "old", state: "unsupported", detail }, + ]); +}); + +it("a host Desktop is installing on shows the step, and the next step replaces it", () => { + const gateway = hosts("0.1.6"); + + gateway.set([ + { + alias: "box", + installing: { step: "node", detail: "uploaded from this computer" }, + }, + ]); + expect(gateway.states()).toEqual([ + { + alias: "box", + state: "installing", + detail: "Installing Node 24 (uploaded from this computer).", + }, + ]); + + gateway.set([{ alias: "box", installing: { step: "verifying" } }]); + expect(gateway.states()).toEqual([ + { alias: "box", state: "installing", detail: "Checking the install." }, + ]); +}); + +it("a host whose install question is open says so, and the answer clears it", () => { + const gateway = hosts("0.1.6"); + + gateway.set([{ alias: "box", asking: "0.1.6" }]); + expect(gateway.states()).toEqual([ + { + alias: "box", + state: "connecting", + detail: "Waiting for an answer: install Whiteboard 0.1.6 on box?", + }, + ]); + + gateway.set([{ alias: "box" }]); + expect(gateway.states()[0]?.detail).toBe("Waiting for a connection to box."); +}); + +it("a declined host keeps the install command and says it was declined", () => { + const gateway = hosts("0.1.6"); + const detail = "Whiteboard is not installed on box."; + + gateway.set([ + { + alias: "box", + problem: { state: "not-installed", detail }, + declined: true, + }, + ]); + + expect(gateway.states()).toEqual([ + { + alias: "box", + state: "not-installed", + detail, + installCommand: "npm install -g @dev.fast/whiteboard@0.1.6", + declined: true, + }, + ]); +}); + +it("an older server after a failed or declined install is incompatible, says why, and offers the install", async () => { + const fake = await startFake({ version: "0.1.5" }); + const gateway = hosts("0.1.6"); + + const installFailure = + "Installing Whiteboard 0.1.6 on box failed while installing the package: npm ERR! 404."; + + gateway.set([ + { alias: "box", endpoint: fake.endpoint, installFailure, declined: true }, + ]); + + await expect + .poll(() => gateway.states()[0]) + .toMatchObject({ + state: "incompatible", + detail: `box runs Whiteboard 0.1.5; this Desktop runs 0.1.6. Install Whiteboard 0.1.6 on box. ${installFailure}`, + declined: true, + }); +}); + +it("a declined host on this Desktop's version is online, with no install offered", async () => { + const fake = await startFake({ version: "0.1.6" }); + const gateway = hosts("0.1.6"); + + gateway.set([{ alias: "box", endpoint: fake.endpoint, declined: true }]); + expect(gateway.states()[0]).toMatchObject({ state: "connecting" }); + expect(gateway.states()[0]).not.toHaveProperty("declined"); + + await expect.poll(() => gateway.states()[0]?.state).toBe("online"); + expect(gateway.states()[0]).not.toHaveProperty("declined"); +}); + +it("a server a user started at another version is incompatible, with no install command", () => { + const gateway = hosts("0.1.7"); + + const detail = + "A Whiteboard server 0.1.6 started by a user is running on box; stop it to use this Desktop's version."; + + gateway.set([{ alias: "box", problem: { state: "incompatible", detail } }]); + + expect(gateway.states()).toEqual([ + { alias: "box", state: "incompatible", detail }, + ]); +}); + +it("refuses a host whose version could not be read", async () => { + const fake = await startFake({ version: "unknown" }); + const gateway = hosts("unknown"); + + gateway.set([{ alias: "devbox", endpoint: fake.endpoint }]); + + await expect.poll(() => gateway.states()[0]?.state).toBe("incompatible"); +}); + +it("shows Desktop's language features on an online host, and keeps the host online when only they change", async () => { + const fake = await startFake({ version }); + const gateway = hosts(); + + const detail = + "language features need the same Whiteboard version on devbox: it runs 1111111, this Desktop 2222222"; + + gateway.set([ + { + alias: "devbox", + endpoint: fake.endpoint, + languageFeatures: false, + languageFeaturesDetail: detail, + }, + ]); + + await expect.poll(() => gateway.states()[0]?.state).toBe("online"); + expect(gateway.states()[0]).toMatchObject({ + languageFeatures: false, + languageFeaturesDetail: detail, + }); + + const languageGroups = [ + { + group: "swift", + installed: true, + detail: "swift was not found on the login shell's PATH", + }, + ]; + + gateway.set([ + { + alias: "devbox", + endpoint: fake.endpoint, + languageFeatures: true, + languageGroups, + }, + ]); + + expect(gateway.states()[0]).toMatchObject({ + state: "online", + languageFeatures: true, + languageGroups, + }); + expect(gateway.states()[0]?.languageFeaturesDetail).toBeUndefined(); + expect(gateway.online().map((host) => host.alias)).toEqual(["devbox"]); +}); + +it("says nothing of language features for a host that is not online", () => { + const gateway = hosts(); + + gateway.set([{ alias: "devbox", languageFeatures: true }]); + + expect(gateway.states()).toEqual([ + { alias: "devbox", state: "connecting", detail: expect.any(String) }, + ]); +}); + +it("treats one server under two aliases as one machine", async () => { + const remote = await startRemote(path.join(root, "a")); + const gateway = hosts(); + + gateway.set([ + { alias: "first", endpoint: remote.endpoint }, + { alias: "second", endpoint: remote.endpoint }, + ]); + + const { serverId } = await remote.health(); + await expect + .poll(() => gateway.states()) + .toEqual([ + { alias: "first", serverId, state: "online" }, + { alias: "second", serverId, state: "online" }, + ]); + expect(gateway.serving(serverId)?.alias).toBe("first"); + expect(gateway.online().map((host) => host.alias)).toEqual(["first"]); +}); + +it("marks the second of two machines with a copied id as a duplicate", async () => { + const first = await startRemote(path.join(root, "a")); + await first.stop(); + await cp(path.join(root, "a"), path.join(root, "c"), { recursive: true }); + const a = await startRemote(path.join(root, "a")); + const c = await startRemote(path.join(root, "c")); + const { serverId } = await a.health(); + expect((await c.health()).serverId).toBe(serverId); + + const gateway = hosts(); + gateway.set([ + { alias: "wb-a", endpoint: a.endpoint }, + { alias: "wb-c", endpoint: c.endpoint }, + ]); + + await expect.poll(() => gateway.states()[1]?.state).toBe("duplicate"); + const [online, duplicate] = gateway.states(); + expect(online).toEqual({ alias: "wb-a", serverId, state: "online" }); + expect(duplicate?.detail).toContain("wb-a"); + expect(duplicate?.detail).toContain("wb-c"); + expect(duplicate?.detail).toContain("whiteboard server reset-id"); + expect(gateway.online().map((host) => host.alias)).toEqual(["wb-a"]); +}); + +it("retries an offline host until it answers", async () => { + const fake = await startFake({ version }); + await fake.stop(); + const gateway = hosts(); + + gateway.set([{ alias: "devbox", endpoint: fake.endpoint }]); + await expect.poll(() => gateway.states()[0]?.state).toBe("offline"); + + await startFake({ version }, fake.port); + await expect + .poll(() => gateway.states()[0]?.state, { timeout: 5_000 }) + .toBe("online"); +}); + +async function copiedStore() { + const first = await startRemote(path.join(root, "a")); + await first.stop(); + await cp(path.join(root, "a"), path.join(root, "c"), { recursive: true }); + const a = await startRemote(path.join(root, "a")); + const c = await startRemote(path.join(root, "c")); + + return { a, c, serverId: (await a.health()).serverId }; +} + +it("keeps a copied store a duplicate while the first alias is down", async () => { + const { a, c, serverId } = await copiedStore(); + const port = Number(new URL(a.endpoint.url).port); + const gateway = hosts(); + + gateway.set([ + { alias: "wb-a", endpoint: a.endpoint }, + { alias: "wb-c", endpoint: c.endpoint }, + ]); + await expect.poll(() => gateway.states()[1]?.state).toBe("duplicate"); + + await a.stop(); + gateway.failed(gateway.serving(serverId)!, "test"); + await expect.poll(() => gateway.states()[0]?.state).toBe("offline"); + + expect(gateway.states()[1]?.state).toBe("duplicate"); + expect(gateway.serving(serverId)).toBeUndefined(); + expect(gateway.online()).toEqual([]); + expect(gateway.unavailable(serverId, "wb-a")).toMatchObject({ + alias: "wb-a", + state: "offline", + }); + + await expect + .poll(() => gateway.states()[0]?.detail) + .toBe("wb-a is offline: it refused the connection; attaching again."); + const restarted = await startRemote(path.join(root, "a"), port); + + gateway.set([ + { alias: "wb-a", endpoint: restarted.endpoint }, + { alias: "wb-c", endpoint: c.endpoint }, + ]); + await expect + .poll(() => gateway.states()[0]?.state, { timeout: 5_000 }) + .toBe("online"); + expect(gateway.states()[1]?.state).toBe("duplicate"); + expect(gateway.serving(serverId)?.alias).toBe("wb-a"); +}); + +it("decides the duplicate by the order of the setting", async () => { + const { a, c, serverId } = await copiedStore(); + const gateway = hosts(); + + gateway.set([ + { alias: "wb-c", endpoint: c.endpoint }, + { alias: "wb-a", endpoint: a.endpoint }, + ]); + + await expect + .poll(() => gateway.states().map((host) => host.state)) + .toEqual(["online", "duplicate"]); + expect(gateway.states()[1]?.detail).toContain("wb-c"); + expect(gateway.serving(serverId)?.alias).toBe("wb-c"); +}); + +it("leaves a host in backoff alone when only another host changes, and checks it at once when its own entry does", async () => { + let failing = true; + let checks = 0; + + const fake = await startFake({ + version, + handle(request, response) { + if (request.url !== "/health") return false; + checks += 1; + + if (!failing) return false; + response.statusCode = 500; + response.end(); + + return true; + }, + }); + + const gateway = hosts(); + + gateway.set([{ alias: "devbox", endpoint: fake.endpoint }]); + await expect + .poll(() => checks, { timeout: 10_000 }) + .toBeGreaterThanOrEqual(4); + expect(gateway.states()[0]?.state).toBe("offline"); + const before = checks; + + failing = false; + gateway.set([ + { alias: "devbox", endpoint: fake.endpoint }, + { alias: "other" }, + ]); + await new Promise((resolve) => setTimeout(resolve, 500)); + + expect(checks).toBe(before); + expect(gateway.states()[0]?.state).toBe("offline"); + + gateway.set([{ alias: "devbox" }, { alias: "other" }]); + gateway.set([ + { alias: "devbox", endpoint: fake.endpoint }, + { alias: "other" }, + ]); + + await expect + .poll(() => gateway.states()[0]?.state, { timeout: 1_000 }) + .toBe("online"); +}, 20_000); + +async function listen(answer: (socket: net.Socket, request: number) => void) { + let requests = 0; + + const server = net.createServer((socket) => { + let received = ""; + + socket.on("data", (chunk) => { + received += chunk.toString(); + + if (!received.endsWith("\r\n\r\nhi")) return; + received = ""; + requests += 1; + answer(socket, requests); + }); + }); + + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + closes.push(() => server.close()); + const dispatcher = new Agent(); + closes.push(() => dispatcher.destroy()); + + // SAFETY: a TCP listener's address() is an AddressInfo. + const { port } = server.address() as AddressInfo; + + const post = async () => { + const response = await send( + { + alias: "devbox", + endpoint: { url: `http://127.0.0.1:${port}`, token: "" }, + dispatcher, + }, + { method: "POST", path: "/ask", body: Buffer.from("hi") }, + ); + + return (await readBody(response.body, 64)).toString(); + }; + + return { post, requests: () => requests }; +} + +it("sends a request again when the host closed the kept-alive socket it reused", async () => { + const host = await listen((socket, request) => { + if (request === 2) socket.destroy(); + else socket.write("HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok"); + }); + + expect(await host.post()).toBe("ok"); + // Wait for undici to free the connection, so the next request reuses it. + await new Promise((resolve) => setImmediate(resolve)); + expect(await host.post()).toBe("ok"); + expect(host.requests()).toBe(3); +}); + +it("does not send a request with a body again when a new connection closed", async () => { + const host = await listen((socket) => socket.destroy()); + + await expect(host.post()).rejects.toMatchObject({ code: "UND_ERR_SOCKET" }); + expect(host.requests()).toBe(1); +}); + +it("a restarted server is offline until Desktop attaches again, then online with the new token", async () => { + const stateDir = path.join(root, "a"); + const a = await startRemote(stateDir); + const port = Number(new URL(a.endpoint.url).port); + const restarted: string[] = []; + + const gateway = createGatewayHosts({ + version, + restarted: (alias) => restarted.push(alias), + }); + + closes.push(() => gateway.close()); + + gateway.set([ + { alias: "wb-a1", endpoint: a.endpoint }, + { alias: "wb-a2", endpoint: a.endpoint }, + ]); + const { serverId } = await a.health(); + await expect + .poll(() => gateway.states().map((host) => host.state)) + .toEqual(["online", "online"]); + + await a.stop(); + const b = await startRemote(stateDir, port); + gateway.failed(gateway.serving(serverId)!, "test"); + + await expect + .poll(() => gateway.states().map((host) => [host.state, host.detail])) + .toEqual([ + ["offline", "wb-a1 restarted; attaching again."], + ["offline", "wb-a2 restarted; attaching again."], + ]); + expect(restarted).toEqual(["wb-a1", "wb-a2"]); + + gateway.set([ + { alias: "wb-a1", endpoint: b.endpoint }, + { alias: "wb-a2", endpoint: b.endpoint }, + ]); + await expect + .poll(() => gateway.states().map((host) => host.state)) + .toEqual(["online", "online"]); + expect(restarted).toHaveLength(2); + expect(gateway.serving(serverId)?.alias).toBe("wb-a1"); +}, 20_000); + +it("a 401 from a host asks Desktop once to attach again", async () => { + const fake = await startFake({ + version, + handle: (request, response) => { + if (request.url === "/health") return false; + response.statusCode = 401; + response.end(); + + return true; + }, + }); + + const restarted: string[] = []; + + const gateway = createGatewayHosts({ + version, + restarted: (alias) => restarted.push(alias), + }); + + closes.push(() => gateway.close()); + + gateway.set([{ alias: "devbox", endpoint: fake.endpoint }]); + await expect.poll(() => gateway.states()[0]?.state).toBe("online"); + const [remote] = gateway.online(); + + for (let i = 0; i < 2; i++) + await ( + await send(remote!, { method: "GET", path: "/reviews-api" }) + ).body.dump(); + + expect(gateway.states()[0]).toMatchObject({ + state: "offline", + detail: "devbox restarted; attaching again.", + }); + expect(restarted).toEqual(["devbox"]); +}); + +it("a server that stops answering /health is attached again after three timeouts in a row", async () => { + let hanging = false; + + const fake = await startFake({ + version, + handle: (request) => hanging && request.url === "/health", + }); + + const restarted: string[] = []; + + const gateway = createGatewayHosts({ + version, + restarted: (alias) => restarted.push(alias), + }); + + closes.push(() => gateway.close()); + gateway.set([{ alias: "devbox", endpoint: fake.endpoint }]); + await expect.poll(() => gateway.states()[0]?.state).toBe("online"); + + hanging = true; + gateway.failed(gateway.online()[0]!, "test"); + + await expect + .poll(() => gateway.states()[0]?.detail, { timeout: 5_000 }) + .toBe("devbox is offline: it did not answer within 3 seconds."); + expect(restarted).toEqual([]); + + await expect.poll(() => restarted, { timeout: 20_000 }).toEqual(["devbox"]); + expect(gateway.states()[0]).toMatchObject({ + state: "offline", + detail: + "devbox is offline: it did not answer within 3 seconds, 3 times in a row; attaching again.", + }); +}, 30_000); + +it("a server gone from behind a working forward asks Desktop to attach again", async () => { + const a = await startRemote(path.join(root, "a")); + const restarted: string[] = []; + + const gateway = createGatewayHosts({ + version, + restarted: (alias) => restarted.push(alias), + }); + + closes.push(() => gateway.close()); + gateway.set([{ alias: "wb-a", endpoint: a.endpoint }]); + await expect.poll(() => gateway.states()[0]?.state).toBe("online"); + + await a.stop(); + gateway.failed(gateway.online()[0]!, "test"); + + await expect.poll(() => restarted).toEqual(["wb-a"]); + expect(gateway.states()[0]).toMatchObject({ + state: "offline", + detail: "wb-a is offline: it refused the connection; attaching again.", + }); +}); diff --git a/packages/review/src/server/review-gateway-hosts.ts b/packages/review/src/server/review-gateway-hosts.ts new file mode 100644 index 000000000..1a57d9576 --- /dev/null +++ b/packages/review/src/server/review-gateway-hosts.ts @@ -0,0 +1,627 @@ +import { addAbortListener } from "node:events"; +import { Readable } from "node:stream"; + +import { + REVIEW_CLIENT_HEADER, + REVIEW_CLIENT_REMOTE, + type REVIEW_REMOTE_INSTALL_STEPS, + REVIEW_REMOTE_VERSION, + type ReviewGatewayHost, + type ReviewGatewayHostState, +} from "@dev.fast/review-protocol"; +import { Agent, type Dispatcher, request } from "undici"; +import { z } from "zod"; + +import { StreamLimitError } from "./bounded-stream.js"; + +const HEALTH_TIMEOUT_MS = 3_000; + +const HEARTBEAT_MS = 10_000; + +const TIMEOUTS_BEFORE_ATTACH = 3; + +export const FIRST_RETRY_MS = 500; + +export const MAX_RETRY_MS = 30_000; + +export const FIRST_BYTE_TIMEOUT_MS = 10_000; + +export const NO_ANSWER = `it did not answer within ${FIRST_BYTE_TIMEOUT_MS / 1_000} seconds`; + +export const UUID = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +export const jitter = (ms: number) => ms * (0.75 + Math.random() * 0.5); + +const INSTALLS = new Set([ + "incompatible", + "not-installed", +]); + +const INSTALL_STEPS: Record< + (typeof REVIEW_REMOTE_INSTALL_STEPS)[number], + string +> = { + preparing: "Preparing to install Whiteboard", + "waiting-for-lock": "Waiting for another install to finish", + node: "Installing Node 24", + package: "Installing the Whiteboard package", + verifying: "Checking the install", + done: "Installed; starting the server", +}; + +const healthSchema = z.object({ + ok: z.literal(true), + serverId: z.string().optional(), + instanceId: z.string(), + version: z.string(), +}); + +export interface GatewayRemote { + readonly alias: string; + readonly endpoint?: { url: string; token: string }; + readonly dispatcher?: Agent; + readonly serverId?: string; + readonly unauthorized?: () => void; +} + +interface Host extends GatewayRemote { + endpoint?: { url: string; token: string }; + dispatcher?: Agent; + unauthorized?: () => void; + problem?: ReviewGatewayHost["problem"]; + languageFeatures?: boolean; + languageFeaturesDetail?: string; + languageGroups?: ReviewGatewayHost["languageGroups"]; + installing?: ReviewGatewayHost["installing"]; + declined?: true; + asking?: string; + installFailure?: string; + serverId?: string; + instanceId?: string; + status: ReviewGatewayHostState["state"]; + detail?: string; + retryMs: number; + retry?: NodeJS.Timeout; + checking?: AbortController; + checked?: boolean; + restarted?: boolean; + timeouts?: number; +} + +export function createGatewayHosts(input: { + version: string; + log?(message: string): void; + remembered?(serverId: string): string | undefined; + machine?(serverId: string, alias: string): void; + changed?(): void; + restarted?(alias: string): void; + heartbeatMs?: number; +}) { + const log = input.log ?? (() => {}); + let hosts: Host[] = []; + let closed = false; + const reported = new Map(); + + const machine = (serverId: string | undefined) => + serverId === undefined + ? undefined + : hosts.find((host) => host.serverId === serverId); + + const pending = (host: Host) => { + if (host.serverId === undefined) return undefined; + const alias = input.remembered?.(host.serverId); + const index = hosts.findIndex((candidate) => candidate.alias === alias); + const remembered = hosts[index]; + + return remembered && + index < hosts.indexOf(host) && + remembered.serverId === undefined + ? remembered + : undefined; + }; + + const unsettledBefore = (host: Host) => + host.serverId === undefined + ? [] + : hosts + .slice(0, hosts.indexOf(host)) + .filter((earlier) => !earlier.problem && !earlier.checked); + + const held = (host: Host) => + host.status === "online" && unsettledBefore(host).length > 0; + + const isDuplicate = (host: Host) => { + const first = machine(host.serverId); + + if (first !== undefined && first.instanceId !== host.instanceId) + return true; + + return pending(host) !== undefined; + }; + + const serving = (serverId: string | undefined) => + serverId === undefined + ? undefined + : hosts.find( + (host) => + host.status === "online" && + host.serverId === serverId && + !held(host) && + !isDuplicate(host), + ); + + function stateOf(host: Host): ReviewGatewayHostState { + const known = { + alias: host.alias, + ...(host.serverId !== undefined && { serverId: host.serverId }), + }; + + if (held(host)) + return { + ...known, + state: "connecting", + detail: `Waiting for ${unsettledBefore(host) + .map((earlier) => earlier.alias) + .join(", ")} to answer before using ${host.alias}.`, + }; + + const waitingFor = pending(host); + + if (waitingFor) + return { + ...known, + state: "duplicate", + detail: `${host.alias} is waiting for ${waitingFor.alias}, which last served this server id and has not answered yet. If they are one machine, remove one of the aliases. If they are two machines, run \`whiteboard server reset-id\` on ${host.alias}.`, + }; + + const first = machine(host.serverId); + + if (isDuplicate(host)) + return { + ...known, + state: "duplicate", + detail: `${first?.alias} and ${host.alias} report the same server id. If they are one machine, remove one of the aliases. If they are two machines, run \`whiteboard server reset-id\` on ${host.alias}.`, + }; + + const detail = + host.status === "incompatible" && host.installFailure + ? `${host.detail ? `${host.detail} ` : ""}${host.installFailure}` + : host.detail; + + return { + ...known, + state: host.status, + ...(detail !== undefined && { detail }), + ...(INSTALLS.has(host.status) && + host.problem?.state !== "incompatible" && { + installCommand: `npm install -g @dev.fast/whiteboard@${input.version}`, + }), + ...(host.declined && + host.status !== "online" && + host.status !== "connecting" && { declined: true as const }), + ...(host.status === "online" && languageOf(host)), + }; + } + + function report() { + const states = hosts.map(stateOf); + + for (const host of hosts) + if ( + host.serverId !== undefined && + serving(host.serverId) === host && + machine(host.serverId) === host + ) + input.machine?.(host.serverId, host.alias); + + for (const state of states) { + const line = `${state.state}${state.detail ? ` (${state.detail})` : ""}`; + + if (reported.get(state.alias) === line) continue; + reported.set(state.alias, line); + log(`Host ${state.alias}: ${line}`); + } + + for (const alias of reported.keys()) + if (!states.some((state) => state.alias === alias)) + reported.delete(alias); + + input.changed?.(); + } + + function restartedHost( + host: Host, + detail = `${host.alias} restarted; attaching again.`, + ) { + if (host.restarted || closed) return; + host.restarted = true; + clearTimeout(host.retry); + host.checking?.abort(); + host.checking = undefined; + host.status = "offline"; + host.detail = detail; + report(); + input.restarted?.(host.alias); + } + + function dispose(host: Host) { + clearTimeout(host.retry); + host.checking?.abort(); + host.checking = undefined; + void host.dispatcher?.destroy(); + } + + function create(given: ReviewGatewayHost): Host { + const host: Host = { + alias: given.alias, + status: "connecting", + retryMs: FIRST_RETRY_MS, + ...languageOf(given), + }; + + if (given.endpoint) host.endpoint = given.endpoint; + + if (given.declined) host.declined = true; + + if (given.installFailure) host.installFailure = given.installFailure; + + if (given.problem) { + host.problem = given.problem; + host.status = given.problem.state; + host.detail = given.problem.detail; + } else if (given.installing && !given.endpoint) { + const { step, detail } = given.installing; + host.installing = given.installing; + host.status = "installing"; + host.detail = `${INSTALL_STEPS[step]}${detail ? ` (${detail})` : ""}.`; + } else if (given.asking && !given.endpoint) { + host.asking = given.asking; + host.detail = `Waiting for an answer: install Whiteboard ${given.asking} on ${given.alias}?`; + } else if (!given.endpoint) + host.detail = `Waiting for a connection to ${given.alias}.`; + else { + host.dispatcher = new Agent(); + host.unauthorized = () => restartedHost(host); + } + + return host; + } + + function retryLater(host: Host) { + const delay = jitter(host.retryMs); + host.retryMs = Math.min(host.retryMs * 2, MAX_RETRY_MS); + host.retry = setTimeout(() => void check(host), delay); + host.retry.unref(); + } + + async function check(host: Host) { + if (closed || !host.endpoint || host.problem || host.restarted) return; + clearTimeout(host.retry); + host.checking?.abort(); + const abort = new AbortController(); + host.checking = abort; + const timeout = AbortSignal.timeout(HEALTH_TIMEOUT_MS); + let health: z.infer | undefined; + let reason = "it did not answer"; + let code: string | undefined; + let timedOut = false; + + try { + const response = await send(host, { + method: "GET", + path: "/health", + headers: { "x-review-token": host.endpoint?.token ?? "" }, + signal: AbortSignal.any([abort.signal, timeout]), + }); + + const parsed = healthSchema.safeParse( + JSON.parse((await readBody(response.body, 64 * 1024)).toString()), + ); + + if (parsed.success) health = parsed.data; + else reason = "it did not answer as a Whiteboard server"; + } catch (error) { + if (timeout.aborted) { + timedOut = true; + reason = `it did not answer within ${HEALTH_TIMEOUT_MS / 1_000} seconds`; + } else if (!abort.signal.aborted) { + code = errorCode(error); + reason = errorText(error); + } + } + + if (host.checking !== abort) return; + host.checking = undefined; + host.checked = true; + + if ( + !health && + host.instanceId !== undefined && + (code === "ECONNRESET" || + code === "ECONNREFUSED" || + code === "UND_ERR_SOCKET") + ) + return restartedHost( + host, + `${host.alias} is offline: ${reason}; attaching again.`, + ); + + host.timeouts = timedOut ? (host.timeouts ?? 0) + 1 : 0; + + if ( + host.instanceId !== undefined && + host.timeouts >= TIMEOUTS_BEFORE_ATTACH + ) + return restartedHost( + host, + `${host.alias} is offline: ${reason}, ${host.timeouts} times in a row; attaching again.`, + ); + + if (!health) { + host.status = "offline"; + host.detail = `${host.alias} is offline: ${reason}.`; + retryLater(host); + } else if (health.serverId === undefined) { + if (host.serverId !== undefined) + for (const other of hosts) + if (other !== host && other.serverId === host.serverId) + void check(other); + + return restartedHost(host); + } else { + const restarted = + host.serverId === health.serverId && + host.instanceId !== health.instanceId; + + host.serverId = health.serverId; + host.instanceId = health.instanceId; + + if (restarted) + for (const other of hosts) + if (other !== host && other.serverId === health.serverId) + void check(other); + host.retryMs = FIRST_RETRY_MS; + + if (restarted) return restartedHost(host); + + if ( + health.version !== "unknown" && + !REVIEW_REMOTE_VERSION.test(health.version) + ) { + host.status = "incompatible"; + host.detail = `${host.alias} reports an invalid version.`; + } else if ( + health.version === "unknown" || + health.version !== input.version + ) { + host.status = "incompatible"; + host.detail = `${host.alias} runs Whiteboard ${health.version}; this Desktop runs ${input.version}. Install Whiteboard ${input.version} on ${host.alias}.`; + } else { + host.status = "online"; + host.detail = undefined; + } + + host.retry = setTimeout( + () => void check(host), + input.heartbeatMs ?? HEARTBEAT_MS, + ); + host.retry.unref(); + } + + report(); + } + + return { + set(list: ReviewGatewayHost[]) { + const previous = new Map(hosts.map((host) => [host.alias, host])); + const next: Host[] = []; + const changed: Host[] = []; + + for (const given of list) { + if (next.some((host) => host.alias === given.alias)) continue; + const current = previous.get(given.alias); + + if ( + current && + JSON.stringify([ + current.endpoint, + current.problem, + current.installing, + current.declined, + current.asking, + current.installFailure, + ]) === + JSON.stringify([ + given.endpoint, + given.problem, + given.installing, + given.declined, + given.asking, + given.installFailure, + ]) + ) { + previous.delete(given.alias); + current.languageFeatures = given.languageFeatures; + current.languageFeaturesDetail = given.languageFeaturesDetail; + current.languageGroups = given.languageGroups; + next.push(current); + } else { + const host = create(given); + next.push(host); + changed.push(host); + } + } + + for (const gone of previous.values()) dispose(gone); + hosts = next; + + for (const host of changed) void check(host); + report(); + }, + states: () => hosts.map(stateOf), + serving: (serverId: string): GatewayRemote | undefined => serving(serverId), + machineAlias: (serverId: string) => machine(serverId)?.alias, + online: (): GatewayRemote[] => + hosts.filter( + (host) => host.status === "online" && serving(host.serverId) === host, + ), + unavailable(serverId: string, alias: string) { + const host = hosts.find( + (candidate) => + (candidate.status !== "online" || held(candidate)) && + !isDuplicate(candidate) && + (candidate.serverId === serverId || + (candidate.serverId === undefined && candidate.alias === alias)), + ); + + return host && stateOf(host); + }, + failed(remote: GatewayRemote, reason: string) { + const host = hosts.find((candidate) => candidate === remote); + + if (!host || host.status !== "online") return; + host.status = "offline"; + host.detail = `${host.alias} is offline: ${reason}.`; + report(); + void check(host); + }, + recheck(remote: GatewayRemote) { + const host = hosts.find((candidate) => candidate === remote); + + if (host && !host.checking) void check(host); + }, + close() { + closed = true; + + for (const host of hosts) dispose(host); + }, + }; +} + +export type GatewayHosts = ReturnType; + +const languageOf = ({ + languageFeatures, + languageFeaturesDetail, + languageGroups, +}: Pick< + ReviewGatewayHost, + "languageFeatures" | "languageFeaturesDetail" | "languageGroups" +>) => ({ + ...(languageFeatures !== undefined && { languageFeatures }), + ...(languageFeaturesDetail !== undefined && { languageFeaturesDetail }), + ...(languageGroups !== undefined && { languageGroups }), +}); + +export const remoteHeaders = (remote: GatewayRemote) => ({ + "x-review-token": remote.endpoint?.token ?? "", + [REVIEW_CLIENT_HEADER]: REVIEW_CLIENT_REMOTE, +}); + +export async function send( + remote: GatewayRemote, + input: { + method: Dispatcher.HttpMethod; + path: string; + headers?: Record; + body?: Buffer | Readable; + signal?: AbortSignal; + headersTimeout?: number; + bodyTimeout?: number; + }, + retry = true, +): Promise { + if (!remote.endpoint) throw new Error(`${remote.alias} has no endpoint.`); + let response: Dispatcher.ResponseData; + + try { + response = await request(new URL(input.path, remote.endpoint.url), { + dispatcher: remote.dispatcher, + method: input.method, + headers: input.headers, + body: input.body, + signal: input.signal, + headersTimeout: input.headersTimeout ?? FIRST_BYTE_TIMEOUT_MS, + bodyTimeout: input.bodyTimeout ?? 0, + }); + } catch (error) { + if ( + retry && + !input.signal?.aborted && + !(input.body instanceof Readable) && + errorCode(error) === "UND_ERR_SOCKET" && + (input.body === undefined || reusedSocket(error)) + ) + return send(remote, input, false); + + throw error; + } + + if (response.statusCode === 401) remote.unauthorized?.(); + + return response; +} + +export async function readBody( + body: Dispatcher.ResponseData["body"], + limit: number, + signal?: AbortSignal, +) { + const parts: Buffer[] = []; + let size = 0; + + const stop = + signal && addAbortListener(signal, () => body.destroy(signal.reason)); + + try { + for await (const part of body) { + // SAFETY: undici's body yields Buffers. + const chunk = part as Buffer; + size += chunk.byteLength; + + if (size > limit) { + body.destroy(); + throw new StreamLimitError(); + } + + parts.push(chunk); + } + } finally { + stop?.[Symbol.dispose](); + } + + return Buffer.concat(parts, size); +} + +const codedError = z.object({ code: z.string() }); + +const socketError = z.object({ socket: z.object({ bytesRead: z.number() }) }); + +// undici has no reusedSocket; a socket that already read bytes served an earlier response. +const reusedSocket = (cause: unknown) => + (socketError.safeParse(cause).data?.socket.bytesRead ?? 0) > 0; + +const ERROR_WORDS = new Map([ + ["ECONNREFUSED", "it refused the connection"], + ["ECONNRESET", "it closed the connection"], + ["UND_ERR_SOCKET", "it closed the connection"], + ["ETIMEDOUT", "it did not answer"], + ["UND_ERR_HEADERS_TIMEOUT", NO_ANSWER], +]); + +export function errorCode(cause: unknown): string { + if (!(cause instanceof Error)) return String(cause); + + return ( + codedError.safeParse(cause).data?.code ?? + codedError.safeParse(cause.cause).data?.code ?? + cause.message + ); +} + +export function errorText(cause: unknown): string { + const code = errorCode(cause); + + return ERROR_WORDS.get(code) ?? code; +} diff --git a/packages/review/src/server/review-gateway-list.test.ts b/packages/review/src/server/review-gateway-list.test.ts new file mode 100644 index 000000000..1325784bb --- /dev/null +++ b/packages/review/src/server/review-gateway-list.test.ts @@ -0,0 +1,209 @@ +import { randomUUID } from "node:crypto"; + +import type { + ReviewApiSummary, + ReviewGatewayHostState, +} from "@dev.fast/review-protocol"; +import { expect, it } from "vitest"; + +import { type ListSource, mergeLists } from "./review-gateway-list.js"; + +const entry = ( + title: string, + extra: Partial = {}, +): ReviewApiSummary => ({ + reviewId: randomUUID(), + version: 1, + title, + createdAt: "2026-09-30T00:00:00.000Z", + repositoryName: "project", + viewedAt: null, + dismissedAt: null, + ...extra, +}); + +function source( + states: ReviewGatewayHostState[], + lists: Record, + serving: string[] = [], + remembered: Record = {}, +): ListSource { + return { + states, + serving: (serverId) => serving.includes(serverId), + serverIdOf: (alias) => remembered[alias], + list: (serverId) => lists[serverId], + }; +} + +const summary = (list: ReviewApiSummary[]) => + list.map((review) => [review.title, review.host, review.hostState]); + +it("lists the laptop first, then each machine in the setting's order", () => { + const merged = mergeLists( + "structural", + [entry("laptop")], + source( + [ + { alias: "b", serverId: "B", state: "online" }, + { alias: "a", serverId: "A", state: "online" }, + ], + { A: [entry("on a")], B: [entry("on b")] }, + ["A", "B"], + ), + ); + + expect(summary(merged)).toEqual([ + ["laptop", undefined, undefined], + ["on b", "b", "online"], + ["on a", "a", "online"], + ]); + expect(merged[1]?.available).toEqual({ + sourceWindows: false, + languageFeatures: false, + }); +}); + +it("offers language features and Source windows only for an online host whose Desktop reported language features", () => { + const merged = mergeLists( + "structural", + [], + source( + [ + { alias: "a", serverId: "A", state: "online", languageFeatures: true }, + { alias: "b", serverId: "B", state: "online", languageFeatures: false }, + { alias: "c", serverId: "C", state: "offline", languageFeatures: true }, + ], + { A: [entry("on a")], B: [entry("on b")], C: [entry("on c")] }, + ["A", "B"], + ), + ); + + expect(merged.map((review) => [review.host, review.available])).toEqual([ + ["a", { sourceWindows: true, languageFeatures: true }], + ["b", { sourceWindows: false, languageFeatures: false }], + ["c", { sourceWindows: false, languageFeatures: false }], + ]); +}); + +it("lists one machine under two aliases once, under the first", () => { + const merged = mergeLists( + "structural", + [], + source( + [ + { alias: "first", serverId: "A", state: "online" }, + { alias: "second", serverId: "A", state: "online" }, + ], + { A: [entry("on a")] }, + ["A"], + ), + ); + + expect(summary(merged)).toEqual([["on a", "first", "online"]]); +}); + +it("leaves a duplicate out", () => { + const merged = mergeLists( + "structural", + [], + source([{ alias: "copy", serverId: "C", state: "duplicate" }], { + C: [entry("copied")], + }), + ); + + expect(merged).toEqual([]); +}); + +it("drops entries whose id is not a UUID", () => { + const merged = mergeLists( + "structural", + [], + source( + [{ alias: "a", serverId: "A", state: "online" }], + { + A: [ + entry("pad", { reviewId: "scratchpad" }), + entry("shared", { reviewId: `shared-${"a".repeat(64)}` }), + entry("kept"), + ], + }, + ["A"], + ), + ); + + expect(summary(merged)).toEqual([["kept", "a", "online"]]); +}); + +it("puts the alias in front of the group key and keeps the label", () => { + const [merged] = mergeLists( + "structural", + [], + source( + [{ alias: "a", serverId: "A", state: "online" }], + { + A: [ + entry("grouped", { + repositoryGroup: { key: "git:/srv/r/.git", label: "r" }, + }), + ], + }, + ["A"], + ), + ); + + expect(merged?.repositoryGroup).toEqual({ + key: "a:git:/srv/r/.git", + label: "r", + }); +}); + +it("keeps the last list of a host that is connecting, offline, incompatible or unreachable, with its state", () => { + const merged = mergeLists( + "structural", + [], + source( + [ + { alias: "starting", state: "connecting" }, + { alias: "gone", serverId: "G", state: "offline" }, + { alias: "old", serverId: "O", state: "incompatible" }, + { alias: "unreached", state: "unreachable" }, + { alias: "never", state: "offline" }, + ], + { + S: [entry("starting")], + G: [entry("gone")], + O: [entry("old")], + U: [entry("unreached")], + }, + [], + { starting: "S", unreached: "U" }, + ), + ); + + expect(summary(merged)).toEqual([ + ["starting", "starting", "connecting"], + ["gone", "gone", "offline"], + ["old", "old", "incompatible"], + ["unreached", "unreached", "unreachable"], + ]); +}); + +it("keeps an id with the machine that lists it first", () => { + const shared = entry("mine"); + const conflicts: string[] = []; + + const merged = mergeLists( + "structural", + [shared], + source( + [{ alias: "a", serverId: "A", state: "online" }], + { A: [{ ...shared, title: "theirs" }] }, + ["A"], + ), + (conflict) => conflicts.push(`${conflict.host}:${conflict.reviewId}`), + ); + + expect(summary(merged)).toEqual([["mine", undefined, undefined]]); + expect(conflicts).toEqual([`a:${shared.reviewId}`]); +}); diff --git a/packages/review/src/server/review-gateway-list.ts b/packages/review/src/server/review-gateway-list.ts new file mode 100644 index 000000000..d59a426eb --- /dev/null +++ b/packages/review/src/server/review-gateway-list.ts @@ -0,0 +1,89 @@ +import type { + ReviewApiSummary, + ReviewGatewayHostState, +} from "@dev.fast/review-protocol"; + +import { UUID } from "./review-gateway-hosts.js"; +import type { ListMode } from "./review-gateway-memory.js"; + +export interface ListSource { + states: readonly ReviewGatewayHostState[]; + serving(serverId: string): boolean; + serverIdOf(alias: string): string | undefined; + list(serverId: string, mode: ListMode): ReviewApiSummary[] | undefined; +} + +function decorate( + entry: ReviewApiSummary, + state: ReviewGatewayHostState, + hostState: NonNullable, +): ReviewApiSummary { + const { alias } = state; + const features = hostState === "online" && state.languageFeatures === true; + + return { + ...entry, + ...(entry.repositoryGroup && { + repositoryGroup: { + key: `${alias}:${entry.repositoryGroup.key}`, + label: entry.repositoryGroup.label, + }, + }), + host: alias, + hostState, + available: { sourceWindows: features, languageFeatures: features }, + }; +} + +const hostStateOf = ( + state: ReviewGatewayHostState, + serving: boolean, +): NonNullable => { + if (serving) return "online"; + + return state.state === "online" || state.state === "duplicate" + ? "offline" + : state.state; +}; + +function remoteEntries(mode: ListMode, source: ListSource) { + const entries: ReviewApiSummary[] = []; + const seen = new Set(); + + for (const state of source.states) { + if (state.state === "duplicate") continue; + const serverId = state.serverId ?? source.serverIdOf(state.alias); + + if (serverId === undefined || seen.has(serverId)) continue; + seen.add(serverId); + const hostState = hostStateOf(state, source.serving(serverId)); + + for (const entry of source.list(serverId, mode) ?? []) + if (UUID.test(entry.reviewId)) + entries.push(decorate(entry, state, hostState)); + } + + return entries; +} + +export function mergeLists( + mode: ListMode, + laptop: readonly ReviewApiSummary[], + source: ListSource, + conflict: (entry: ReviewApiSummary) => void = () => {}, +) { + const ids = new Set(laptop.map((entry) => entry.reviewId)); + const merged = [...laptop]; + + for (const entry of remoteEntries(mode, source)) { + if (ids.has(entry.reviewId)) { + conflict(entry); + continue; + } + + ids.add(entry.reviewId); + merged.push(entry); + } + + return merged; +} diff --git a/packages/review/src/server/review-gateway-memory.test.ts b/packages/review/src/server/review-gateway-memory.test.ts new file mode 100644 index 000000000..5e096e58c --- /dev/null +++ b/packages/review/src/server/review-gateway-memory.test.ts @@ -0,0 +1,135 @@ +import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { afterEach, beforeEach, expect, it } from "vitest"; + +import { + gatewayMemoryPath, + openGatewayMemory, +} from "./review-gateway-memory.js"; + +let home: string; + +beforeEach(async () => { + home = await mkdtemp(path.join(tmpdir(), "review-gateway-memory-")); +}); + +afterEach(async () => { + await rm(home, { recursive: true, force: true }); +}); + +it("keeps each server's reviews under its last alias in an owner-only file", async () => { + const memory = openGatewayMemory(home); + memory.remember("server-1", "devbox", "review-1"); + memory.remember("server-1", "devbox", "review-2"); + memory.remember("server-2", "other", "review-3"); + await memory.flush(); + + expect((await stat(gatewayMemoryPath(home))).mode & 0o777).toBe(0o600); + + const reopened = openGatewayMemory(home); + expect(reopened.owner("review-2")).toEqual({ + serverId: "server-1", + alias: "devbox", + }); + expect(reopened.owner("review-3")).toEqual({ + serverId: "server-2", + alias: "other", + }); + expect(reopened.owner("review-4")).toBeUndefined(); +}); + +it("loses nothing when a server's alias is renamed", async () => { + const memory = openGatewayMemory(home); + memory.remember("server-1", "devbox", "review-1"); + memory.remember("server-1", "renamed", "review-2"); + await memory.flush(); + + const reopened = openGatewayMemory(home); + expect(reopened.owner("review-1")).toEqual({ + serverId: "server-1", + alias: "renamed", + }); + expect(JSON.parse(await readFile(gatewayMemoryPath(home), "utf8"))).toEqual({ + "server-1": { alias: "renamed", reviewIds: ["review-1", "review-2"] }, + }); +}); + +it("forgets a review its owner no longer has", async () => { + const memory = openGatewayMemory(home); + memory.remember("server-1", "devbox", "review-1"); + memory.remember("server-1", "devbox", "review-2"); + memory.forget("review-1"); + await memory.flush(); + + expect(openGatewayMemory(home).owner("review-1")).toBeUndefined(); + expect(openGatewayMemory(home).owner("review-2")).toBeDefined(); +}); + +it("starts empty from an unreadable file and says so", async () => { + await writeFile(gatewayMemoryPath(home), "{not json"); + const messages: string[] = []; + const memory = openGatewayMemory(home, (message) => messages.push(message)); + + expect(memory.owner("review-1")).toBeUndefined(); + expect(messages).toEqual([ + expect.stringContaining("Ignoring unreadable remote review memory"), + ]); +}); + +it("keeps each server's last list, and knows the owner of every review in it", async () => { + const entry = { + reviewId: "review-1", + version: 1, + title: "Listed", + createdAt: "2026-09-30T00:00:00.000Z", + repositoryName: "project", + viewedAt: null, + dismissedAt: null, + }; + + const memory = openGatewayMemory(home); + memory.setList("server-1", "devbox", "structural", [entry]); + memory.setList("server-1", "devbox", "structural", [ + { ...entry, title: "Renamed" }, + ]); + await memory.flush(); + + const reopened = openGatewayMemory(home); + expect(reopened.list("server-1", "structural")).toEqual([ + { ...entry, title: "Renamed" }, + ]); + expect(reopened.list("server-1", "textual")).toBeUndefined(); + expect(reopened.owner("review-1")).toEqual({ + serverId: "server-1", + alias: "devbox", + }); + expect(reopened.serverIdOf("devbox")).toBe("server-1"); +}); + +it("gives an id two lists hold to the server first in the given order, and follows a renamed alias", async () => { + const entry = { + reviewId: "review-1", + version: 1, + title: "Listed twice", + createdAt: "2026-09-30T00:00:00.000Z", + repositoryName: "project", + viewedAt: null, + dismissedAt: null, + }; + + const memory = openGatewayMemory(home); + memory.setList("server-1", "one", "structural", [entry]); + memory.setList("server-2", "two", "structural", [entry]); + + expect(memory.owner("review-1", ["server-2", "server-1"])?.serverId).toBe( + "server-2", + ); + expect(memory.owner("review-1", ["server-1", "server-2"])?.serverId).toBe( + "server-1", + ); + + memory.setList("server-2", "renamed", "structural", [entry]); + expect(memory.alias("server-2")).toBe("renamed"); +}); diff --git a/packages/review/src/server/review-gateway-memory.ts b/packages/review/src/server/review-gateway-memory.ts new file mode 100644 index 000000000..955597e9e --- /dev/null +++ b/packages/review/src/server/review-gateway-memory.ts @@ -0,0 +1,214 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; + +import type { ReviewApiSummary } from "@dev.fast/review-protocol"; +import { errorMessage, writePrivateJsonAtomic } from "@dev.fast/trace-core"; +import { z } from "zod"; + +export type ListMode = "structural" | "textual"; + +const listEntrySchema = z.looseObject({ + reviewId: z.string(), + version: z.number(), + title: z.string(), + createdAt: z.string(), + repositoryName: z.string(), + viewedAt: z.string().nullable(), + dismissedAt: z.string().nullable(), + repositoryGroup: z.object({ key: z.string(), label: z.string() }).optional(), +}); + +export const listEntriesSchema = z.array(z.unknown()).transform( + (entries) => + // SAFETY: the schema checks the fields the gateway and Home rely on. + entries.filter( + (entry) => listEntrySchema.safeParse(entry).success, + ) as ReviewApiSummary[], +); + +const lastListSchema = z.object({ + structural: listEntriesSchema.optional(), + textual: listEntriesSchema.optional(), +}); + +const memorySchema = z.record( + z.string(), + z.object({ + alias: z.string(), + reviewIds: z.array(z.string()), + lastList: lastListSchema.optional(), + }), +); + +interface Server { + alias: string; + reviewIds: Set; + lastList: Partial>; +} + +export function gatewayMemoryPath(home: string) { + return path.join(home, "remotes.json"); +} + +export function openGatewayMemory( + home: string, + log: (message: string) => void = () => {}, +) { + const file = gatewayMemoryPath(home); + const servers = new Map(); + const owners = new Map(); + const listed = new Map>(); + + const index = () => { + listed.clear(); + + for (const [serverId, server] of servers) + for (const list of Object.values(server.lastList)) + for (const entry of list) { + let serverIds = listed.get(entry.reviewId); + + if (!serverIds) listed.set(entry.reviewId, (serverIds = new Set())); + serverIds.add(serverId); + } + }; + + try { + const saved = memorySchema.parse(JSON.parse(readFileSync(file, "utf8"))); + + for (const [serverId, { alias, reviewIds, lastList }] of Object.entries( + saved, + )) { + servers.set(serverId, { + alias, + reviewIds: new Set(reviewIds), + lastList: lastList ?? {}, + }); + + for (const id of reviewIds) owners.set(id, serverId); + } + + index(); + } catch (error) { + // SAFETY: fs rejects with a Node ErrnoException carrying `code`. + if ((error as NodeJS.ErrnoException).code !== "ENOENT") + log(`Ignoring unreadable remote review memory ${file}: ${String(error)}`); + } + + let writing = Promise.resolve(); + let queued = false; + + const save = () => { + if (queued) return; + queued = true; + + writing = writing + .then(() => { + queued = false; + + return writePrivateJsonAtomic( + file, + Object.fromEntries( + [...servers].map(([serverId, { alias, reviewIds, lastList }]) => [ + serverId, + { + alias, + reviewIds: [...reviewIds], + ...(Object.keys(lastList).length > 0 && { lastList }), + }, + ]), + ), + ); + }) + .catch((cause: unknown) => + log(`Could not save remote review memory: ${errorMessage(cause)}`), + ); + }; + + return { + owner(reviewId: string, order: readonly string[] = []) { + const candidates = listed.get(reviewId); + + const serverId = + owners.get(reviewId) ?? + (candidates && + (order.find((candidate) => candidates.has(candidate)) ?? + [...candidates][0])); + + const server = serverId && servers.get(serverId); + + return server ? { serverId, alias: server.alias } : undefined; + }, + serverIdOf(alias: string) { + for (const [serverId, server] of servers) + if (server.alias === alias) return serverId; + + return undefined; + }, + list: (serverId: string, mode: ListMode) => + servers.get(serverId)?.lastList[mode], + setList( + serverId: string, + alias: string, + mode: ListMode, + reviews: ReviewApiSummary[], + ) { + let server = servers.get(serverId); + + if (!server) { + server = { alias, reviewIds: new Set(), lastList: {} }; + servers.set(serverId, server); + } else if ( + server.alias === alias && + JSON.stringify(server.lastList[mode]) === JSON.stringify(reviews) + ) + return; + + server.alias = alias; + server.lastList[mode] = reviews; + index(); + save(); + }, + alias: (serverId: string) => servers.get(serverId)?.alias, + rename(serverId: string, alias: string) { + const server = servers.get(serverId); + + if (!server || server.alias === alias) return; + server.alias = alias; + save(); + }, + remember(serverId: string, alias: string, reviewId: string) { + const server = servers.get(serverId); + + if (server?.alias === alias && owners.get(reviewId) === serverId) return; + + const previous = owners.get(reviewId); + + if (previous !== undefined && previous !== serverId) + servers.get(previous)?.reviewIds.delete(reviewId); + + if (server) { + server.alias = alias; + server.reviewIds.add(reviewId); + } else + servers.set(serverId, { + alias, + reviewIds: new Set([reviewId]), + lastList: {}, + }); + + owners.set(reviewId, serverId); + save(); + }, + forget(reviewId: string) { + const serverId = owners.get(reviewId); + + if (serverId === undefined) return; + owners.delete(reviewId); + servers.get(serverId)?.reviewIds.delete(reviewId); + save(); + }, + flush: () => writing, + }; +} + +export type GatewayMemory = ReturnType; diff --git a/packages/review/src/server/review-gateway-pushes.test.ts b/packages/review/src/server/review-gateway-pushes.test.ts new file mode 100644 index 000000000..c49a70d53 --- /dev/null +++ b/packages/review/src/server/review-gateway-pushes.test.ts @@ -0,0 +1,690 @@ +import { randomUUID } from "node:crypto"; +import { mkdtemp, rm } from "node:fs/promises"; +import type { IncomingMessage, ServerResponse } from "node:http"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; + +import { + type JsonValue, + REVIEW_HOST_HEADER, + ReviewApiClient, + type ReviewVerbRequest, + parseReviewDesktopVerbFrame, +} from "@dev.fast/review-protocol"; +import { readReviewPackageVersion } from "@review/package-paths.js"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; + +import { GlobalReviewDesktopVerbRelay } from "./global-verb-relay.js"; +import { + seed, + startFake, + startGateway, + startRemote, + stopAll, +} from "./review-gateway-test-utils.js"; + +const version = readReviewPackageVersion(import.meta.url); + +// Another host on the same home: the store picks the review up from the database. +function copyReview(from: string, to: string, title: string) { + const db = new DatabaseSync(path.join(root, "laptop", "review-api.db")); + + try { + db.prepare( + "INSERT INTO reviews SELECT ?,version,next_id FROM reviews WHERE id=?", + ).run(to, from); + db.prepare( + "INSERT INTO versions SELECT ?,version,json_set(snapshot,'$.reviewId',?,'$.title',?) FROM versions WHERE review_id=?", + ).run(to, to, title, from); + } finally { + db.close(); + } +} + +let root: string; + +const detaches: (() => void)[] = []; + +beforeEach(async () => { + root = await mkdtemp(path.join(tmpdir(), "review-pushes-")); + vi.stubEnv("DEV_REVIEW_HOME", root); + vi.stubEnv("DEV_FAST_REVIEW_TELEMETRY_DISABLED", "1"); +}); + +afterEach(async () => { + for (const detach of detaches.splice(0)) detach(); + await stopAll(); + vi.unstubAllEnvs(); + await rm(root, { recursive: true, force: true }); +}); + +const OPENED: JsonValue = { ok: true, result: { softwareMapEnabled: false } }; + +function attachWindow( + relay: GlobalReviewDesktopVerbRelay, + answer: (request: ReviewVerbRequest) => Promise | JsonValue = () => + OPENED, +) { + const abort = new AbortController(); + const received: ReviewVerbRequest[] = []; + + relay.attach({ + signal: abort.signal, + write(frame) { + const { id, request } = parseReviewDesktopVerbFrame( + JSON.parse(frame.slice("data: ".length)), + ); + + received.push(request); + void Promise.resolve(answer(request)).then((response) => + relay.acceptResult({ id, response }), + ); + }, + close() {}, + }); + + detaches.push(() => abort.abort()); + + return received; +} + +const opened = (received: ReviewVerbRequest[]) => + received.flatMap((request) => + request.name === "openApiReview" ? [request.args.reviewId] : [], + ); + +it("opens a review a remote creates with open: true, and knows its owner before the window asks", async () => { + const a = await startRemote(path.join(root, "a")); + const relay = new GlobalReviewDesktopVerbRelay({ maxClients: 1 }); + + const laptop = await startGateway( + root, + [{ alias: "wb-a", endpoint: a.endpoint }], + { relay }, + ); + + const first: { status: number; host: string | null }[] = []; + + const received = attachWindow(relay, async (request) => { + if (request.name === "openApiReview") { + const response = await laptop.request( + `/${request.args.reviewId}?full=true`, + ); + + first.push({ + status: response.status, + host: response.headers.get(REVIEW_HOST_HEADER), + }); + } + + return OPENED; + }); + + await expect.poll(async () => (await a.health()).desktopAttached).toBe(true); + + expect(await a.api("/capabilities")).toMatchObject({ + desktopAvailable: true, + }); + expect(received.map((request) => request.name)).toContain( + "authoringCapabilities", + ); + + const reviewId = await seed(a.api, root, "Pushed", { open: true }); + + expect(opened(received)).toEqual([reviewId]); + expect(first).toEqual([{ status: 200, host: "wb-a" }]); + expect( + laptop.localPaths.filter( + (entry) => entry === `/reviews-api/${reviewId}/activity`, + ), + ).toHaveLength(2); +}); + +async function pushingRemote(frames: JsonValue[] = []) { + const results: JsonValue[] = []; + const controls: IncomingMessage[] = []; + const streams: ServerResponse[] = []; + + const fake = await startFake({ + version, + handle(request, response) { + if (request.url === "/control") { + controls.push(request); + response.setHeader("content-type", "text/event-stream"); + response.write(": attached\n\n"); + streams.push(response); + + for (const frame of frames) + response.write(`data: ${JSON.stringify(frame)}\n\n`); + + return true; + } + + if (request.url === "/control/result") { + let body = ""; + request.on("data", (chunk: Buffer) => (body += chunk.toString())); + request.on("end", () => { + results.push(JSON.parse(body)); + response.setHeader("content-type", "application/json"); + response.end('{"ok":true}'); + }); + + return true; + } + + return false; + }, + }); + + const push = (frame: JsonValue) => + streams.at(-1)?.write(`data: ${JSON.stringify(frame)}\n\n`); + + return { fake, results, controls, push }; +} + +const verb = (id: string, request: JsonValue) => ({ + event: "desktop-verb", + id, + request, +}); + +it("refuses a remote's push to open the scratchpad or a shared review", async () => { + const shared = `shared-${"a".repeat(64)}`; + + const remote = await pushingRemote([ + { + event: "desktop-verb", + id: "pad", + request: { + name: "openApiReview", + args: { reviewId: "scratchpad", title: "Pad" }, + }, + }, + { + event: "desktop-verb", + id: "shared", + request: { + name: "openApiReview", + args: { reviewId: shared, title: "S" }, + }, + }, + ]); + + const relay = new GlobalReviewDesktopVerbRelay({ maxClients: 1 }); + const received = attachWindow(relay); + + await startGateway( + root, + [{ alias: "wb-a", endpoint: remote.fake.endpoint }], + { + relay, + }, + ); + + await vi.waitFor(() => expect(remote.results).toHaveLength(2)); + expect(remote.results).toEqual( + expect.arrayContaining([ + { + id: "pad", + response: { ok: false, error: expect.stringContaining("scratchpad") }, + }, + { + id: "shared", + response: { ok: false, error: expect.stringContaining(shared) }, + }, + ]), + ); + expect(received).toEqual([]); +}); + +it("sends a push to every Desktop attached to the remote", async () => { + const a = await startRemote(path.join(root, "a")); + const windows: ReviewVerbRequest[][] = []; + + for (const name of ["one", "two"]) { + const relay = new GlobalReviewDesktopVerbRelay({ maxClients: 1 }); + windows.push(attachWindow(relay)); + await startGateway(root, [{ alias: "wb-a", endpoint: a.endpoint }], { + relay, + home: path.join(root, name), + }); + } + + await expect + .poll(async () => { + await a.api("/capabilities"); + + return windows.map((received) => + received.some((request) => request.name === "authoringCapabilities"), + ); + }) + .toEqual([true, true]); + + const reviewId = await seed(a.api, root, "For both", { open: true }); + + await vi.waitFor(() => + expect(windows.map(opened)).toEqual([[reviewId], [reviewId]]), + ); +}); + +it("attaches again when a remote drops its /control stream, without a storm", async () => { + const remote = await pushingRemote([]); + const relay = new GlobalReviewDesktopVerbRelay({ maxClients: 1 }); + attachWindow(relay); + + const laptop = await startGateway( + root, + [{ alias: "wb-a", endpoint: remote.fake.endpoint }], + { relay }, + ); + + await vi.waitFor(() => expect(remote.controls).toHaveLength(1)); + + remote.controls[0]!.socket.destroy(); + + await vi.waitFor(() => expect(remote.controls).toHaveLength(2), { + timeout: 5_000, + }); + expect(laptop.gateway.hosts()[0]?.state).toBe("online"); + + await new Promise((resolve) => setTimeout(resolve, 1_500)); + expect(remote.controls).toHaveLength(2); +}); + +it("refuses every verb from a remote but capabilities, focus and opening a review", async () => { + const remote = await pushingRemote([ + verb("shot", { name: "captureScreenshot", args: {} }), + verb("home", { + name: "openReview", + args: { reviewUuid: randomUUID(), active: true }, + }), + ]); + + const relay = new GlobalReviewDesktopVerbRelay({ maxClients: 1 }); + const received = attachWindow(relay); + + await startGateway( + root, + [{ alias: "wb-a", endpoint: remote.fake.endpoint }], + { + relay, + }, + ); + + await vi.waitFor(() => expect(remote.results).toHaveLength(2)); + expect(remote.results).toEqual( + expect.arrayContaining([ + { + id: "shot", + response: { + ok: false, + error: "captureScreenshot is not available from another machine.", + }, + }, + { + id: "home", + response: { + ok: false, + error: "openReview is not available from another machine.", + }, + }, + ]), + ); + expect(received).toEqual([]); +}); + +it("refuses a push to open another machine's review or the laptop's", async () => { + const a = await startRemote(path.join(root, "a")); + const onA = await seed(a.api, root, "On a"); + const b = await pushingRemote(); + const relay = new GlobalReviewDesktopVerbRelay({ maxClients: 1 }); + const received = attachWindow(relay); + + const laptop = await startGateway( + root, + [ + { alias: "wb-a", endpoint: a.endpoint }, + { alias: "wb-b", endpoint: b.fake.endpoint }, + ], + { relay }, + ); + + const onLaptop = await seed(laptop.api, root, "On the laptop"); + + const hostOf = async () => + (await laptop.request(`/${onA}`)).headers.get(REVIEW_HOST_HEADER); + + await expect.poll(hostOf).toBe("wb-a"); + await vi.waitFor(() => expect(b.controls).toHaveLength(1)); + + b.push( + verb("a", { name: "openApiReview", args: { reviewId: onA, title: "A" } }), + ); + b.push( + verb("laptop", { + name: "openApiReview", + args: { reviewId: onLaptop, title: "L" }, + }), + ); + + await vi.waitFor(() => expect(b.results).toHaveLength(2)); + expect(b.results).toEqual( + expect.arrayContaining([ + { id: "a", response: { ok: false, error: `${onA} belongs to wb-a.` } }, + { + id: "laptop", + response: { ok: false, error: `${onLaptop} belongs to the laptop.` }, + }, + ]), + ); + expect(received).toEqual([]); + expect(await hostOf()).toBe("wb-a"); +}); + +it("attaches to remotes only while a window is attached to the laptop", async () => { + const a = await startRemote(path.join(root, "a")); + const relay = new GlobalReviewDesktopVerbRelay({ maxClients: 1 }); + + const laptop = await startGateway( + root, + [{ alias: "wb-a", endpoint: a.endpoint }], + { relay }, + ); + + await expect.poll(() => laptop.gateway.hosts()[0]?.state).toBe("online"); + await new Promise((resolve) => setTimeout(resolve, 500)); + expect((await a.health()).desktopAttached).toBe(false); + expect(await a.api("/capabilities")).toMatchObject({ + desktopAvailable: false, + }); + + const unopened = await seed(a.api, root, "Not opened", { open: true }); + expect(unopened).toEqual(expect.any(String)); + + const received = attachWindow(relay); + + await expect.poll(async () => (await a.health()).desktopAttached).toBe(true); + expect(await a.api("/capabilities")).toMatchObject({ + desktopAvailable: true, + }); + + const opened = await seed(a.api, root, "Opened", { open: true }); + expect( + received.flatMap((request) => + request.name === "openApiReview" ? [request.args.reviewId] : [], + ), + ).toEqual([opened]); + + detaches.splice(0).forEach((detach) => detach()); + await expect.poll(async () => (await a.health()).desktopAttached).toBe(false); +}); + +it("keeps a local review on the laptop when a remote lists its id, for every route", async () => { + const relay = new GlobalReviewDesktopVerbRelay({ maxClients: 1 }); + const received = attachWindow(relay); + const laptop = await startGateway(root, [], { relay }); + const onLaptop = await seed(laptop.api, root, "Local original"); + const remoteOnly = randomUUID(); + + const summary = (reviewId: string, title: string) => ({ + reviewId, + version: 1, + title, + createdAt: new Date(0).toISOString(), + repositoryName: "project", + viewedAt: null, + dismissedAt: null, + }); + + const impostor = { ...summary(onLaptop, "Remote impostor"), document: [] }; + const results: JsonValue[] = []; + let control: ServerResponse | undefined; + + const fake = await startFake({ + version, + reviewIds: [onLaptop, remoteOnly], + handle(request, response) { + if (request.url?.startsWith("/reviews-api/watch")) { + response.setHeader("content-type", "application/x-ndjson"); + response.write( + `${JSON.stringify({ + kind: "list", + mode: "structural", + reviews: [ + summary(onLaptop, "Remote impostor"), + summary(remoteOnly, "Remote only"), + ], + })}\n`, + ); + response.write( + `${JSON.stringify({ kind: "review", reviewId: onLaptop, value: impostor })}\n`, + ); + + return true; + } + + if (request.url === "/control") { + response.setHeader("content-type", "text/event-stream"); + response.write(": attached\n\n"); + control = response; + + return true; + } + + if (request.url === "/control/result") { + let body = ""; + request.on("data", (chunk: Buffer) => (body += chunk.toString())); + request.on("end", () => { + results.push(JSON.parse(body)); + response.end('{"ok":true}'); + }); + + return true; + } + + return false; + }, + }); + + laptop.gateway.setHosts([{ alias: "wb-a", endpoint: fake.endpoint }]); + + await expect + .poll(async () => + (await laptop.request(`/${remoteOnly}`)).headers.get(REVIEW_HOST_HEADER), + ) + .toBe("wb-a"); + + const read = await laptop.request(`/${onLaptop}?full=true`); + expect(read.headers.has(REVIEW_HOST_HEADER)).toBe(false); + expect(await read.json()).toMatchObject({ title: "Local original" }); + + const list = ( + await laptop.api<{ reviewId: string; title: string; host?: string }[]>("") + ).filter((entry) => entry.reviewId !== "scratchpad"); + + expect(list.map(({ title, host }) => ({ title, host }))).toEqual([ + { title: "Local original", host: undefined }, + { title: "Remote only", host: "wb-a" }, + ]); + + const client = new ReviewApiClient({ + serverUrl: laptop.url, + token: "laptop-token", + }); + + const abort = new AbortController(); + detaches.push(() => abort.abort()); + + for await (const line of client.watch( + [{ reviewId: onLaptop, mode: "structural" }], + abort.signal, + )) { + expect(line).toMatchObject({ + kind: "review", + reviewId: onLaptop, + value: { title: "Local original" }, + }); + break; + } + + const renamed = await laptop.request("/commands", { + method: "POST", + body: JSON.stringify({ + operation: { type: "rename", reviewId: onLaptop, title: "Renamed" }, + }), + }); + + expect(renamed.status).toBe(200); + expect(renamed.headers.has(REVIEW_HOST_HEADER)).toBe(false); + expect(laptop.local.store.summary(onLaptop)?.title).toBe("Renamed"); + + control?.write( + `data: ${JSON.stringify(verb("push", { name: "openApiReview", args: { reviewId: onLaptop, title: "I" } }))}\n\n`, + ); + await vi.waitFor(() => expect(results).toHaveLength(1)); + expect(results).toEqual([ + { + id: "push", + response: { ok: false, error: `${onLaptop} belongs to the laptop.` }, + }, + ]); + expect(received).toEqual([]); + + expect( + fake.requests.filter((request) => request.url?.includes(onLaptop)), + ).toEqual([]); + expect( + laptop.logged.filter((line) => line.includes(`${onLaptop} is also listed`)), + ).toHaveLength(1); + + copyReview(onLaptop, remoteOnly, "Imported"); + + expect( + (await laptop.api<{ reviewId: string; host?: string }[]>("")).find( + (entry) => entry.reviewId === remoteOnly, + ), + ).not.toHaveProperty("host"); + expect( + (await laptop.request(`/${remoteOnly}?full=true`)).headers.has( + REVIEW_HOST_HEADER, + ), + ).toBe(false); +}); + +it("routes an id to the laptop as soon as the laptop gains it, after a remote served it", async () => { + const laptop = await startGateway(root, []); + const local = await seed(laptop.api, root, "Local"); + const reviewId = randomUUID(); + + const fake = await startFake({ + version, + reviewIds: [reviewId], + handle(request, response) { + if (!request.url?.startsWith("/reviews-api/watch")) return false; + response.setHeader("content-type", "application/x-ndjson"); + response.write( + `${JSON.stringify({ + kind: "list", + mode: "structural", + reviews: [ + { + reviewId, + version: 1, + title: "Remote", + createdAt: new Date(0).toISOString(), + repositoryName: "project", + viewedAt: null, + dismissedAt: null, + }, + ], + })}\n`, + ); + + return true; + }, + }); + + laptop.gateway.setHosts([{ alias: "wb-a", endpoint: fake.endpoint }]); + + const hostOf = async () => + (await laptop.request(`/${reviewId}?full=true`)).headers.get( + REVIEW_HOST_HEADER, + ); + + await expect.poll(hostOf).toBe("wb-a"); + + copyReview(local, reviewId, "Now local"); + + const read = await laptop.request(`/${reviewId}?full=true`); + expect(read.headers.has(REVIEW_HOST_HEADER)).toBe(false); + expect(await read.json()).toMatchObject({ title: "Now local" }); + + const renamed = await laptop.request("/commands", { + method: "POST", + body: JSON.stringify({ + operation: { type: "rename", reviewId, title: "Renamed here" }, + }), + }); + + expect(renamed.headers.has(REVIEW_HOST_HEADER)).toBe(false); + expect(laptop.local.store.summary(reviewId)?.title).toBe("Renamed here"); +}); + +it("follows a local review on the laptop when its first request is a watch and a remote lists it", async () => { + const laptop = await startGateway(root, []); + const local = await seed(laptop.api, root, "Local"); + const other = randomUUID(); + + const summary = (reviewId: string, title: string) => ({ + reviewId, + version: 1, + title, + createdAt: new Date(0).toISOString(), + repositoryName: "project", + viewedAt: null, + dismissedAt: null, + }); + + const fake = await startFake({ + version, + reviewIds: [local, other], + handle(request, response) { + if (!request.url?.startsWith("/reviews-api/watch")) return false; + response.setHeader("content-type", "application/x-ndjson"); + response.write( + `${JSON.stringify({ kind: "list", mode: "structural", reviews: [summary(local, "Impostor"), summary(other, "Other")] })}\n`, + ); + + return true; + }, + }); + + laptop.gateway.setHosts([{ alias: "wb-a", endpoint: fake.endpoint }]); + + await expect + .poll(async () => + (await laptop.request(`/${other}`)).headers.get(REVIEW_HOST_HEADER), + ) + .toBe("wb-a"); + + const client = new ReviewApiClient({ + serverUrl: laptop.url, + token: "laptop-token", + }); + + const abort = new AbortController(); + detaches.push(() => abort.abort()); + + for await (const line of client.watch( + [{ reviewId: local, mode: "structural" }], + abort.signal, + )) { + expect(line).toMatchObject({ reviewId: local, value: { title: "Local" } }); + break; + } + + expect( + fake.requests.filter((request) => request.url?.includes(local)), + ).toEqual([]); +}); diff --git a/packages/review/src/server/review-gateway-pushes.ts b/packages/review/src/server/review-gateway-pushes.ts new file mode 100644 index 000000000..3fc83b087 --- /dev/null +++ b/packages/review/src/server/review-gateway-pushes.ts @@ -0,0 +1,142 @@ +import { + type ReviewVerbRequest, + type ReviewVerbResponse, + parseJsonText, + parseReviewDesktopVerbFrame, +} from "@dev.fast/review-protocol"; + +import type { ReviewDesktopVerbRelay } from "./global-verb-relay.js"; +import { + FIRST_BYTE_TIMEOUT_MS, + type GatewayHosts, + type GatewayRemote, + UUID, + errorText, + remoteHeaders, + send, +} from "./review-gateway-hosts.js"; +import { keepOpen, readLines } from "./review-gateway-transport.js"; + +const REMOTE_VERBS = new Set([ + "authoringCapabilities", + "openApiReview", + "focusWindow", +]); + +export function createGatewayPushes(input: { + hosts: GatewayHosts; + relay: ReviewDesktopVerbRelay; + claim(remote: GatewayRemote, reviewId: string): Promise; + log(message: string): void; +}) { + const { hosts, relay } = input; + const links = new Map(); + + async function answer( + remote: GatewayRemote, + request: ReviewVerbRequest, + ): Promise { + if (!REMOTE_VERBS.has(request.name)) + return { + ok: false, + error: `${request.name} is not available from another machine.`, + }; + + if (request.name === "openApiReview") { + const { reviewId } = request.args; + + if (!UUID.test(reviewId)) + return { + ok: false, + error: `${reviewId} cannot be opened from ${remote.alias}: a review on another machine has a UUID.`, + }; + + const refused = await input.claim(remote, reviewId); + + if (refused) return { ok: false, error: refused }; + } + + return relay.dispatch(request); + } + + async function reply( + remote: GatewayRemote, + id: string, + response: ReviewVerbResponse, + ) { + try { + const answered = await send(remote, { + method: "POST", + path: "/control/result", + headers: { + ...remoteHeaders(remote), + "content-type": "application/json", + }, + body: Buffer.from(JSON.stringify({ id, response })), + signal: AbortSignal.timeout(FIRST_BYTE_TIMEOUT_MS), + }); + + await answered.body.dump(); + } catch (error) { + input.log( + `Could not answer ${remote.alias}'s push: ${errorText(error)}.`, + ); + } + } + + async function push(remote: GatewayRemote, data: string) { + let frame: ReturnType; + + try { + frame = parseReviewDesktopVerbFrame(parseJsonText(data)); + } catch { + input.log(`Ignored an unreadable push from ${remote.alias}.`); + + return; + } + + let response: ReviewVerbResponse; + + try { + response = await answer(remote, frame.request); + } catch (error) { + response = { ok: false, error: errorText(error) }; + } + + await reply(remote, frame.id, response); + } + + return { + changed() { + const online = new Set(relay.attached ? hosts.online() : []); + + for (const [remote, abort] of links) + if (!online.has(remote)) { + abort.abort(); + links.delete(remote); + } + + for (const remote of online) { + if (links.has(remote)) continue; + const abort = new AbortController(); + links.set(remote, abort); + + keepOpen({ + hosts, + remote, + path: "/control", + signal: abort.signal, + read: (body) => + readLines(body, (line) => { + if (line.startsWith("data: ")) + void push(remote, line.slice("data: ".length)); + }), + }); + } + }, + close() { + for (const abort of links.values()) abort.abort(); + links.clear(); + }, + }; +} diff --git a/packages/review/src/server/review-gateway-streams.test.ts b/packages/review/src/server/review-gateway-streams.test.ts new file mode 100644 index 000000000..3c04f447c --- /dev/null +++ b/packages/review/src/server/review-gateway-streams.test.ts @@ -0,0 +1,938 @@ +import { randomUUID } from "node:crypto"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { type IncomingMessage, get } from "node:http"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { + type JsonObject, + ReviewApiClient, + type ReviewApiSummary, + type ReviewStreamLine, +} from "@dev.fast/review-protocol"; +import { readReviewPackageVersion } from "@review/package-paths.js"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; + +import { gatewayMemoryPath } from "./review-gateway-memory.js"; +import { + seed, + startFake, + startGateway, + startRemote, + stopAll, +} from "./review-gateway-test-utils.js"; + +const version = readReviewPackageVersion(import.meta.url); + +let root: string; + +const stops: (() => void)[] = []; + +beforeEach(async () => { + root = await mkdtemp(path.join(tmpdir(), "review-streams-")); + vi.stubEnv("DEV_REVIEW_HOME", root); + vi.stubEnv("DEV_FAST_REVIEW_TELEMETRY_DISABLED", "1"); +}); + +afterEach(async () => { + for (const stop of stops.splice(0)) stop(); + await stopAll(); + vi.unstubAllEnvs(); + await rm(root, { recursive: true, force: true }); +}); + +type Api = (route: string, init?: RequestInit) => Promise; + +type Subscription = { reviewId: string | null; mode?: "structural" }; + +function follow(url: string, subscriptions: Subscription[]) { + const client = new ReviewApiClient({ serverUrl: url, token: "laptop-token" }); + const abort = new AbortController(); + const lines: ReviewStreamLine[] = []; + const all: ReviewStreamLine[] = []; + let ended = false; + + void (async () => { + for await (const line of client.watch(subscriptions, abort.signal)) { + lines.push(line); + all.push(line); + } + })() + .catch(() => undefined) + .finally(() => { + ended = true; + }); + + stops.push(() => abort.abort()); + + const until = async ( + test: (line: ReviewStreamLine) => boolean, + timeout = 5_000, + ) => { + let found: ReviewStreamLine | undefined; + + await vi.waitFor( + () => { + const index = lines.findIndex(test); + expect(index).toBeGreaterThanOrEqual(0); + found = lines.splice(0, index + 1).at(-1); + }, + { timeout }, + ); + + return found!; + }; + + return { lines, all, until, ended: () => ended }; +} + +const listed = (line: ReviewStreamLine) => + line.kind === "list" ? line.reviews : undefined; + +const entry = (line: ReviewStreamLine, title: string) => + listed(line)?.find((review) => review.title === title); + +const valueOf = (line: ReviewStreamLine, reviewId: string) => + line.kind === "review" && line.reviewId === reviewId && "value" in line + ? line.value + : undefined; + +const errorOf = (line: ReviewStreamLine, reviewId: string) => + line.kind === "review" && line.reviewId === reviewId && "error" in line + ? line.error + : undefined; + +const focusOf = (line: ReviewStreamLine) => + line.kind === "review" && "value" in line + ? ( + line.value.activity as { + activities?: { focus?: { description: string } }[]; + } + ).activities?.[0]?.focus?.description + : undefined; + +const activities = new Map(); + +const step = async (api: Api, reviewId: string, description: string) => { + const activityId = activities.get(reviewId); + + const result = await api<{ activityId?: string }>( + `/${reviewId}/activity/${activityId ? "update" : "begin"}`, + { + method: "POST", + body: JSON.stringify({ activityId, focus: { description } }), + }, + ); + + if (!activityId && result.activityId) + activities.set(reviewId, result.activityId); + + return result; +}; + +const command = (api: Api, operation: JsonObject) => + api("/commands", { + method: "POST", + body: JSON.stringify({ operation }), + }); + +const withoutPad = (reviews: ReviewApiSummary[]) => + reviews.filter((review) => review.kind !== "scratchpad"); + +it("lists the laptop's reviews, then each host's in the setting's order, with host, state and availability", async () => { + const a = await startRemote(path.join(root, "a")); + const b = await startRemote(path.join(root, "b")); + const onA = await seed(a.api, root, "On a"); + await seed(b.api, root, "On b"); + + const laptop = await startGateway(root, [ + { alias: "wb-b", endpoint: b.endpoint }, + { alias: "wb-a", endpoint: a.endpoint }, + ]); + + await seed(laptop.api, root, "On the laptop"); + + await expect + .poll( + async () => + withoutPad(await laptop.api("")).map((review) => [ + review.title, + review.host ?? null, + review.hostState ?? null, + ]), + { timeout: 5_000 }, + ) + .toEqual([ + ["On the laptop", null, null], + ["On b", "wb-b", "online"], + ["On a", "wb-a", "online"], + ]); + + const merged = await laptop.api(""); + + const asSent = (await a.api("")).find( + (review) => review.reviewId === onA, + )!; + + const fromA = merged.find((review) => review.reviewId === onA)!; + const { repositoryGroup, ...rest } = asSent; + + expect(fromA).toEqual({ + ...rest, + repositoryGroup: { + key: `wb-a:${repositoryGroup!.key}`, + label: repositoryGroup!.label, + }, + host: "wb-a", + hostState: "online", + available: { sourceWindows: false, languageFeatures: false }, + }); + + const onLaptop = merged.find((review) => review.title === "On the laptop")!; + expect(onLaptop).not.toHaveProperty("host"); + expect(onLaptop).not.toHaveProperty("hostState"); + expect(onLaptop).not.toHaveProperty("available"); + + const stream = follow(laptop.url, [{ reviewId: null }]); + expect(listed(await stream.until((line) => line.kind === "list"))).toEqual( + merged, + ); +}); + +it("shows create, rename, dismiss and delete on every machine in the list", async () => { + const a = await startRemote(path.join(root, "a")); + const b = await startRemote(path.join(root, "b")); + + const laptop = await startGateway(root, [ + { alias: "wb-a", endpoint: a.endpoint }, + { alias: "wb-b", endpoint: b.endpoint }, + ]); + + await expect + .poll(() => laptop.gateway.hosts().map((host) => host.state)) + .toEqual(["online", "online"]); + + const stream = follow(laptop.url, [{ reviewId: null }]); + + for (const [api, host] of [ + [laptop.api, undefined], + [a.api, "wb-a"], + [b.api, "wb-b"], + ] as const) { + const title = `Made on ${host ?? "the laptop"}`; + const reviewId = await seed(api, root, title); + + expect( + entry(await stream.until((line) => !!entry(line, title)), title)?.host, + ).toBe(host); + + await command(api, { + type: "rename", + reviewId, + title: `${title}, renamed`, + }); + await stream.until((line) => !!entry(line, `${title}, renamed`)); + + await command(api, { type: "attention", reviewId, action: "dismiss" }); + await stream.until( + (line) => !!entry(line, `${title}, renamed`)?.dismissedAt, + ); + + await command(api, { type: "delete", reviewId }); + await stream.until( + (line) => + line.kind === "list" && + !line.reviews.some((review) => review.reviewId === reviewId), + ); + } + + expect(stream.ended()).toBe(false); +}); + +it("sends each machine's edits once, for the right review", async () => { + const a = await startRemote(path.join(root, "a")); + const b = await startRemote(path.join(root, "b")); + const onA = await seed(a.api, root, "On a"); + const onB = await seed(b.api, root, "On b"); + + const laptop = await startGateway(root, [ + { alias: "wb-a", endpoint: a.endpoint }, + { alias: "wb-b", endpoint: b.endpoint }, + ]); + + const onLaptop = await seed(laptop.api, root, "On the laptop"); + + await expect + .poll(() => laptop.gateway.hosts().map((host) => host.state)) + .toEqual(["online", "online"]); + + const stream = follow(laptop.url, [ + { reviewId: onLaptop }, + { reviewId: onA }, + { reviewId: onB }, + ]); + + await vi.waitFor(() => + expect( + stream.lines.map((line) => line.kind === "review" && line.reviewId), + ).toEqual(expect.arrayContaining([onLaptop, onA, onB])), + ); + await new Promise((resolve) => setTimeout(resolve, 300)); + expect( + stream.lines.map((line) => line.kind === "review" && line.reviewId), + ).toHaveLength(3); + stream.lines.splice(0); + + const machines = { [onLaptop]: laptop.api, [onA]: a.api, [onB]: b.api }; + const order = [onA, onLaptop, onB, onB, onA, onLaptop, onA]; + + for (const [index, reviewId] of order.entries()) { + await step(machines[reviewId]!, reviewId, `Step ${index}`); + + await vi.waitFor(() => expect(stream.lines).toHaveLength(1)); + const [line] = stream.lines.splice(0); + expect(line).toMatchObject({ kind: "review", reviewId }); + expect(focusOf(line!)).toBe(`Step ${index}`); + } + + await new Promise((resolve) => setTimeout(resolve, 300)); + expect(stream.lines).toEqual([]); +}); + +async function rawLines(url: string, token: string, reviewId: string) { + const response = await fetch( + `${url}/reviews-api/watch?subscriptions=${encodeURIComponent( + JSON.stringify([{ reviewId }]), + )}`, + { headers: { "x-review-token": token } }, + ); + + const reader = response.body!.getReader(); + stops.push(() => void reader.cancel().catch(() => undefined)); + let pending = Buffer.alloc(0); + + return async () => { + for (;;) { + const end = pending.indexOf(0x0a); + + if (end >= 0) { + const line = pending.subarray(0, end); + pending = pending.subarray(end + 1); + + return Buffer.from(line); + } + + const { value, done } = await reader.read(); + + if (done) throw new Error("The stream ended."); + pending = Buffer.concat([pending, value]); + } + }; +} + +it("passes the laptop's list and stream through as they are while no host is set, then ends the stream when one is", async () => { + const laptop = await startGateway(root, []); + const a = await startRemote(path.join(root, "a")); + await seed(a.api, root, "On a"); + + const watch = `/reviews-api/watch?subscriptions=${encodeURIComponent( + JSON.stringify([{ reviewId: null, mode: "structural" }]), + )}`; + + const firstLine = async (response: Response) => { + const reader = response.body!.getReader(); + stops.push(() => void reader.cancel().catch(() => undefined)); + let text = ""; + + while (!text.includes("\n")) + text += Buffer.from((await reader.read()).value!).toString(); + + return { line: text.slice(0, text.indexOf("\n")), reader }; + }; + + const through = await firstLine( + await fetch(`${laptop.url}${watch}`, { + headers: { "x-review-token": "laptop-token" }, + }), + ); + + expect(through.line).toBe((await firstLine(await laptop.direct(watch))).line); + expect(await (await laptop.request("")).text()).toBe( + await (await laptop.direct("/reviews-api")).text(), + ); + + laptop.gateway.setHosts([{ alias: "wb-a", endpoint: a.endpoint }]); + + for (;;) if ((await through.reader.read()).done) break; + + const merged = follow(laptop.url, [{ reviewId: null }]); + await merged.until((line) => entry(line, "On a")?.host === "wb-a"); +}); + +it("forwards a remote's review line byte for byte", async () => { + const a = await startRemote(path.join(root, "a")); + const onA = await seed(a.api, root, "On a"); + + const laptop = await startGateway(root, [ + { alias: "wb-a", endpoint: a.endpoint }, + ]); + + await expect.poll(() => laptop.gateway.hosts()[0]?.state).toBe("online"); + + const direct = await rawLines(a.endpoint.url, a.endpoint.token, onA); + const through = await rawLines(laptop.url, "laptop-token", onA); + await direct(); + await through(); + + await step(a.api, onA, "Bytes"); + const sent = await direct(); + expect(sent.toString()).toContain("Bytes"); + expect((await through()).equals(sent)).toBe(true); +}); + +it("marks a stopped host's review and list entries, keeps the others live, and resumes when it returns", async () => { + const a = await startRemote(path.join(root, "a")); + const b = await startRemote(path.join(root, "b")); + const onA = await seed(a.api, root, "On a"); + const onB = await seed(b.api, root, "On b"); + + const laptop = await startGateway(root, [ + { alias: "wb-a", endpoint: a.endpoint }, + { alias: "wb-b", endpoint: b.endpoint }, + ]); + + const onLaptop = await seed(laptop.api, root, "On the laptop"); + + await expect + .poll(() => laptop.gateway.hosts().map((host) => host.state)) + .toEqual(["online", "online"]); + + const stream = follow(laptop.url, [ + { reviewId: null }, + { reviewId: onLaptop }, + { reviewId: onA }, + { reviewId: onB }, + ]); + + await stream.until((line) => entry(line, "On b")?.hostState === "online"); + await vi.waitFor(() => + expect(stream.lines.some((line) => !!valueOf(line, onB))).toBe(true), + ); + + await b.stop(); + + await vi.waitFor(() => { + expect(stream.all.some((line) => !!errorOf(line, onB))).toBe(true); + expect( + stream.all.some((line) => entry(line, "On b")?.hostState === "offline"), + ).toBe(true); + }); + expect(stream.all.flatMap((line) => errorOf(line, onB) ?? [])).toEqual([ + expect.stringContaining("wb-b is offline"), + ]); + stream.lines.splice(0); + + await step(a.api, onA, "A while b is down"); + await stream.until((line) => focusOf(line) === "A while b is down"); + await step(laptop.api, onLaptop, "Laptop while b is down"); + await stream.until((line) => focusOf(line) === "Laptop while b is down"); + + const again = await startRemote(path.join(root, "b")); + expect(again.endpoint.url).not.toBe(b.endpoint.url); + + laptop.gateway.setHosts([ + { alias: "wb-a", endpoint: a.endpoint }, + { alias: "wb-b", endpoint: again.endpoint }, + ]); + + const since = stream.all.length; + + await vi.waitFor(() => { + const lines = stream.all.slice(since); + + expect(lines.map((line) => valueOf(line, onB)).find(Boolean)).toMatchObject( + { title: "On b" }, + ); + expect( + lines.some((line) => entry(line, "On b")?.hostState === "online"), + ).toBe(true); + }); + + await step(again.api, onB, "B is back"); + await stream.until((line) => focusOf(line) === "B is back"); + expect(stream.all.filter((line) => !!errorOf(line, onB))).toHaveLength(1); + expect(stream.ended()).toBe(false); +}, 30_000); + +it("lists a host's last reviews as connecting, then offline, after a restart while it is unreachable", async () => { + const a = await startRemote(path.join(root, "a")); + const b = await startRemote(path.join(root, "b")); + await seed(a.api, root, "On a"); + await seed(b.api, root, "On b"); + + const hosts = [ + { alias: "wb-a", endpoint: a.endpoint }, + { alias: "wb-b", endpoint: b.endpoint }, + ]; + + const first = await startGateway(root, hosts); + + await expect + .poll(async () => + (await first.api("")).map((review) => review.title), + ) + .toEqual(expect.arrayContaining(["On a", "On b"])); + await first.close(); + await b.stop(); + + const hung = await startFake({ version, handle: () => true }); + + const second = await startGateway(root, [ + hosts[0]!, + { alias: "wb-b", endpoint: hung.endpoint }, + ]); + + expect( + (await second.api("")) + .filter((review) => review.host === "wb-b") + .map((review) => [review.title, review.hostState]), + ).toEqual([["On b", "connecting"]]); + + await expect + .poll( + async () => + withoutPad(await second.api("")).map((review) => [ + review.title, + review.host, + review.hostState, + ]), + { timeout: 5_000 }, + ) + .toEqual([ + ["On a", "wb-a", "online"], + ["On b", "wb-b", "offline"], + ]); +}); + +it("leaves a remote's scratchpad and shared reviews out of the list", async () => { + const kept = randomUUID(); + + const summary = (reviewId: string, title: string) => ({ + reviewId, + version: 1, + title, + createdAt: new Date(0).toISOString(), + repositoryName: "project", + viewedAt: null, + dismissedAt: null, + }); + + const fake = await startFake({ + version, + handle(request, response) { + if (!request.url?.startsWith("/reviews-api/watch")) return false; + response.setHeader("content-type", "application/x-ndjson"); + response.write( + `${JSON.stringify({ + kind: "list", + mode: "structural", + reviews: [ + summary("scratchpad", "Remote pad"), + summary(`shared-${"a".repeat(64)}`, "Remote shared"), + summary(kept, "Remote review"), + ], + })}\n`, + ); + + return true; + }, + }); + + const laptop = await startGateway(root, [ + { alias: "wb-a", endpoint: fake.endpoint }, + ]); + + await expect + .poll(async () => + (await laptop.api("")) + .filter((review) => review.host) + .map((review) => review.reviewId), + ) + .toEqual([kept]); +}); + +it("keeps the laptop's and other hosts' lines coming while a host hangs", async () => { + const hungId = randomUUID(); + + const hung = await startFake({ + version, + reviewIds: [hungId], + handle: (request) => + request.url?.startsWith("/reviews-api/watch") || + request.url === "/control", + }); + + const a = await startRemote(path.join(root, "a")); + const onA = await seed(a.api, root, "On a"); + + const laptop = await startGateway(root, [ + { alias: "wb-h", endpoint: hung.endpoint }, + { alias: "wb-a", endpoint: a.endpoint }, + ]); + + const onLaptop = await seed(laptop.api, root, "On the laptop"); + + await expect + .poll(() => laptop.gateway.hosts().map((host) => host.state)) + .toEqual(["online", "online"]); + + const started = Date.now(); + + const stream = follow(laptop.url, [ + { reviewId: null }, + { reviewId: hungId }, + { reviewId: onLaptop }, + { reviewId: onA }, + ]); + + await vi.waitFor(() => { + expect(stream.lines.some((line) => !!entry(line, "On a"))).toBe(true); + expect(stream.lines.some((line) => !!valueOf(line, onLaptop))).toBe(true); + expect(stream.lines.some((line) => !!valueOf(line, onA))).toBe(true); + }); + expect(Date.now() - started).toBeLessThan(3_000); + + const error = errorOf( + await stream.until((line) => !!errorOf(line, hungId), 12_000), + hungId, + ); + + expect(error).toContain("wb-h is offline"); + expect(Date.now() - started).toBeLessThan(12_000); +}, 20_000); + +it("gives a reader that stops reading the latest state once, not a backlog", async () => { + const a = await startRemote(path.join(root, "a")); + const onA = await seed(a.api, root, "On a"); + + await command(a.api, { + type: "edit", + reviewId: onA, + edit: { + type: "insert", + content: { type: "markdown", markdown: "x".repeat(900 * 1024) }, + }, + }); + + const laptop = await startGateway(root, [ + { alias: "wb-a", endpoint: a.endpoint }, + ]); + + await expect.poll(() => laptop.gateway.hosts()[0]?.state).toBe("online"); + + const response = await new Promise((resolve, reject) => + get( + `${laptop.url}/reviews-api/watch?subscriptions=${encodeURIComponent( + JSON.stringify([{ reviewId: onA }]), + )}`, + { headers: { "x-review-token": "laptop-token" } }, + resolve, + ).on("error", reject), + ); + + stops.push(() => response.destroy()); + response.setEncoding("utf8"); + const lines: string[] = []; + let pending = ""; + + response.on("data", (chunk: string) => { + pending += chunk; + let end: number; + + while ((end = pending.indexOf("\n")) !== -1) { + lines.push(pending.slice(0, end)); + pending = pending.slice(end + 1); + } + }); + + await vi.waitFor(() => expect(lines).toHaveLength(1)); + response.pause(); + const updates = 40; + + for (let index = 1; index <= updates; index++) + await step(a.api, onA, `Step ${index}`); + + response.resume(); + const steps: string[] = []; + + await vi.waitFor( + () => { + for (const line of lines.splice(0)) + steps.push(focusOf(JSON.parse(line)) ?? ""); + expect(steps.at(-1)).toBe(`Step ${updates}`); + }, + { timeout: 10_000 }, + ); + expect(steps.filter((value) => value === `Step ${updates}`)).toHaveLength(1); + expect(steps.length).toBeLessThan(updates / 2); +}, 30_000); + +it("sends nothing for a review whose host is still connecting, then its value", async () => { + const serverId = randomUUID(); + const reviewId = randomUUID(); + const home = path.join(root, "laptop"); + + await mkdir(home, { recursive: true }); + await writeFile( + gatewayMemoryPath(home), + JSON.stringify({ [serverId]: { alias: "wb-b", reviewIds: [reviewId] } }), + ); + + const value = { reviewId, title: "On b", activity: {}, coverageRevision: 0 }; + + const b = await startFake({ + version, + serverId, + reviewIds: [reviewId], + handle(request, response) { + if (request.url === "/health") { + setTimeout( + () => + response.writeHead(200, { "content-type": "application/json" }).end( + JSON.stringify({ + ok: true, + serverId, + instanceId: "b", + serverPid: 1, + desktopAttached: false, + version, + commit: null, + }), + ), + 1_500, + ); + + return true; + } + + if ( + !request.url?.includes(reviewId) || + !request.url.startsWith("/reviews-api/watch") + ) + return false; + response.setHeader("content-type", "application/x-ndjson"); + response.write( + `${JSON.stringify({ kind: "review", reviewId, value })}\n`, + ); + + return true; + }, + }); + + const laptop = await startGateway(root, [ + { alias: "wb-b", endpoint: b.endpoint }, + ]); + + expect(laptop.gateway.hosts()[0]?.state).toBe("connecting"); + const stream = follow(laptop.url, [{ reviewId }]); + + await new Promise((resolve) => setTimeout(resolve, 1_000)); + expect(stream.all).toEqual([]); + + const line = await stream.until((next) => !!valueOf(next, reviewId)); + expect(valueOf(line, reviewId)).toMatchObject({ title: "On b" }); + expect(stream.all.filter((next) => !!errorOf(next, reviewId))).toEqual([]); +}); + +it("finds a host that stops answering while its streams are open", async () => { + let hang = false; + + const hung = await startFake({ + version, + handle(request, response) { + if (hang) return true; + + if (!request.url?.startsWith("/reviews-api/watch")) return false; + response.setHeader("content-type", "application/x-ndjson"); + response.flushHeaders(); + + return true; + }, + }); + + const a = await startRemote(path.join(root, "a")); + const onA = await seed(a.api, root, "On a"); + + const laptop = await startGateway( + root, + [ + { alias: "wb-h", endpoint: hung.endpoint }, + { alias: "wb-a", endpoint: a.endpoint }, + ], + { heartbeatMs: 1_000 }, + ); + + const onLaptop = await seed(laptop.api, root, "On the laptop"); + + await expect + .poll(() => laptop.gateway.hosts().map((host) => host.state)) + .toEqual(["online", "online"]); + await vi.waitFor(() => + expect( + hung.requests.some((request) => + request.url?.startsWith("/reviews-api/watch"), + ), + ).toBe(true), + ); + + hang = true; + const started = Date.now(); + + await vi.waitFor( + () => expect(laptop.gateway.hosts()[0]?.state).toBe("offline"), + { timeout: 6_000 }, + ); + expect(Date.now() - started).toBeLessThan(1_000 + 3_000 + 500); + + const quick = Date.now(); + expect((await laptop.request(`/${onA}`)).status).toBe(200); + expect((await laptop.request(`/${onLaptop}`)).status).toBe(200); + expect(Date.now() - quick).toBeLessThan(1_000); +}, 20_000); + +it("closes a client's streams to remotes when the client leaves", async () => { + const reviewId = randomUUID(); + const closed = Promise.withResolvers(); + let opened = false; + + const fake = await startFake({ + version, + reviewIds: [reviewId], + handle(request, response) { + if ( + !request.url?.startsWith("/reviews-api/watch") || + !request.url.includes(reviewId) + ) + return false; + opened = true; + response.setHeader("content-type", "application/x-ndjson"); + response.write( + `${JSON.stringify({ kind: "review", reviewId, value: { reviewId } })}\n`, + ); + response.on("close", () => closed.resolve()); + + return true; + }, + }); + + const laptop = await startGateway(root, [ + { alias: "wb-a", endpoint: fake.endpoint }, + ]); + + await expect.poll(() => laptop.gateway.hosts()[0]?.state).toBe("online"); + const abort = new AbortController(); + + const response = await fetch( + `${laptop.url}/reviews-api/watch?subscriptions=${encodeURIComponent( + JSON.stringify([{ reviewId }]), + )}`, + { headers: { "x-review-token": "laptop-token" }, signal: abort.signal }, + ); + + const reader = response.body!.getReader(); + await reader.read(); + expect(opened).toBe(true); + + abort.abort(); + await closed.promise; +}); + +it("refuses a remote's review line that carries another review, once in the log", async () => { + const reviewId = randomUUID(); + const other = randomUUID(); + + const fake = await startFake({ + version, + reviewIds: [reviewId], + handle(request, response) { + if (!request.url?.startsWith("/reviews-api/watch")) return false; + response.setHeader("content-type", "application/x-ndjson"); + + for (const id of [other, "scratchpad", other]) + response.write( + `${JSON.stringify({ kind: "review", reviewId, value: { reviewId: id, version: 1 } })}\n`, + ); + + return true; + }, + }); + + const laptop = await startGateway(root, [ + { alias: "wb-a", endpoint: fake.endpoint }, + ]); + + await expect.poll(() => laptop.gateway.hosts()[0]?.state).toBe("online"); + + const stream = follow(laptop.url, [{ reviewId, mode: "structural" }]); + + expect( + errorOf(await stream.until((line) => !!errorOf(line, reviewId)), reviewId), + ).toBe("wb-a answered with another review, so the answer was refused."); + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(stream.all.some((line) => valueOf(line, reviewId))).toBe(false); + expect( + laptop.logged.filter((line) => line.startsWith("Refused wb-a's update")), + ).toEqual([ + `Refused wb-a's update for ${reviewId}: it carried another review.`, + ]); +}); + +it("takes the tutorial marker off a remote's snapshot, in a read and in a line", async () => { + const reviewId = randomUUID(); + + const snapshot = JSON.stringify({ + reviewId, + version: 1, + origin: { tutorial: true, branch: "main" }, + }); + + const fake = await startFake({ + version, + reviewIds: [reviewId], + handle(request, response) { + if (request.url?.startsWith("/reviews-api/watch")) { + response.setHeader("content-type", "application/x-ndjson"); + response.write( + `{"kind":"review","reviewId":"${reviewId}","value":${snapshot}}\n`, + ); + + return true; + } + + if (!request.url?.startsWith(`/reviews-api/${reviewId}?`)) return false; + response.setHeader("content-type", "application/json"); + response.end(snapshot); + + return true; + }, + }); + + const laptop = await startGateway(root, [ + { alias: "wb-a", endpoint: fake.endpoint }, + ]); + + await expect.poll(() => laptop.gateway.hosts()[0]?.state).toBe("online"); + + expect(await laptop.api(`/${reviewId}?full=true`)).toMatchObject({ + reviewId, + origin: { branch: "main" }, + }); + expect( + (await laptop.api<{ origin: object }>(`/${reviewId}?full=true`)).origin, + ).not.toHaveProperty("tutorial"); + + const stream = follow(laptop.url, [{ reviewId, mode: "structural" }]); + const line = await stream.until((next) => !!valueOf(next, reviewId)); + expect(valueOf(line, reviewId)).toMatchObject({ origin: { branch: "main" } }); + expect(valueOf(line, reviewId)?.origin).not.toHaveProperty("tutorial"); +}); diff --git a/packages/review/src/server/review-gateway-streams.ts b/packages/review/src/server/review-gateway-streams.ts new file mode 100644 index 000000000..04bdfe812 --- /dev/null +++ b/packages/review/src/server/review-gateway-streams.ts @@ -0,0 +1,530 @@ +import type http from "node:http"; + +import { + type JsonObject, + type ReviewApiSummary, + type ReviewGatewayHostState, + isJsonObject, + parseJsonText, +} from "@dev.fast/review-protocol"; +import { coverageModeSchema } from "@review/review-api/review-progress.js"; +import { z } from "zod"; + +import { + type GatewayHosts, + type GatewayRemote, + UUID, +} from "./review-gateway-hosts.js"; +import { mergeLists } from "./review-gateway-list.js"; +import { + type GatewayMemory, + type ListMode, + listEntriesSchema, +} from "./review-gateway-memory.js"; +import { + chunks, + keepOpen, + readLines, + reconnect, +} from "./review-gateway-transport.js"; +import { serverJson } from "./review-server-core.js"; + +const LIST_MODES: readonly ListMode[] = ["structural", "textual"]; + +const subscriptionsSchema = z.array( + z.strictObject({ + reviewId: z.string().min(1).nullable(), + mode: coverageModeSchema, + }), +); + +const lineSchema = z.discriminatedUnion("kind", [ + z.object({ + kind: z.literal("list"), + mode: z.enum(LIST_MODES), + reviews: z.array(z.unknown()), + }), + z.object({ + kind: z.literal("review"), + reviewId: z.string(), + value: z.unknown().optional(), + }), +]); + +export type Located = + | "laptop" + | { remote: GatewayRemote } + | { down: ReviewGatewayHostState } + | undefined; + +export const downDetail = (down: ReviewGatewayHostState) => + down.detail ?? `${down.alias} is ${down.state}.`; + +export const isSnapshotOf = ( + value: unknown, + reviewId: string, +): value is JsonObject => + UUID.test(reviewId) && isJsonObject(value) && value.reviewId === reviewId; + +export function withoutTutorial(value: JsonObject): JsonObject | undefined { + if (!isJsonObject(value.origin) || !("tutorial" in value.origin)) + return undefined; + const { tutorial: _, ...origin } = value.origin; + + return { ...value, origin }; +} + +const watchPath = (subscriptions: object[]) => + `/reviews-api/watch?subscriptions=${encodeURIComponent(JSON.stringify(subscriptions))}`; + +function parseLine(text: string) { + try { + return lineSchema.safeParse(parseJsonText(text)).data; + } catch { + return undefined; + } +} + +export function createGatewayStreams(input: { + hosts: GatewayHosts; + memory: GatewayMemory; + local(request: Request): Response | Promise; + locate(reviewId: string): Located; + lookup(reviewId: string): Promise; + onLaptop(reviewId: string): void; + log(message: string): void; +}) { + const { hosts, memory } = input; + const feeds = new Map(); + const clients = new Set(); + const passThrough = new Set<() => void>(); + const conflicts = new Set(); + const refused = new Set(); + let onlineKey = ""; + + const merge = (mode: ListMode, laptop: ReviewApiSummary[]) => + mergeLists( + mode, + laptop, + { + states: hosts.states(), + serving: (serverId) => hosts.serving(serverId) !== undefined, + serverIdOf: (alias) => memory.serverIdOf(alias), + list: (serverId, listMode) => memory.list(serverId, listMode), + }, + (entry) => { + if (laptop.some((local) => local.reviewId === entry.reviewId)) + input.onLaptop(entry.reviewId); + + if (conflicts.has(entry.reviewId)) return; + conflicts.add(entry.reviewId); + input.log( + `Review ${entry.reviewId} is also listed by ${entry.host}; the machine listed first keeps it.`, + ); + }, + ); + + function feedLine(remote: GatewayRemote, text: string) { + const line = parseLine(text); + + if (line?.kind !== "list" || remote.serverId === undefined) return; + + memory.setList( + remote.serverId, + hosts.machineAlias(remote.serverId) ?? remote.alias, + line.mode, + listEntriesSchema + .parse(line.reviews) + .filter((entry) => UUID.test(entry.reviewId)), + ); + + for (const client of clients) client.lists(); + } + + function syncFeeds() { + const online = new Set(hosts.online()); + + for (const [remote, abort] of feeds) + if (!online.has(remote)) { + abort.abort(); + feeds.delete(remote); + } + + for (const remote of online) { + if (feeds.has(remote)) continue; + const abort = new AbortController(); + feeds.set(remote, abort); + + keepOpen({ + hosts, + remote, + path: watchPath(LIST_MODES.map((mode) => ({ reviewId: null, mode }))), + signal: abort.signal, + read: (body) => readLines(body, (text) => feedLine(remote, text)), + }); + } + } + + interface Client { + lists(): void; + refresh(): void; + retryUnclaimed(): void; + stop(): void; + } + + function openClient( + subscriptions: z.infer, + signal: AbortSignal, + ): Response { + const modes = new Set(); + const reviews = new Map(); + + for (const { reviewId, mode } of subscriptions) + if (reviewId === null) modes.add(mode); + else if (!reviews.has(reviewId)) reviews.set(reviewId, mode); + + const encoder = new TextEncoder(); + const pending = new Map(); + const sent = new Map(); + const laptopLists = new Map(); + const reportedDown = new Set(); + const resolving = new Set(); + const looked = new Set(); + const unclaimed = new Set(); + + const upstreams = new Map< + "laptop" | GatewayRemote, + { key: string; abort: AbortController } + >(); + + let controller: ReadableStreamDefaultController | undefined; + let flushing = false; + let stopped = false; + + const flush = () => { + if (!controller || !pending.size) return; + + if (controller.desiredSize === null || controller.desiredSize <= 0) + return; + const lines: string[] = []; + + for (const [item, line] of pending) + if (sent.get(item) !== line) { + lines.push(line); + sent.set(item, line); + } + + pending.clear(); + + if (lines.length) + controller.enqueue(encoder.encode(`${lines.join("\n")}\n`)); + }; + + const emit = (item: string, line: string) => { + if (stopped) return; + pending.set(item, line); + + if (flushing) return; + flushing = true; + queueMicrotask(() => { + flushing = false; + flush(); + }); + }; + + const emitList = (mode: ListMode) => { + const laptop = laptopLists.get(mode); + + if (laptop) + emit( + `list:${mode}`, + JSON.stringify({ kind: "list", mode, reviews: merge(mode, laptop) }), + ); + }; + + const emitReview = (reviewId: string, text: string) => { + reportedDown.delete(reviewId); + emit(`review:${reviewId}`, text); + }; + + const down = (reviewId: string, state: ReviewGatewayHostState) => { + if (reportedDown.has(reviewId)) return; + reportedDown.add(reviewId); + emit( + `review:${reviewId}`, + JSON.stringify({ kind: "review", reviewId, error: downDetail(state) }), + ); + }; + + const openLaptop = (ids: Map, abort: AbortSignal) => { + const path = watchPath([ + ...[...modes].map((mode) => ({ reviewId: null, mode })), + ...[...ids].map(([reviewId, mode]) => ({ reviewId, mode })), + ]); + + void reconnect(abort, async () => { + try { + const response = await input.local( + new Request(`http://gateway${path}`), + ); + + if (response.body) + await readLines(chunks(response.body, abort), (text) => { + const line = parseLine(text); + + if (line?.kind === "list" && modes.has(line.mode)) { + // SAFETY: the laptop's own server writes ReviewApiSummary entries. + laptopLists.set(line.mode, line.reviews as ReviewApiSummary[]); + emitList(line.mode); + } else if (line?.kind === "review" && ids.has(line.reviewId)) + emitReview(line.reviewId, text); + }); + } catch {} + }); + }; + + const openRemote = ( + remote: GatewayRemote, + ids: Map, + abort: AbortSignal, + ) => + keepOpen({ + hosts, + remote, + path: watchPath( + [...ids].map(([reviewId, mode]) => ({ reviewId, mode })), + ), + signal: abort, + read: (body) => + readLines(body, (text) => { + const line = parseLine(text); + + if (line?.kind !== "review" || !ids.has(line.reviewId)) return; + const { reviewId, value } = line; + + if (value === undefined) return emitReview(reviewId, text); + + if (isSnapshotOf(value, reviewId)) { + const stripped = withoutTutorial(value); + + return emitReview( + reviewId, + stripped + ? JSON.stringify({ + kind: "review", + reviewId, + value: stripped, + }) + : text, + ); + } + + if (!refused.has(`${remote.alias} ${reviewId}`)) { + refused.add(`${remote.alias} ${reviewId}`); + input.log( + `Refused ${remote.alias}'s update for ${reviewId}: it carried another review.`, + ); + } + + emitReview( + reviewId, + JSON.stringify({ + kind: "review", + reviewId, + error: `${remote.alias} answered with another review, so the answer was refused.`, + }), + ); + }), + }); + + const where = (reviewId: string): Located => + unclaimed.has(reviewId) + ? "laptop" + : looked.has(reviewId) + ? input.locate(reviewId) + : undefined; + + const resolve = (reviewId: string) => { + if (resolving.has(reviewId)) return; + resolving.add(reviewId); + + void input + .lookup(reviewId) + .catch(() => undefined) + .then(() => { + resolving.delete(reviewId); + looked.add(reviewId); + + if (input.locate(reviewId) === undefined) unclaimed.add(reviewId); + client.refresh(); + }); + }; + + const client: Client = { + lists() { + for (const mode of modes) emitList(mode); + }, + refresh() { + if (stopped) return; + client.lists(); + + const wanted = new Map< + "laptop" | GatewayRemote, + Map + >(); + + const want = (key: "laptop" | GatewayRemote, reviewId: string) => { + let ids = wanted.get(key); + + if (!ids) wanted.set(key, (ids = new Map())); + ids.set(reviewId, reviews.get(reviewId)!); + }; + + if (modes.size) wanted.set("laptop", new Map()); + + for (const reviewId of reviews.keys()) { + const located = where(reviewId); + + if (located === undefined) resolve(reviewId); + else if (located === "laptop") want("laptop", reviewId); + else if ("remote" in located) want(located.remote, reviewId); + else if (located.down.state !== "connecting") + down(reviewId, located.down); + } + + for (const [key, upstream] of upstreams) { + const ids = wanted.get(key); + + if (ids && JSON.stringify([...ids]) === upstream.key) continue; + upstream.abort.abort(); + upstreams.delete(key); + } + + for (const [key, ids] of wanted) { + if (upstreams.has(key)) continue; + const abort = new AbortController(); + upstreams.set(key, { key: JSON.stringify([...ids]), abort }); + + if (key === "laptop") openLaptop(ids, abort.signal); + else openRemote(key, ids, abort.signal); + } + }, + retryUnclaimed: () => unclaimed.clear(), + stop() { + if (stopped) return; + stopped = true; + clients.delete(client); + + for (const upstream of upstreams.values()) upstream.abort.abort(); + upstreams.clear(); + + try { + controller?.close(); + } catch {} + }, + }; + + signal.addEventListener("abort", () => client.stop(), { once: true }); + + const body = new ReadableStream({ + start(started) { + controller = started; + clients.add(client); + client.refresh(); + }, + pull: flush, + cancel: () => client.stop(), + }); + + return new Response(body, { + headers: { + "content-type": "application/x-ndjson", + "cache-control": "no-store", + }, + }); + } + + async function passLaptop(request: Request): Promise { + const response = await input.local(request); + + if (!response.body) return response; + let stop = () => {}; + + const { readable, writable } = new TransformStream({ + start(controller) { + stop = () => controller.terminate(); + }, + }); + + passThrough.add(stop); + void response.body + .pipeTo(writable) + .catch(() => undefined) + .finally(() => passThrough.delete(stop)); + + return new Response(readable, response); + } + + return { + watch(request: Request): Response | Promise { + if (!hosts.states().length) return passLaptop(request); + let subscriptions: z.infer; + + try { + subscriptions = subscriptionsSchema.parse( + JSON.parse( + new URL(request.url).searchParams.get("subscriptions") ?? "", + ), + ); + } catch { + return input.local(request); + } + + return openClient(subscriptions, request.signal); + }, + async list(request: Request): Promise { + const response = await input.local(request); + + if (!response.ok || !hosts.states().length) return response; + + const mode = + coverageModeSchema.safeParse( + new URL(request.url).searchParams.get("mode") ?? undefined, + ).data ?? "structural"; + + const laptop: unknown = await response.json(); + + // SAFETY: the laptop's own server answers ReviewApiSummary[]. + return serverJson(200, merge(mode, laptop as ReviewApiSummary[])); + }, + changed() { + syncFeeds(); + + if (hosts.states().length) { + for (const stop of passThrough) stop(); + passThrough.clear(); + } + + const key = JSON.stringify( + hosts.online().map((remote) => [remote.alias, remote.serverId]), + ); + + for (const client of clients) { + if (key !== onlineKey) client.retryUnclaimed(); + client.refresh(); + } + + onlineKey = key; + }, + close() { + for (const abort of feeds.values()) abort.abort(); + feeds.clear(); + + for (const stop of passThrough) stop(); + + for (const client of clients) client.stop(); + }, + }; +} + +export type GatewayStreams = ReturnType; diff --git a/packages/review/src/server/review-gateway-test-utils.ts b/packages/review/src/server/review-gateway-test-utils.ts new file mode 100644 index 000000000..c931285de --- /dev/null +++ b/packages/review/src/server/review-gateway-test-utils.ts @@ -0,0 +1,344 @@ +import { execFileSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { once } from "node:events"; +import { mkdir, writeFile } from "node:fs/promises"; +import { + type IncomingMessage, + type Server, + type ServerResponse, + createServer, +} from "node:http"; +import type { AddressInfo } from "node:net"; +import path from "node:path"; + +import type { + ReviewGatewayHost, + ReviewServerHealthWithToken, +} from "@dev.fast/review-protocol"; +import { getRequestListener } from "@hono/node-server"; +import { readReviewPackageVersion } from "@review/package-paths.js"; +import { createReviewApi } from "@review/review-api/http.js"; +import { openReviewProfile } from "@review/review-api/profile.js"; +import type { Result } from "@review/review-api/store.js"; +import type { ReviewServerDiscovery } from "@review/server-discovery.js"; +import { Hono } from "hono"; + +import { + GlobalReviewDesktopVerbRelay, + type ReviewDesktopVerbRelay, +} from "./global-verb-relay.js"; +import { runHeadlessServer } from "./headless-host.js"; +import { createReviewGateway } from "./review-gateway.js"; + +const version = readReviewPackageVersion(import.meta.url); + +const stops: (() => Promise)[] = []; + +export async function stopAll() { + for (const stop of stops.splice(0).reverse()) + await stop().catch(() => undefined); +} + +export async function startRemote(stateDir: string, port?: number) { + const controller = new AbortController(); + const ready = Promise.withResolvers(); + + const running = runHeadlessServer({ + stateDir, + ...(port !== undefined && { port }), + signal: controller.signal, + onReady: ready.resolve, + }); + + const stop = async () => { + controller.abort(); + await running; + }; + + stops.push(stop); + + const discovery = await Promise.race([ + ready.promise, + running.then(() => { + throw new Error("Server exited before readiness"); + }), + ]); + + const api = async (route: string, init: RequestInit = {}): Promise => { + const response = await fetch(`${discovery.url}/reviews-api${route}`, { + ...init, + headers: { + "content-type": "application/json", + "x-review-token": discovery.token, + }, + }); + + if (!response.ok) + throw new Error(`${route}: ${response.status} ${await response.text()}`); + + // SAFETY: test helper; callers name the shape the route answers. + return (await response.json()) as T; + }; + + const health = async () => + // SAFETY: /health answers ReviewServerHealthWithToken to the token. + (await ( + await fetch(`${discovery.url}/health`, { + headers: { "x-review-token": discovery.token }, + }) + ).json()) as ReviewServerHealthWithToken; + + return { + discovery, + stop, + api, + health, + endpoint: { url: discovery.url, token: discovery.token }, + }; +} + +export type FakeHandler = ( + request: IncomingMessage, + response: ServerResponse, +) => boolean | void; + +export async function startFake( + options: { + version: string; + serverId?: string; + instanceId?: string; + token?: string; + reviewIds?: string[]; + handle?: FakeHandler; + }, + port = 0, +) { + const requests: IncomingMessage[] = []; + const token = options.token ?? "fake-token"; + + const health: ReviewServerHealthWithToken = { + ok: true, + instanceId: options.instanceId ?? randomUUID(), + serverId: options.serverId ?? randomUUID(), + serverPid: process.pid, + desktopAttached: false, + version: options.version, + commit: null, + }; + + const server: Server = createServer((request, response) => { + requests.push(request); + + if (options.handle?.(request, response)) return; + + if (request.url === "/health") { + const { ok, instanceId, desktopAttached, version } = health; + response.setHeader("content-type", "application/json"); + response.end( + JSON.stringify( + request.headers["x-review-token"] === token + ? health + : { ok, instanceId, desktopAttached, version }, + ), + ); + + return; + } + + const owned = options.reviewIds?.find((id) => + request.url?.startsWith(`/reviews-api/${id}`), + ); + + response.statusCode = owned ? 200 : 404; + response.setHeader("content-type", "application/json"); + response.end( + owned + ? JSON.stringify({ reviewId: owned }) + : '{"ok":false,"error":"Review not found."}', + ); + }); + + server.listen(port, "127.0.0.1"); + await once(server, "listening"); + + const stop = async () => { + if (!server.listening) return; + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + }; + + stops.push(stop); + + // SAFETY: a TCP listener's address() is an AddressInfo. + const address = server.address() as AddressInfo; + + return { + requests, + health, + stop, + port: address.port, + endpoint: { url: `http://127.0.0.1:${address.port}`, token }, + }; +} + +export async function startGateway( + root: string, + hosts: ReviewGatewayHost[], + options: { + version?: string; + home?: string; + relay?: ReviewDesktopVerbRelay; + heartbeatMs?: number; + slowRouteMs?: number; + } = {}, +) { + const home = options.home ?? path.join(root, "laptop"); + const local = await openReviewProfile(home, { manageWorkspaces: true }); + await local.store.ensureScratchpad(); + + const laptop = new Hono().route( + "/reviews-api", + createReviewApi( + local.store, + local.data, + undefined, + undefined, + undefined, + () => true, + ), + ); + + const logged: string[] = []; + const localPaths: string[] = []; + const relay = options.relay ?? new GlobalReviewDesktopVerbRelay(); + + const gateway = createReviewGateway({ + local: (request) => { + localPaths.push(new URL(request.url).pathname); + + return laptop.fetch(request); + }, + version: options.version ?? version, + home, + relay, + ...(options.heartbeatMs !== undefined && { + heartbeatMs: options.heartbeatMs, + }), + ...(options.slowRouteMs !== undefined && { + slowRouteMs: options.slowRouteMs, + }), + log: (message) => logged.push(message), + }); + + gateway.setHosts(hosts); + + const server = createServer(getRequestListener(gateway.fetch)); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + + let closed = false; + + const close = async () => { + if (closed) return; + closed = true; + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + await gateway.close(); + await local.data.close(); + await local.store.close(); + }; + + stops.push(close); + + // SAFETY: a TCP listener's address() is an AddressInfo. + const url = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + + const request = (route: string, init: RequestInit = {}) => + fetch(`${url}/reviews-api${route}`, { + ...init, + headers: { + "content-type": "application/json", + "x-review-token": "laptop-token", + ...init.headers, + }, + }); + + const api = async (route: string, init: RequestInit = {}): Promise => { + const response = await request(route, init); + + if (!response.ok) + throw new Error(`${route}: ${response.status} ${await response.text()}`); + + // SAFETY: test helper; callers name the shape the route answers. + return (await response.json()) as T; + }; + + return { + gateway, + local, + direct: (route: string) => + laptop.fetch(new Request(`http://laptop${route}`)), + request, + api, + close, + logged, + localPaths, + url, + relay, + }; +} + +export async function repository(root: string) { + const directory = path.join(root, `repo-${randomUUID()}`); + await mkdir(directory, { recursive: true }); + + const git = (...args: string[]) => + execFileSync("git", ["-C", directory, ...args], { + encoding: "utf8", + }).trim(); + + git("init", "-q", "-b", "main"); + git("config", "user.name", "Review Test"); + git("config", "user.email", "review-test@example.invalid"); + await writeFile(path.join(directory, "example.ts"), "export const a = 1;\n"); + git("add", "."); + git("commit", "-qm", "base"); + const base = git("rev-parse", "HEAD"); + await writeFile(path.join(directory, "example.ts"), "export const a = 2;\n"); + git("commit", "-qam", "head"); + + return { directory, base, head: git("rev-parse", "HEAD") }; +} + +export async function seed( + api: (route: string, init?: RequestInit) => Promise, + root: string, + title: string, + extra: { open?: boolean } = {}, +) { + const repo = await repository(root); + + const registered = await api<{ id: string }>("/repositories", { + method: "POST", + body: JSON.stringify({ path: repo.directory }), + }); + + const created = await api("/commands", { + method: "POST", + body: JSON.stringify({ + operation: { + type: "create", + title, + target: { + kind: "commits", + repositoryId: registered.id, + base: repo.base, + head: repo.head, + }, + ...extra, + }, + }), + }); + + return created.reviewId; +} diff --git a/packages/review/src/server/review-gateway-transport.ts b/packages/review/src/server/review-gateway-transport.ts new file mode 100644 index 000000000..846884308 --- /dev/null +++ b/packages/review/src/server/review-gateway-transport.ts @@ -0,0 +1,119 @@ +import { setTimeout as delay } from "node:timers/promises"; + +import { + FIRST_RETRY_MS, + type GatewayHosts, + type GatewayRemote, + MAX_RETRY_MS, + NO_ANSWER, + errorCode, + jitter, + remoteHeaders, + send, +} from "./review-gateway-hosts.js"; + +const STEADY_MS = 10_000; + +const MAX_LINE_CHARS = 64 * 1024 * 1024; + +const sleep = (ms: number, signal: AbortSignal) => + delay(ms, undefined, { signal }).catch(() => undefined); + +export async function reconnect( + signal: AbortSignal, + attempt: () => Promise, +) { + let wait = FIRST_RETRY_MS; + + while (!signal.aborted) { + const started = Date.now(); + await attempt(); + + if (signal.aborted) return; + + if (Date.now() - started >= STEADY_MS) wait = FIRST_RETRY_MS; + await sleep(jitter(wait), signal); + wait = Math.min(wait * 2, MAX_RETRY_MS); + } +} + +export async function readLines( + body: AsyncIterable, + line: (text: string) => void, +) { + const decoder = new TextDecoder(); + let pending = ""; + + for await (const chunk of body) { + pending += decoder.decode(chunk, { stream: true }); + let end: number; + + while ((end = pending.indexOf("\n")) !== -1) { + line(pending.slice(0, end)); + pending = pending.slice(end + 1); + } + + if (pending.length > MAX_LINE_CHARS) throw new Error("A line is too long."); + } +} + +export function chunks( + body: ReadableStream, + signal: AbortSignal, +): AsyncIterable { + const reader = body.getReader(); + const cancel = () => void reader.cancel().catch(() => undefined); + + return { + async *[Symbol.asyncIterator]() { + signal.addEventListener("abort", cancel, { once: true }); + + if (signal.aborted) cancel(); + + try { + for (;;) { + const { value, done } = await reader.read(); + + if (done) return; + yield value; + } + } finally { + signal.removeEventListener("abort", cancel); + cancel(); + } + }, + }; +} + +export function keepOpen(input: { + hosts: GatewayHosts; + remote: GatewayRemote; + path: string; + signal: AbortSignal; + read(body: AsyncIterable): Promise; +}) { + const { remote, signal } = input; + + void reconnect(signal, async () => { + let timedOut = false; + + try { + const response = await send(remote, { + method: "GET", + path: input.path, + headers: remoteHeaders(remote), + signal, + }); + + if (response.statusCode === 200) await input.read(response.body); + else await response.body.dump(); + } catch (error) { + timedOut = errorCode(error) === "UND_ERR_HEADERS_TIMEOUT"; + } + + if (signal.aborted) return; + + if (timedOut) input.hosts.failed(remote, NO_ANSWER); + else input.hosts.recheck(remote); + }); +} diff --git a/packages/review/src/server/review-gateway.test.ts b/packages/review/src/server/review-gateway.test.ts new file mode 100644 index 000000000..3d7b9b7a7 --- /dev/null +++ b/packages/review/src/server/review-gateway.test.ts @@ -0,0 +1,1564 @@ +import { randomUUID } from "node:crypto"; +import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { + type JsonObject, + REVIEW_HOST_HEADER, + type ReviewGatewayHost, +} from "@dev.fast/review-protocol"; +import { + findReviewPackageRoot, + readReviewPackageVersion, +} from "@review/package-paths.js"; +import { openReviewProfile } from "@review/review-api/profile.js"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; + +import { createGlobalReviewServer } from "./desktop-server.js"; +import { gatewayMemoryPath } from "./review-gateway-memory.js"; +import { + type FakeHandler, + repository as createRepository, + seed as seedReview, + startFake, + startGateway as startLaptopGateway, + startRemote, + stopAll, +} from "./review-gateway-test-utils.js"; + +const version = readReviewPackageVersion(import.meta.url); + +const LAPTOP_TOKEN = "laptop-token"; + +let root: string; + +const cleanups: (() => Promise)[] = []; + +beforeEach(async () => { + root = await mkdtemp(path.join(tmpdir(), "review-gateway-")); + vi.stubEnv("DEV_REVIEW_HOME", root); + vi.stubEnv("DEV_FAST_REVIEW_TELEMETRY_DISABLED", "1"); +}); + +afterEach(async () => { + for (const cleanup of cleanups.splice(0).reverse()) + await cleanup().catch(() => undefined); + await stopAll(); + vi.unstubAllEnvs(); + await rm(root, { recursive: true, force: true }); +}); + +const startGateway = (hosts: ReviewGatewayHost[], gatewayVersion = version) => + startLaptopGateway(root, hosts, { version: gatewayVersion }); + +const repository = () => createRepository(root); + +type Remote = Awaited>; + +const seed = (remote: Remote, title: string) => + seedReview(remote.api, root, title); + +const command = (operation: JsonObject) => JSON.stringify({ operation }); + +it("sends a read, a command and a file request to the machine that owns the review", async () => { + const a = await startRemote(path.join(root, "a")); + const b = await startRemote(path.join(root, "b")); + const onA = await seed(a, "On a"); + const onB = await seed(b, "On b"); + + const { request, gateway, local } = await startGateway([ + { alias: "wb-a", endpoint: a.endpoint }, + { alias: "wb-b", endpoint: b.endpoint }, + ]); + + await expect + .poll(() => gateway.hosts().map((host) => host.state)) + .toEqual(["online", "online"]); + + const readA = await request(`/${onA}?full=true`); + expect(readA.status).toBe(200); + expect(readA.headers.get(REVIEW_HOST_HEADER)).toBe("wb-a"); + expect(await readA.json()).toMatchObject({ reviewId: onA, title: "On a" }); + + const readB = await request(`/${onB}?full=true`); + expect(readB.headers.get(REVIEW_HOST_HEADER)).toBe("wb-b"); + expect(await readB.json()).toMatchObject({ title: "On b" }); + + const renamed = await request("/commands", { + method: "POST", + body: command({ type: "rename", reviewId: onA, title: "Renamed on a" }), + }); + + expect(renamed.status).toBe(200); + expect(renamed.headers.get(REVIEW_HOST_HEADER)).toBe("wb-a"); + expect(await a.api(`/${onA}?full=true`)).toMatchObject({ + title: "Renamed on a", + }); + + const file = await request(`/${onB}/file?side=head&file=example.ts`); + expect(file.status).toBe(200); + expect(file.headers.get(REVIEW_HOST_HEADER)).toBe("wb-b"); + const body = await file.json(); + expect(body).toMatchObject({ text: "export const a = 2;\n" }); + expect(body).not.toHaveProperty("localPath"); + + const repo = await repository(); + + const registered = await request("/repositories", { + method: "POST", + body: JSON.stringify({ path: repo.directory }), + }); + + expect(registered.headers.has(REVIEW_HOST_HEADER)).toBe(false); + const { id: repositoryId } = await registered.json(); + + const created = await request("/commands", { + method: "POST", + body: command({ + type: "create", + title: "On the laptop", + target: { + kind: "commits", + repositoryId, + base: repo.base, + head: repo.head, + }, + }), + }); + + expect(created.status).toBe(200); + expect(created.headers.has(REVIEW_HOST_HEADER)).toBe(false); + const { reviewId: onLaptop } = await created.json(); + expect(local.store.summary(onLaptop)?.title).toBe("On the laptop"); + + const readLaptop = await request(`/${onLaptop}`); + expect(readLaptop.status).toBe(200); + expect(readLaptop.headers.has(REVIEW_HOST_HEADER)).toBe(false); + + const missing = await request(`/${randomUUID()}`); + expect(missing.status).toBe(404); + expect(missing.headers.has(REVIEW_HOST_HEADER)).toBe(false); +}); + +it("replaces the laptop's token with the remote's and marks the caller remote", async () => { + const reviewId = randomUUID(); + const fake = await startFake({ version, reviewIds: [reviewId] }); + + const { request, gateway } = await startGateway([ + { alias: "wb-a", endpoint: fake.endpoint }, + ]); + + await expect.poll(() => gateway.hosts()[0]?.state).toBe("online"); + + const response = await request( + `/${reviewId}/progress?token=${LAPTOP_TOKEN}`, + { + headers: { "x-review-client": "local" }, + }, + ); + + expect(response.status).toBe(200); + + const forwarded = fake.requests.filter((entry) => + entry.url?.startsWith(`/reviews-api/${reviewId}/progress`), + ); + + expect(forwarded).toHaveLength(1); + const [seen] = forwarded; + expect(seen?.url).toBe(`/reviews-api/${reviewId}/progress`); + expect(seen?.headers["x-review-token"]).toBe(fake.endpoint.token); + expect(seen?.headers["x-review-client"]).toBe("remote"); + expect(JSON.stringify(seen?.headers)).not.toContain(LAPTOP_TOKEN); +}); + +it("answers per-review telemetry for a remote review itself and refuses laptop-only routes", async () => { + const reviewId = randomUUID(); + const fake = await startFake({ version, reviewIds: [reviewId] }); + + const { request, gateway } = await startGateway([ + { alias: "wb-a", endpoint: fake.endpoint }, + ]); + + await expect.poll(() => gateway.hosts()[0]?.state).toBe("online"); + + const telemetry = await request(`/${reviewId}/telemetry/event`, { + method: "POST", + body: "{}", + }); + + expect(telemetry.status).toBe(200); + expect(telemetry.headers.get(REVIEW_HOST_HEADER)).toBe("wb-a"); + + for (const [method, route] of [ + ["POST", "open"], + ["GET", "agent-traces"], + ["GET", "workspaces"], + ["POST", "environment"], + ] as const) { + const refused = await request(`/${reviewId}/${route}`, { + method, + ...(method === "POST" && { body: "{}" }), + }); + + expect(refused.status).toBe(404); + expect(await refused.json()).toMatchObject({ + error: expect.stringContaining( + "not available for a review on another machine", + ), + }); + } + + expect( + fake.requests.filter( + (entry) => + entry.url?.startsWith(`/reviews-api/${reviewId}/`) && + !entry.url.endsWith("/activity"), + ), + ).toEqual([]); +}); + +it("gives a remote review's language context with its path on that machine and its server id", async () => { + const a = await startRemote(path.join(root, "a")); + const onA = await seed(a, "On a"); + + const { request, gateway } = await startGateway([ + { alias: "wb-a", endpoint: a.endpoint }, + ]); + + await expect.poll(() => gateway.hosts()[0]?.state).toBe("online"); + + const read = async () => { + const response = await request(`/${onA}/language-context?side=head`); + expect(response.status).toBe(200); + expect(response.headers.get(REVIEW_HOST_HEADER)).toBe("wb-a"); + + return response.json(); + }; + + await expect + .poll(async () => (await read()).remoteRootPath, { timeout: 20_000 }) + .toEqual(expect.any(String)); + + const context = await read(); + expect(context).toEqual({ + remoteRootPath: expect.any(String), + identity: expect.stringMatching(/^[0-9a-f]{64}$/), + serverId: (await a.health()).serverId, + }); + expect( + await readFile(path.join(context.remoteRootPath, "example.ts"), "utf8"), + ).toBe("export const a = 2;\n"); +}); + +it.each([ + ["a remoteRootPath that is not a string", { remoteRootPath: 7 }], + ["another server's id", { serverId: "another-server" }], +])("refuses a remote language context with %s", async (_, change) => { + const reviewId = randomUUID(); + const serverId = randomUUID(); + + const fake = await startFake({ + version, + serverId, + reviewIds: [reviewId], + handle(request, response) { + if (!request.url?.startsWith(`/reviews-api/${reviewId}/language-context`)) + return false; + response.setHeader("content-type", "application/json"); + response.end( + JSON.stringify({ + remoteRootPath: "/home/dev/repo", + identity: "a".repeat(64), + serverId, + ...change, + }), + ); + + return true; + }, + }); + + const { request, gateway } = await startGateway([ + { alias: "wb-a", endpoint: fake.endpoint }, + ]); + + await expect.poll(() => gateway.hosts()[0]?.state).toBe("online"); + + const response = await request(`/${reviewId}/language-context`); + expect(response.status).toBe(502); + expect(response.headers.get(REVIEW_HOST_HEADER)).toBe("wb-a"); +}); + +it("waits past the 10 s limit for a slow language context and keeps the host online when it never answers", async () => { + const slow = randomUUID(); + const hung = randomUUID(); + const serverId = randomUUID(); + + const fake = await startFake({ + version, + serverId, + reviewIds: [slow, hung], + handle(request, response) { + if (request.url?.startsWith(`/reviews-api/${hung}/language-context`)) + return true; + + if (!request.url?.startsWith(`/reviews-api/${slow}/language-context`)) + return false; + + setTimeout(() => { + response.setHeader("content-type", "application/json"); + response.end( + JSON.stringify({ + remoteRootPath: "/home/dev/repo", + identity: "a", + serverId, + }), + ); + }, 11_000); + + return true; + }, + }); + + const { request, gateway } = await startLaptopGateway( + root, + [{ alias: "wb-a", endpoint: fake.endpoint }], + { slowRouteMs: 14_000 }, + ); + + await expect.poll(() => gateway.hosts()[0]?.state).toBe("online"); + + const [answered, unanswered] = await Promise.all([ + request(`/${slow}/language-context`), + request(`/${hung}/language-context`), + ]); + + expect(answered.status).toBe(200); + expect(await answered.json()).toMatchObject({ + remoteRootPath: "/home/dev/repo", + }); + expect(unanswered.status).toBe(504); + expect(unanswered.headers.get(REVIEW_HOST_HEADER)).toBe("wb-a"); + expect(gateway.hosts()[0]?.state).toBe("online"); + expect((await request(`/${slow}/progress`)).status).toBe(200); +}, 30_000); + +it("streams a remote answer line by line and closes the remote connection when the client leaves", async () => { + const reviewId = randomUUID(); + const next = Promise.withResolvers(); + const closed = Promise.withResolvers(); + + const fake = await startFake({ + version, + reviewIds: [reviewId], + handle(request, response) { + if (!request.url?.startsWith(`/reviews-api/${reviewId}/structural-diff`)) + return false; + response.setHeader("content-type", "application/x-ndjson"); + response.write('{"line":1}\n'); + void next.promise.then(() => response.write('{"line":2}\n')); + response.on("close", () => closed.resolve()); + + return true; + }, + }); + + const { request, gateway } = await startGateway([ + { alias: "wb-a", endpoint: fake.endpoint }, + ]); + + await expect.poll(() => gateway.hosts()[0]?.state).toBe("online"); + + const abort = new AbortController(); + + const response = await request(`/${reviewId}/structural-diff`, { + signal: abort.signal, + }); + + const reader = response + .body!.pipeThrough(new TextDecoderStream()) + .getReader(); + + expect((await reader.read()).value).toBe('{"line":1}\n'); + next.resolve(); + expect((await reader.read()).value).toBe('{"line":2}\n'); + + abort.abort(); + await closed.promise; +}); + +it.each([ + ["file", "GET", "localPath"], + ["file", "GET", "localRoot"], + ["file", "GET", "workspacePath"], + ["file", "GET", "filePath"], + ["language-context", "GET", "rootPath"], + ["ask/agents", "GET", "localPath"], + ["ask/agents/codex/offer", "GET", "localPath"], + ["ask/mentions", "GET", "localPath"], + ["ask/threads", "GET", "localPath"], +])( + "refuses a remote %s answer that carries %s", + async (route, method, field) => { + const reviewId = randomUUID(); + const leaked = "/home/dev/secret/project"; + + const fake = await startFake({ + version, + reviewIds: [reviewId], + handle(request, response) { + if (!request.url?.startsWith(`/reviews-api/${reviewId}/${route}`)) + return false; + response.setHeader("content-type", "application/json"); + response.end( + JSON.stringify({ nested: { [field]: leaked }, text: "x" }), + ); + + return true; + }, + }); + + const { request, gateway, logged } = await startGateway([ + { alias: "wb-a", endpoint: fake.endpoint }, + ]); + + await expect.poll(() => gateway.hosts()[0]?.state).toBe("online"); + + const response = await request(`/${reviewId}/${route}`, { + method, + ...(method === "POST" && { body: "{}" }), + }); + + expect(response.status).toBe(502); + expect(await response.text()).not.toContain(leaked); + expect(logged.join("\n")).toContain(field); + }, +); + +/** A remote whose navigator answers `answers[file]` for `?file=`; `gone` with 409. */ +async function startNavigator( + answers: Record, + serverId = "C0FFEE-42", +) { + const reviewId = randomUUID(); + + const fake = await startFake({ + version, + serverId, + reviewIds: [reviewId], + handle(request, response) { + if (!request.url?.startsWith(`/reviews-api/${reviewId}/navigator`)) + return false; + + const file = new URL(request.url, "http://remote").searchParams.get( + "file", + ); + + response.statusCode = file === "gone" ? 409 : 200; + response.setHeader("content-type", "application/json"); + response.end(JSON.stringify(answers[file ?? ""])); + + return true; + }, + }); + + const started = await startGateway([ + { alias: "wb-a", endpoint: fake.endpoint }, + ]); + + await expect.poll(() => started.gateway.hosts()[0]?.state).toBe("online"); + + const open = (file: string) => + started.request(`/${reviewId}/navigator?file=${file}`, { + method: "POST", + body: "{}", + }); + + return { ...started, open }; +} + +it("answers a remote navigator with vscode-remote URIs on its host", async () => { + const { open } = await startNavigator({ + file: { + workspacePath: "/home/dev/.review/ws/a b.code-workspace", + filePath: "/home/dev/repo (1)/src/a#b+c@d!'*.ts", + }, + empty: { + workspacePath: "/home/dev/.review/ws/x.code-workspace", + emptySide: true, + }, + unicode: { workspacePath: "/home/dév/ws.code-workspace" }, + }); + + const response = await open("file"); + expect(response.status).toBe(200); + expect(response.headers.get(REVIEW_HOST_HEADER)).toBe("wb-a"); + expect(await response.json()).toEqual({ + workspaceUri: + "vscode-remote://whiteboard+c0ffee-42/home/dev/.review/ws/a%20b.code-workspace", + fileUri: + "vscode-remote://whiteboard+c0ffee-42/home/dev/repo%20%281%29/src/a%23b%2Bc%40d%21%27%2A.ts", + remoteAuthority: "whiteboard+c0ffee-42", + }); + + expect(await (await open("empty")).json()).toEqual({ + workspaceUri: + "vscode-remote://whiteboard+c0ffee-42/home/dev/.review/ws/x.code-workspace", + remoteAuthority: "whiteboard+c0ffee-42", + emptySide: true, + }); + + expect(await (await open("unicode")).json()).toMatchObject({ + workspaceUri: + "vscode-remote://whiteboard+c0ffee-42/home/d%C3%A9v/ws.code-workspace", + }); +}); + +it.each([ + ["a relative path", "home/dev/repo/a.ts"], + ["a .. segment", "/home/dev/repo/../../etc/passwd"], + ["a trailing .. segment", "/home/dev/.."], + ["a control character", "/home/dev/repo\n/a.ts"], + ["a DEL character", "/home/dev/repo\u007f/a.ts"], + ["a lone surrogate", "/home/dev/\ud800.ts"], + ["a leading //", "//evil/share/a.ts"], + ["an inner //", "/home/dev//a.ts"], + ["a scheme", "file:///home/dev/a.ts"], + ["a : before the first /", "c:/home/dev/a.ts"], + ["no string", 7], +])( + "refuses a remote navigator answer whose path has %s", + async (_, hostile) => { + const { open, logged } = await startNavigator({ + good: { + workspacePath: "/home/dev/ws.code-workspace", + filePath: "/home/dev/a.ts", + }, + workspace: { workspacePath: hostile }, + file: { workspacePath: "/home/dev/ws.code-workspace", filePath: hostile }, + }); + + expect((await open("good")).status).toBe(200); + + for (const file of ["workspace", "file"]) { + const response = await open(file); + expect(response.status).toBe(502); + expect(response.headers.get(REVIEW_HOST_HEADER)).toBe("wb-a"); + const text = await response.text(); + expect(text).not.toContain("vscode-remote"); + expect(text).not.toContain(String(hostile)); + } + + expect(logged.join("\n")).toContain("/navigator"); + }, +); + +it("refuses a remote navigator answer with another field or an unusable server id", async () => { + const fields = await startNavigator({ + good: { workspacePath: "/home/dev/ws.code-workspace" }, + extra: { + workspacePath: "/home/dev/ws.code-workspace", + localPath: "/home/dev", + }, + }); + + expect((await fields.open("good")).status).toBe(200); + expect((await fields.open("extra")).status).toBe(502); + + const id = await startNavigator( + { good: { workspacePath: "/home/dev/ws.code-workspace" } }, + "not_an+authority", + ); + + expect((await id.open("good")).status).toBe(502); +}); + +it("passes a remote navigator's refusal through", async () => { + const { open } = await startNavigator({ + good: { workspacePath: "/home/dev/ws.code-workspace" }, + gone: { ok: false, error: "The file is gone." }, + }); + + expect((await open("good")).status).toBe(200); + expect(await (await open("gone")).json()).toEqual({ + ok: false, + error: "The file is gone.", + }); +}); + +it("reaches the laptop for the scratchpad and shared reviews, even when a remote has one", async () => { + const fake = await startFake({ + version, + reviewIds: ["scratchpad", `shared-${"a".repeat(64)}`], + }); + + const { request, gateway } = await startGateway([ + { alias: "wb-a", endpoint: fake.endpoint }, + ]); + + await expect.poll(() => gateway.hosts()[0]?.state).toBe("online"); + + const pad = await request("/scratchpad?full=true"); + expect(pad.status).toBe(200); + expect(pad.headers.has(REVIEW_HOST_HEADER)).toBe(false); + expect(await pad.json()).toMatchObject({ reviewId: "scratchpad" }); + + const shared = await request(`/shared-${"a".repeat(64)}`); + expect(shared.headers.has(REVIEW_HOST_HEADER)).toBe(false); + + expect( + fake.requests.filter((entry) => /scratchpad|shared-/.test(entry.url ?? "")), + ).toEqual([]); +}); + +it("answers 503 naming the version to install for a review on another version, also from its memory file", async () => { + const a = await startRemote(path.join(root, "a")); + const onA = await seed(a, "On a"); + + const first = await startGateway([{ alias: "wb-a", endpoint: a.endpoint }]); + await expect.poll(() => first.gateway.hosts()[0]?.state).toBe("online"); + expect((await first.request(`/${onA}`)).status).toBe(200); + await first.close(); + + const second = await startGateway( + [{ alias: "wb-a", endpoint: a.endpoint }], + "0.0.0-other", + ); + + await expect + .poll(() => second.gateway.hosts()[0]?.state) + .toBe("incompatible"); + + const response = await second.request(`/${onA}`); + expect(response.status).toBe(503); + expect(response.headers.get(REVIEW_HOST_HEADER)).toBe("wb-a"); + expect(await response.json()).toMatchObject({ + error: expect.stringContaining("Install Whiteboard 0.0.0-other on wb-a."), + }); +}); + +it("marks a host that stops answering offline within 11 s while others answer", async () => { + const hung = randomUUID(); + let hang = false; + + const fake = await startFake({ + version, + reviewIds: [hung], + handle: () => hang, + }); + + const b = await startRemote(path.join(root, "b")); + const onB = await seed(b, "On b"); + + const { request, gateway, local } = await startGateway([ + { alias: "wb-a", endpoint: fake.endpoint }, + { alias: "wb-b", endpoint: b.endpoint }, + ]); + + await expect + .poll(() => gateway.hosts().map((host) => host.state)) + .toEqual(["online", "online"]); + expect((await request(`/${hung}/progress`)).status).toBe(200); + const repo = await repository(); + + const { id: repositoryId } = await ( + await request("/repositories", { + method: "POST", + body: JSON.stringify({ path: repo.directory }), + }) + ).json(); + + const onLaptop = await local.store.execute({ + operation: { + type: "create", + title: "On the laptop", + target: { + kind: "commits", + repositoryId, + base: repo.base, + head: repo.head, + }, + }, + }); + + hang = true; + const started = Date.now(); + const stuck = request(`/${hung}/progress`); + + await new Promise((resolve) => setTimeout(resolve, 500)); + const quick = Date.now(); + expect((await request(`/${onLaptop.reviewId}`)).status).toBe(200); + expect((await request(`/${onB}`)).status).toBe(200); + expect(Date.now() - quick).toBeLessThan(3_000); + + expect((await stuck).status).toBe(504); + expect(gateway.hosts()[0]?.state).toBe("offline"); + expect(Date.now() - started).toBeLessThan(11_000); + + const refused = await request(`/${hung}/progress`); + expect(refused.status).toBe(503); +}, 20_000); + +it("mounts the gateway in the Desktop server, with host states behind the token", async () => { + const a = await startRemote(path.join(root, "a")); + const onA = await seed(a, "On a"); + + const local = await openReviewProfile(path.join(root, "laptop"), { + manageWorkspaces: true, + }); + + const packageRoot = findReviewPackageRoot(import.meta.url); + + const desktop = createGlobalReviewServer({ + reviewStore: local.store, + reviewData: local.data, + appPid: process.pid, + packageRoot, + toolingRoot: packageRoot, + port: 0, + token: LAPTOP_TOKEN, + discoveryPath: path.join(root, "desktop.json"), + }); + + cleanups.push(async () => { + await desktop.close(); + await local.data.close(); + await local.store.close(); + }); + await desktop.listen(); + + const get = (route: string, headers: Record = {}) => + fetch(`${desktop.url}${route}`, { + headers: { "x-review-token": LAPTOP_TOKEN, ...headers }, + }); + + expect((await fetch(`${desktop.url}/remote-hosts`)).status).toBe(401); + expect(await (await get("/remote-hosts")).json()).toEqual([]); + + desktop.setRemoteHosts([{ alias: "wb-a", endpoint: a.endpoint }]); + await expect + .poll(async () => (await get("/remote-hosts")).json()) + .toEqual([ + { alias: "wb-a", serverId: (await a.health()).serverId, state: "online" }, + ]); + + const read = await get(`/reviews-api/${onA}?full=true`, { + origin: "vscode-file://vscode-app", + }); + + expect(read.status).toBe(200); + expect(read.headers.get(REVIEW_HOST_HEADER)).toBe("wb-a"); + expect(read.headers.get("access-control-expose-headers")).toContain( + REVIEW_HOST_HEADER, + ); + expect(await read.json()).toMatchObject({ title: "On a" }); + + const unknown = await get("/reviews-api/status/unknown"); + expect(unknown.status).toBe(404); + expect(await unknown.json()).toEqual({ ok: false, error: "Not found." }); +}); + +it("never serves a review from a copied store while its machine is down", async () => { + const first = await startRemote(path.join(root, "a")); + const onA = await seed(first, "On a"); + await first.stop(); + await cp(path.join(root, "a"), path.join(root, "c"), { recursive: true }); + const a = await startRemote(path.join(root, "a")); + const c = await startRemote(path.join(root, "c")); + + const { request, gateway } = await startGateway([ + { alias: "wb-a", endpoint: a.endpoint }, + { alias: "wb-c", endpoint: c.endpoint }, + ]); + + await expect + .poll(() => gateway.hosts().map((host) => host.state)) + .toEqual(["online", "duplicate"]); + expect( + (await request(`/${onA}?full=true`)).headers.get(REVIEW_HOST_HEADER), + ).toBe("wb-a"); + + await a.stop(); + const statuses: number[] = []; + + await expect + .poll(async () => { + const { status } = await request(`/${onA}?full=true`); + statuses.push(status); + + return status; + }) + .toBe(503); + expect(statuses.every((status) => status === 502 || status === 503)).toBe( + true, + ); + + const refused = await request(`/${onA}?full=true`); + expect(refused.status).toBe(503); + expect(refused.headers.get(REVIEW_HOST_HEADER)).toBe("wb-a"); + expect(await refused.json()).toMatchObject({ + error: expect.stringContaining("wb-a is offline"), + }); + expect(gateway.hosts().map((host) => host.state)).toEqual([ + "offline", + "duplicate", + ]); +}); + +it("refuses a redirect from a remote and drops its cookies", async () => { + const reviewId = randomUUID(); + + const fake = await startFake({ + version, + reviewIds: [reviewId], + handle(request, response) { + if (request.url?.startsWith(`/reviews-api/${reviewId}/progress`)) { + response.writeHead(302, { location: "/remote-hosts" }).end(); + + return true; + } + + if (request.url?.startsWith(`/reviews-api/${reviewId}/commits`)) { + response + .writeHead(200, { + "content-type": "application/json", + "set-cookie": ["a=1", "b=2"], + }) + .end("[]"); + + return true; + } + + return false; + }, + }); + + const { request, gateway } = await startGateway([ + { alias: "wb-a", endpoint: fake.endpoint }, + ]); + + await expect.poll(() => gateway.hosts()[0]?.state).toBe("online"); + + const redirected = await request(`/${reviewId}/progress`, { + redirect: "manual", + }); + + expect(redirected.status).toBe(502); + expect(redirected.headers.has("location")).toBe(false); + + const commits = await request(`/${reviewId}/commits`); + expect(commits.status).toBe(200); + expect(commits.headers.has("set-cookie")).toBe(false); +}); + +it("forgets a review its owner no longer has", async () => { + const a = await startRemote(path.join(root, "a")); + const onA = await seed(a, "On a"); + + const { request, gateway } = await startGateway([ + { alias: "wb-a", endpoint: a.endpoint }, + ]); + + const memoryFile = gatewayMemoryPath(path.join(root, "laptop")); + + await expect.poll(() => gateway.hosts()[0]?.state).toBe("online"); + expect((await request(`/${onA}?full=true`)).status).toBe(200); + await expect.poll(() => readFile(memoryFile, "utf8")).toContain(onA); + + await a.api("/commands", { + method: "POST", + body: command({ type: "delete", reviewId: onA }), + }); + + const gone = await request(`/${onA}?full=true`); + expect(gone.status).toBe(404); + expect(gone.headers.get(REVIEW_HOST_HEADER)).toBe("wb-a"); + await expect.poll(() => readFile(memoryFile, "utf8")).not.toContain(onA); +}); + +async function freePort() { + const probe = await startFake({ version }); + + await probe.stop(); + + return probe.port; +} + +async function rememberOwner( + serverId: string, + alias: string, + reviewId: string, +) { + const home = path.join(root, "laptop"); + + await mkdir(home, { recursive: true }); + await writeFile( + gatewayMemoryPath(home), + JSON.stringify({ [serverId]: { alias, reviewIds: [reviewId] } }), + ); +} + +const states = (gateway: { hosts(): { state: string }[] }) => + gateway.hosts().map((host) => host.state); + +it.each([ + ["another instance: a is the machine, c the duplicate", false, "duplicate"], + ["the same instance: one machine under two aliases", true, "online"], +])( + "holds a copy as duplicate until the remembered alias reports (%s)", + async (_name, sameInstance, cState) => { + const serverId = randomUUID(); + const reviewId = randomUUID(); + await rememberOwner(serverId, "wb-a", reviewId); + const port = await freePort(); + const c = await startFake({ version, serverId, reviewIds: [reviewId] }); + + const { request, gateway } = await startGateway([ + { + alias: "wb-a", + endpoint: { url: `http://127.0.0.1:${port}`, token: "a" }, + }, + { alias: "wb-c", endpoint: c.endpoint }, + ]); + + await expect.poll(() => states(gateway)).toEqual(["offline", "duplicate"]); + const detail = gateway.hosts()[1]?.detail; + expect(detail).toContain("wb-a"); + expect(detail).toContain("wb-c"); + expect(detail).toContain("whiteboard server reset-id"); + expect(detail).toContain("wb-c is waiting for wb-a"); + + const refused = await request(`/${reviewId}`); + expect(refused.status).toBe(503); + expect(refused.headers.get(REVIEW_HOST_HEADER)).toBe("wb-a"); + expect(await refused.json()).toMatchObject({ + error: expect.stringContaining("wb-a is offline"), + }); + expect(c.requests.some((entry) => entry.url?.includes(reviewId))).toBe( + false, + ); + + await startFake( + { + version, + serverId, + token: "a", + reviewIds: [reviewId], + ...(sameInstance && { instanceId: c.health.instanceId }), + }, + port, + ); + + await expect + .poll(() => states(gateway), { timeout: 5_000 }) + .toEqual(["online", cState]); + const served = await request(`/${reviewId}`); + expect(served.status).toBe(200); + expect(served.headers.get(REVIEW_HOST_HEADER)).toBe("wb-a"); + }, +); + +it("takes a reporting alias as the machine when the remembered alias left the setting", async () => { + const serverId = randomUUID(); + const reviewId = randomUUID(); + await rememberOwner(serverId, "wb-a", reviewId); + const c = await startFake({ version, serverId, reviewIds: [reviewId] }); + + const { request, gateway } = await startGateway([ + { alias: "wb-c", endpoint: c.endpoint }, + ]); + + await expect.poll(() => states(gateway)).toEqual(["online"]); + const served = await request(`/${reviewId}`); + expect(served.status).toBe(200); + expect(served.headers.get(REVIEW_HOST_HEADER)).toBe("wb-c"); + await expect + .poll(async () => + JSON.parse( + await readFile(gatewayMemoryPath(path.join(root, "laptop")), "utf8"), + ), + ) + .toEqual({ [serverId]: { alias: "wb-c", reviewIds: [reviewId] } }); +}); + +function slowHealth( + delayMs: number | undefined, + serverId: string, + instanceId: string, +): FakeHandler { + return (request, response) => { + if (request.url !== "/health") return false; + + if (delayMs !== undefined) + setTimeout( + () => + response.writeHead(200, { "content-type": "application/json" }).end( + JSON.stringify({ + ok: true, + serverId, + instanceId, + serverPid: process.pid, + desktopAttached: false, + version, + commit: null, + }), + ), + delayMs, + ); + + return true; + }; +} + +const memoryOf = async () => + JSON.parse( + await readFile(gatewayMemoryPath(path.join(root, "laptop")), "utf8"), + ); + +it("holds a later alias until an earlier one answers, then setting order decides", async () => { + const serverId = randomUUID(); + const reviewId = randomUUID(); + + const a = await startFake({ + version, + serverId, + reviewIds: [reviewId], + handle: slowHealth(1_000, serverId, randomUUID()), + }); + + const c = await startFake({ version, serverId, reviewIds: [reviewId] }); + + const { request, gateway } = await startGateway([ + { alias: "wb-a", endpoint: a.endpoint }, + { alias: "wb-c", endpoint: c.endpoint }, + ]); + + await expect.poll(() => gateway.hosts()[1]?.serverId).toBe(serverId); + expect(gateway.hosts()[1]).toMatchObject({ + state: "connecting", + detail: expect.stringContaining("wb-a"), + }); + + await expect.poll(() => states(gateway)).toEqual(["online", "duplicate"]); + const served = await request(`/${reviewId}`); + expect(served.headers.get(REVIEW_HOST_HEADER)).toBe("wb-a"); + await expect + .poll(memoryOf) + .toEqual({ [serverId]: { alias: "wb-a", reviewIds: [reviewId] } }); + expect(c.requests.map((entry) => entry.url)).toEqual(["/health"]); +}); + +it("takes a later alias as the machine once an earlier one fails its first check", async () => { + const serverId = randomUUID(); + const reviewId = randomUUID(); + + const a = await startFake({ + version, + serverId, + handle: slowHealth(undefined, serverId, randomUUID()), + }); + + const c = await startFake({ version, serverId, reviewIds: [reviewId] }); + + const { request, gateway } = await startGateway([ + { alias: "wb-a", endpoint: a.endpoint }, + { alias: "wb-c", endpoint: c.endpoint }, + ]); + + await expect.poll(() => gateway.hosts()[1]?.serverId).toBe(serverId); + expect(states(gateway)).toEqual(["connecting", "connecting"]); + await expect + .poll(() => states(gateway), { timeout: 5_000 }) + .toEqual(["offline", "online"]); + expect((await request(`/${reviewId}`)).headers.get(REVIEW_HOST_HEADER)).toBe( + "wb-c", + ); + await expect + .poll(memoryOf) + .toEqual({ [serverId]: { alias: "wb-c", reviewIds: [reviewId] } }); +}); + +it("holds a second alias of the same machine until the first answers", async () => { + const serverId = randomUUID(); + const instanceId = randomUUID(); + const reviewId = randomUUID(); + + const a = await startFake({ + version, + serverId, + instanceId, + reviewIds: [reviewId], + handle: slowHealth(1_000, serverId, instanceId), + }); + + const b = await startFake({ + version, + serverId, + instanceId, + reviewIds: [reviewId], + }); + + const { request, gateway } = await startGateway([ + { alias: "wb-a", endpoint: a.endpoint }, + { alias: "wb-b", endpoint: b.endpoint }, + ]); + + await expect.poll(() => gateway.hosts()[1]?.serverId).toBe(serverId); + expect(gateway.hosts()[1]?.state).toBe("connecting"); + await expect.poll(() => states(gateway)).toEqual(["online", "online"]); + expect((await request(`/${reviewId}`)).headers.get(REVIEW_HOST_HEADER)).toBe( + "wb-a", + ); +}); + +it("without memory, uses a later alias while an earlier one is down, then setting order", async () => { + const serverId = randomUUID(); + const reviewId = randomUUID(); + const port = await freePort(); + const c = await startFake({ version, serverId, reviewIds: [reviewId] }); + + const { request, gateway } = await startGateway([ + { + alias: "wb-a", + endpoint: { url: `http://127.0.0.1:${port}`, token: "a" }, + }, + { alias: "wb-c", endpoint: c.endpoint }, + ]); + + await expect.poll(() => states(gateway)).toEqual(["offline", "online"]); + expect((await request(`/${reviewId}`)).headers.get(REVIEW_HOST_HEADER)).toBe( + "wb-c", + ); + await expect + .poll(memoryOf) + .toEqual({ [serverId]: { alias: "wb-c", reviewIds: [reviewId] } }); + + await startFake( + { version, serverId, token: "a", reviewIds: [reviewId] }, + port, + ); + + await expect + .poll(() => states(gateway), { timeout: 5_000 }) + .toEqual(["online", "duplicate"]); + await expect + .poll(memoryOf) + .toEqual({ [serverId]: { alias: "wb-a", reviewIds: [reviewId] } }); + expect((await request(`/${reviewId}`)).headers.get(REVIEW_HOST_HEADER)).toBe( + "wb-a", + ); +}); + +it("does not hold the first alias behind a later remembered one", async () => { + const serverId = randomUUID(); + const reviewId = randomUUID(); + await rememberOwner(serverId, "wb-c", reviewId); + const a = await startFake({ version, serverId, reviewIds: [reviewId] }); + + const c = await startFake({ + version, + serverId, + handle: slowHealth(1_000, serverId, randomUUID()), + }); + + const { request, gateway } = await startGateway([ + { alias: "wb-a", endpoint: a.endpoint }, + { alias: "wb-c", endpoint: c.endpoint }, + ]); + + await expect.poll(() => states(gateway)).toEqual(["online", "connecting"]); + expect((await request(`/${reviewId}`)).headers.get(REVIEW_HOST_HEADER)).toBe( + "wb-a", + ); + + await expect.poll(() => states(gateway)).toEqual(["online", "duplicate"]); + await expect + .poll(memoryOf) + .toEqual({ [serverId]: { alias: "wb-a", reviewIds: [reviewId] } }); +}); + +it("holds a later remembered alias until the first alias answers", async () => { + const serverId = randomUUID(); + const reviewId = randomUUID(); + await rememberOwner(serverId, "wb-c", reviewId); + + const a = await startFake({ + version, + serverId, + reviewIds: [reviewId], + handle: slowHealth(1_000, serverId, randomUUID()), + }); + + const c = await startFake({ version, serverId, reviewIds: [reviewId] }); + + const { request, gateway } = await startGateway([ + { alias: "wb-a", endpoint: a.endpoint }, + { alias: "wb-c", endpoint: c.endpoint }, + ]); + + await expect.poll(() => gateway.hosts()[1]?.serverId).toBe(serverId); + expect(states(gateway)).toEqual(["connecting", "connecting"]); + const held = await request(`/${reviewId}`); + expect(held.status).toBe(503); + expect(held.headers.get(REVIEW_HOST_HEADER)).toBe("wb-c"); + expect(await held.json()).toMatchObject({ + error: expect.stringContaining("Waiting for wb-a"), + }); + + await expect.poll(() => states(gateway)).toEqual(["online", "duplicate"]); + expect((await request(`/${reviewId}`)).headers.get(REVIEW_HOST_HEADER)).toBe( + "wb-a", + ); + await expect + .poll(memoryOf) + .toEqual({ [serverId]: { alias: "wb-a", reviewIds: [reviewId] } }); + expect(c.requests.some((entry) => entry.url?.includes(reviewId))).toBe(false); +}); + +it("refuses a remote's snapshot of another review, and passes its own byte for byte", async () => { + const reviewId = randomUUID(); + const own = `{"reviewId":"${reviewId}", "title":"Own"}`; + + const fake = await startFake({ + version, + reviewIds: [reviewId], + handle(request, response) { + if (!request.url?.startsWith(`/reviews-api/${reviewId}?`)) return false; + response.setHeader("content-type", "application/json"); + response.end( + request.url.includes("version=") + ? own + : JSON.stringify({ reviewId: "scratchpad", title: "Other" }), + ); + + return true; + }, + }); + + const { request, gateway, logged } = await startGateway([ + { alias: "wb-a", endpoint: fake.endpoint }, + ]); + + await expect.poll(() => gateway.hosts()[0]?.state).toBe("online"); + + const other = await request(`/${reviewId}?full=true`); + expect(other.status).toBe(502); + expect(await other.json()).toEqual({ + ok: false, + error: "wb-a answered with another review, so the answer was refused.", + }); + expect(logged).toContain( + `Refused wb-a's answer for ${reviewId}: it carried another review.`, + ); + + const mine = await request(`/${reviewId}?full=true&version=1`); + expect(mine.status).toBe(200); + expect(await mine.text()).toBe(own); +}); + +it("answers 504 for an answer that stalls after its headers, and keeps the host online", async () => { + const reviewId = randomUUID(); + + const fake = await startFake({ + version, + reviewIds: [reviewId], + handle(request, response) { + if (!request.url?.startsWith(`/reviews-api/${reviewId}/file`)) + return false; + response.writeHead(200, { "content-type": "application/json" }); + response.write('{"text":"'); + + return true; + }, + }); + + const { request, gateway } = await startGateway([ + { alias: "wb-a", endpoint: fake.endpoint }, + ]); + + await expect.poll(() => gateway.hosts()[0]?.state).toBe("online"); + + const started = Date.now(); + const stalled = await request(`/${reviewId}/file?side=head&file=a.ts`); + + expect(stalled.status).toBe(504); + expect(await stalled.json()).toEqual({ + ok: false, + error: "wb-a did not answer: its answer stalled for 10 seconds.", + }); + expect(Date.now() - started).toBeLessThan(11_500); + expect(gateway.hosts()[0]?.state).toBe("online"); +}, 15_000); + +it("passes a whole answer that arrives slowly but steadily, past 10 seconds", async () => { + const reviewId = randomUUID(); + const timers: NodeJS.Timeout[] = []; + + cleanups.push(async () => { + for (const timer of timers) clearInterval(timer); + }); + + const fake = await startFake({ + version, + reviewIds: [reviewId], + handle(request, response) { + if (!request.url?.startsWith(`/reviews-api/${reviewId}/file`)) + return false; + response.writeHead(200, { "content-type": "application/json" }); + response.write('{"text":"'); + let sent = 0; + + const timer = setInterval(() => { + if (++sent <= 12) return void response.write("x"); + clearInterval(timer); + response.end('"}'); + }, 1_000); + + timers.push(timer); + + return true; + }, + }); + + const { request, gateway } = await startGateway([ + { alias: "wb-a", endpoint: fake.endpoint }, + ]); + + await expect.poll(() => gateway.hosts()[0]?.state).toBe("online"); + + const slow = await request(`/${reviewId}/file?side=head&file=a.ts`); + + expect(slow.status).toBe(200); + expect(await slow.json()).toEqual({ text: "x".repeat(12) }); + expect(gateway.hosts()[0]?.state).toBe("online"); +}, 20_000); + +it("ends a forwarded stream when the heartbeat finds its host gone", async () => { + const reviewId = randomUUID(); + let hang = false; + + const fake = await startFake({ + version, + reviewIds: [reviewId], + handle(request, response) { + if (request.url?.startsWith(`/reviews-api/${reviewId}/structural-diff`)) { + response.writeHead(200, { "content-type": "application/x-ndjson" }); + response.write('{"type":"file"}\n'); + + return true; + } + + return hang; + }, + }); + + const laptop = await startLaptopGateway( + root, + [{ alias: "wb-a", endpoint: fake.endpoint }], + { heartbeatMs: 1_000 }, + ); + + await expect.poll(() => laptop.gateway.hosts()[0]?.state).toBe("online"); + + const stream = await laptop.request(`/${reviewId}/structural-diff`); + expect(stream.status).toBe(200); + const read = stream.text(); + + hang = true; + const started = Date.now(); + + await expect(read).rejects.toThrow("terminated"); + expect(laptop.gateway.hosts()[0]?.state).toBe("offline"); + expect(Date.now() - started).toBeLessThan(1_000 + 3_000 + 500); +}, 10_000); + +const ASK_ROUTES: [string, string][] = [ + ["GET", "ask/agents"], + ["GET", "ask/agents/codex/offer"], + ["GET", "ask/mentions?query=f"], + ["GET", "ask/threads"], + ["POST", "ask"], + ["POST", "ask/t1/open"], + ["POST", "ask/t1/prompt"], + ["POST", "ask/t1/permission"], + ["POST", "ask/t1/permissions"], + ["POST", "ask/t1/files"], + ["POST", "ask/t1/choice"], + ["POST", "ask/t1/retry"], + ["POST", "ask/t1/cancel"], + ["POST", "ask/t1/close"], + ["DELETE", "ask/t1"], +]; + +it("forwards every Ask route of a remote review and marks the caller remote", async () => { + const reviewId = randomUUID(); + const seen: string[] = []; + + const fake = await startFake({ + version, + reviewIds: [reviewId], + handle(request, response) { + if (!request.url?.startsWith(`/reviews-api/${reviewId}/ask`)) + return false; + seen.push( + `${request.method} ${request.url.split(`/${reviewId}/`)[1]} ${request.headers["x-review-client"]}`, + ); + response.setHeader("content-type", "application/json"); + response.end('{"ok":true}'); + + return true; + }, + }); + + const { request, gateway } = await startGateway([ + { alias: "wb-a", endpoint: fake.endpoint }, + ]); + + await expect.poll(() => gateway.hosts()[0]?.state).toBe("online"); + + for (const [method, route] of ASK_ROUTES) { + const response = await request(`/${reviewId}/${route}`, { + method, + body: method === "GET" ? undefined : "{}", + }); + + expect(response.status, `${method} ${route}`).toBe(200); + } + + expect(seen).toEqual( + ASK_ROUTES.map(([method, route]) => `${method} ${route} remote`), + ); +}); + +it("keeps an Ask watch stream open across a 12 s silence and ends it when the host goes", async () => { + const reviewId = randomUUID(); + let hang = false; + let remote: import("node:http").ServerResponse | undefined; + + const fake = await startFake({ + version, + reviewIds: [reviewId], + handle(request, response) { + if (request.url === "/health") return hang; + + if (!request.url?.startsWith(`/reviews-api/${reviewId}/ask/t1/watch`)) + return false; + remote = response; + response.setHeader("content-type", "text/event-stream"); + response.write('data: {"n":1}\n\n'); + + return true; + }, + }); + + const { request, gateway } = await startGateway([ + { alias: "wb-a", endpoint: fake.endpoint }, + ]); + + await expect.poll(() => gateway.hosts()[0]?.state).toBe("online"); + + const response = await request(`/${reviewId}/ask/t1/watch`); + expect(response.status).toBe(200); + + const reader = response + .body!.pipeThrough(new TextDecoderStream()) + .getReader(); + + expect((await reader.read()).value).toContain('{"n":1}'); + await new Promise((resolve) => setTimeout(resolve, 12_000)); + remote!.write('data: {"n":2}\n\n'); + expect((await reader.read()).value).toContain('{"n":2}'); + + hang = true; + await expect + .poll(() => gateway.hosts()[0]?.state, { timeout: 15_000 }) + .toBe("offline"); + await expect(reader.read()).rejects.toThrow("terminated"); +}, 40_000); + +it("passes a 2 MB Ask files body to the remote intact", async () => { + const reviewId = randomUUID(); + let received = 0; + + const fake = await startFake({ + version, + reviewIds: [reviewId], + handle(request, response) { + if (!request.url?.startsWith(`/reviews-api/${reviewId}/ask/t1/files`)) + return false; + request.on("data", (chunk: Buffer) => (received += chunk.length)); + request.on("end", () => response.end('{"ok":true}')); + + return true; + }, + }); + + const { request, gateway } = await startGateway([ + { alias: "wb-a", endpoint: fake.endpoint }, + ]); + + await expect.poll(() => gateway.hosts()[0]?.state).toBe("online"); + + const body = JSON.stringify({ paths: ["x".repeat(2 * 1024 * 1024)] }); + + const response = await request(`/${reviewId}/ask/t1/files`, { + method: "POST", + body, + }); + + expect(response.status).toBe(200); + expect(received).toBe(Buffer.byteLength(body)); +}); + +it.each([ + ["GET", "ask/agents/codex/offer"], + ["POST", "ask/t1/permissions"], + ["POST", "ask/t1/choice"], +])( + "waits past the 10 s limit for a slow %s %s and keeps the host online when it never answers", + async (method, route) => { + const slow = randomUUID(); + const hung = randomUUID(); + + const fake = await startFake({ + version, + reviewIds: [slow, hung], + handle(request, response) { + if (request.url?.startsWith(`/reviews-api/${hung}/${route}`)) + return true; + + if (!request.url?.startsWith(`/reviews-api/${slow}/${route}`)) + return false; + + setTimeout(() => { + response.setHeader("content-type", "application/json"); + response.end('{"ok":true}'); + }, 11_000); + + return true; + }, + }); + + const { request, gateway } = await startLaptopGateway( + root, + [{ alias: "wb-a", endpoint: fake.endpoint }], + { slowRouteMs: 14_000 }, + ); + + await expect.poll(() => gateway.hosts()[0]?.state).toBe("online"); + + const init = method === "POST" ? { method, body: "{}" } : {}; + + const [answered, unanswered] = await Promise.all([ + request(`/${slow}/${route}`, init), + request(`/${hung}/${route}`, init), + ]); + + expect(answered.status).toBe(200); + expect(unanswered.status).toBe(504); + expect(unanswered.headers.get(REVIEW_HOST_HEADER)).toBe("wb-a"); + expect(gateway.hosts()[0]?.state).toBe("online"); + }, + 30_000, +); diff --git a/packages/review/src/server/review-gateway.ts b/packages/review/src/server/review-gateway.ts new file mode 100644 index 000000000..f01d845ff --- /dev/null +++ b/packages/review/src/server/review-gateway.ts @@ -0,0 +1,780 @@ +import { Readable } from "node:stream"; +import type { ReadableStream as WebReadableStream } from "node:stream/web"; + +import { + type JsonValue, + REVIEW_CLIENT_HEADER, + REVIEW_HOST_HEADER, + type ReviewGatewayHost, + type ReviewGatewayHostState, + type ReviewRemoteNavigatorAnswer, + isJsonObject, + parseJsonText, +} from "@dev.fast/review-protocol"; +import { Hono } from "hono"; +import type { Dispatcher } from "undici"; +import { z } from "zod"; + +import { StreamLimitError, readBoundedStream } from "./bounded-stream.js"; +import type { ReviewDesktopVerbRelay } from "./global-verb-relay.js"; +import { DEFAULT_MAX_REQUEST_BYTES } from "./http-json.js"; +import { + FIRST_BYTE_TIMEOUT_MS, + type GatewayRemote, + UUID, + createGatewayHosts, + errorCode, + errorText, + readBody, + remoteHeaders, + send, +} from "./review-gateway-hosts.js"; +import { openGatewayMemory } from "./review-gateway-memory.js"; +import { createGatewayPushes } from "./review-gateway-pushes.js"; +import { + type Located, + createGatewayStreams, + downDetail, + isSnapshotOf, + withoutTutorial, +} from "./review-gateway-streams.js"; +import { serverJson } from "./review-server-core.js"; + +const LAPTOP_ROUTES = new Set([ + "repositories", + "resources", + "workspace-cleanup", + "authoring", + "capabilities", + "status", + "instructions", + "sharing", + "commands", +]); + +interface RemoteRoute { + method: string; + path: string; + /** May wait on preparing a checkout or launching an agent. */ + slow?: true; + wholeBody?: true; +} + +const REMOTE_ROUTES: readonly RemoteRoute[] = [ + { method: "GET", path: "" }, + { method: "GET", path: "progress" }, + { method: "POST", path: "progress" }, + { method: "GET", path: "commits" }, + { method: "GET", path: "diff" }, + { method: "GET", path: "structural-diff" }, + { method: "GET", path: "stack" }, + { method: "GET", path: "tree" }, + { method: "GET", path: "file", wholeBody: true }, + { method: "GET", path: "resources/:name" }, + { method: "GET", path: "maps/:name" }, + { method: "POST", path: "navigator", wholeBody: true }, + { method: "POST", path: "copy-context" }, + { method: "GET", path: "language-context", slow: true, wholeBody: true }, + { method: "GET", path: "ask/agents", wholeBody: true }, + { + method: "GET", + path: "ask/agents/:agent/offer", + slow: true, + wholeBody: true, + }, + { method: "GET", path: "ask/mentions", slow: true, wholeBody: true }, + { method: "GET", path: "ask/threads", wholeBody: true }, + { method: "GET", path: "ask/:thread/watch" }, + { method: "POST", path: "ask", slow: true }, + { + method: "POST", + path: "ask/:thread/:action{open|choice|permissions}", + slow: true, + }, + { + method: "POST", + path: "ask/:thread/:action{prompt|permission|files|retry|cancel|close}", + }, + { method: "DELETE", path: "ask/:thread" }, +]; + +const SLOW_ROUTE_TIMEOUT_MS = 120_000; + +const PATH_FIELDS = new Set([ + "localPath", + "rootPath", + "workspacePath", + "filePath", + "localRoot", +]); + +const PATH_ROUTE_MAX_BYTES = 64 * 1024 * 1024; + +const BODY_IDLE_MS = 10_000; + +const BODY_MAX_MS = 120_000; + +const STALLED = `its answer stalled for ${BODY_IDLE_MS / 1_000} seconds`; + +const TOO_LONG = `its answer took longer than ${BODY_MAX_MS / 1_000} seconds`; + +const HOP_HEADERS = new Set([ + "connection", + "keep-alive", + "transfer-encoding", + "content-length", + "te", + "trailer", + "upgrade", +]); + +const DROPPED_RESPONSE_HEADERS = new Set(["location", "set-cookie"]); + +const DROPPED_REQUEST_HEADERS = new Set([ + ...HOP_HEADERS, + "host", + "origin", + "accept-encoding", + "x-review-token", + REVIEW_CLIENT_HEADER, + REVIEW_HOST_HEADER, +]); + +const remoteLanguageContext = z.object({ + remoteRootPath: z.string().nullable(), + serverId: z.string(), +}); + +/** Absolute POSIX, no `..` segment, no control character or lone surrogate, + * no `//`; a `:` before the first `/` (a scheme) is not absolute. */ +const hostPath = z + .string() + .refine( + (value) => + value.startsWith("/") && + value.isWellFormed() && + !value.includes("//") && + !/\p{Cc}/u.test(value) && + !value.split("/").includes(".."), + ); + +const remoteNavigator = z.strictObject({ + workspacePath: hostPath, + filePath: hostPath.optional(), + emptySide: z.literal(true).optional(), +}); + +/** VS Code's `URI.toString()` path encoding: unreserved characters and `/` + * stay, everything else is percent-encoded. */ +const URI_PATH_KEPT = /[A-Za-z0-9\-._~/]/; + +const commandTarget = z.object({ + operation: z.object({ reviewId: z.string().optional() }).optional(), +}); + +type Owner = + | { remote: GatewayRemote } + | { down: ReviewGatewayHostState } + | undefined; + +export function createReviewGateway(input: { + local(request: Request): Response | Promise; + version: string; + home: string; + relay: ReviewDesktopVerbRelay; + heartbeatMs?: number; + slowRouteMs?: number; + restarted?(alias: string): void; + log?(message: string): void; +}) { + const log = input.log ?? (() => {}); + const memory = openGatewayMemory(input.home, log); + let changing = false; + let closed = false; + + const hosts = createGatewayHosts({ + version: input.version, + log, + remembered: (serverId) => memory.alias(serverId), + machine: (serverId, alias) => memory.rename(serverId, alias), + ...(input.heartbeatMs !== undefined && { heartbeatMs: input.heartbeatMs }), + restarted: (alias) => input.restarted?.(alias), + changed() { + if (changing) return; + changing = true; + queueMicrotask(() => { + changing = false; + + if (closed) return; + const online = new Set(hosts.online()); + + for (const [abort, remote] of streaming) + if (!online.has(remote)) abort.abort(); + streams.changed(); + pushes.changed(); + }); + }, + }); + + const laptopIds = new Set(); + const lookups = new Map>(); + const streaming = new Map(); + + async function ownership(remote: GatewayRemote, reviewId: string) { + try { + const response = await send(remote, { + method: "GET", + path: `/reviews-api/${encodeURIComponent(reviewId)}/activity`, + headers: remoteHeaders(remote), + }); + + await response.body.dump(); + + return response.statusCode; + } catch (error) { + hosts.failed(remote, errorText(error)); + + return undefined; + } + } + + async function onLaptop(reviewId: string) { + if (laptopIds.has(reviewId)) return true; + + const laptop = await input.local( + new Request( + `http://gateway/reviews-api/${encodeURIComponent(reviewId)}/activity`, + ), + ); + + await laptop.body?.cancel(); + + if (laptop.ok) laptopIds.add(reviewId); + + return laptop.ok; + } + + const order = () => + hosts + .states() + .flatMap( + (state) => state.serverId ?? memory.serverIdOf(state.alias) ?? [], + ); + + async function lookup(reviewId: string): Promise { + const online = hosts.online(); + const found = Promise.withResolvers(); + let owner: GatewayRemote | undefined; + let left = online.length; + + if (!left) found.resolve(undefined); + + for (const remote of online) + void ownership(remote, reviewId).then((status) => { + left -= 1; + + if (status === 200 && remote.serverId !== undefined) { + if (owner) + log( + `Review ${reviewId} is on ${owner.alias} and ${remote.alias}; ${owner.alias} keeps it.`, + ); + else { + owner = remote; + memory.remember( + remote.serverId, + hosts.machineAlias(remote.serverId) ?? remote.alias, + reviewId, + ); + found.resolve({ remote }); + } + } + + if (!left) found.resolve(undefined); + }); + + return found.promise; + } + + function locate(reviewId: string): Located { + if (!UUID.test(reviewId) || laptopIds.has(reviewId)) return "laptop"; + const known = memory.owner(reviewId, order()); + + if (!known) return undefined; + const remote = hosts.serving(known.serverId); + + if (remote) return { remote }; + const down = hosts.unavailable(known.serverId, known.alias); + + return down && { down }; + } + + async function ownerOf(reviewId: string): Promise { + if (UUID.test(reviewId)) await onLaptop(reviewId); + const located = locate(reviewId); + + if (located === "laptop") return undefined; + + if (located && "remote" in located) { + const { remote } = located; + + if (remote.serverId !== undefined) + memory.remember( + remote.serverId, + hosts.machineAlias(remote.serverId) ?? remote.alias, + reviewId, + ); + + return located; + } + + if (located) return located; + let pending = lookups.get(reviewId); + + if (!pending) { + pending = lookup(reviewId).finally(() => lookups.delete(reviewId)); + lookups.set(reviewId, pending); + } + + return pending; + } + + function answer( + alias: string, + status: number, + body: { ok: boolean; error?: string } | ReviewRemoteNavigatorAnswer, + ) { + const response = serverJson(status, body); + response.headers.set(REVIEW_HOST_HEADER, alias); + + return response; + } + + function unavailable(down: ReviewGatewayHostState) { + return answer(down.alias, 503, { ok: false, error: downDetail(down) }); + } + + async function forward( + remote: GatewayRemote, + request: Request, + options: { reviewId: string; route?: RemoteRoute; body?: Buffer }, + ): Promise { + const url = new URL(request.url); + const { route } = options; + const routePath = url.pathname.split("/").slice(3).join("/"); + + // The laptop's token never leaves the laptop, in a header or the query. + if (url.searchParams.has("token")) url.searchParams.delete("token"); + + const headers: Record = remoteHeaders(remote); + + request.headers.forEach((value, key) => { + if (!DROPPED_REQUEST_HEADERS.has(key)) headers[key] = value; + }); + + const limit = route?.slow + ? (input.slowRouteMs ?? SLOW_ROUTE_TIMEOUT_MS) + : FIRST_BYTE_TIMEOUT_MS; + + const snapshotRequest = + route?.path === "" && url.searchParams.get("full") === "true"; + + const whole = snapshotRequest || route?.wholeBody === true; + const abort = new AbortController(); + + // SAFETY: Node's Request body is its own web stream; the DOM type only + // names the same object. + const stream = request.body as WebReadableStream | null; + let response: Dispatcher.ResponseData; + + try { + response = await send(remote, { + method: request.method, + path: `${url.pathname}${url.search}`, + headers, + body: options.body ?? (stream ? Readable.fromWeb(stream) : undefined), + signal: AbortSignal.any([request.signal, abort.signal]), + headersTimeout: limit, + ...(whole && { bodyTimeout: BODY_IDLE_MS }), + }); + } catch (error) { + const timedOut = errorCode(error) === "UND_ERR_HEADERS_TIMEOUT"; + + const reason = timedOut + ? `it did not answer within ${limit / 1_000} seconds` + : errorText(error); + + if (!route?.slow && !request.signal.aborted) hosts.failed(remote, reason); + + return answer(remote.alias, timedOut ? 504 : 502, { + ok: false, + error: `${remote.alias} did not answer: ${reason}.`, + }); + } + + const status = response.statusCode; + + // A review server never redirects; a remote must not steer the UI. + if (status >= 300 && status < 400 && status !== 304) { + await response.body.dump(); + log(`Refused ${remote.alias}'s redirect (${status}).`); + + return answer(remote.alias, 502, { + ok: false, + error: `${remote.alias} answered with a redirect, so the answer was refused.`, + }); + } + + if (status === 404) void confirmOwner(remote, options.reviewId); + + const out = new Headers(); + + for (const [key, value] of Object.entries(response.headers)) + if ( + value !== undefined && + !HOP_HEADERS.has(key) && + !DROPPED_RESPONSE_HEADERS.has(key) && + !key.startsWith("access-control-") + ) + out.set(key, Array.isArray(value) ? value.join(", ") : value); + + out.set(REVIEW_HOST_HEADER, remote.alias); + + const snapshot = snapshotRequest && status === 200; + + if (snapshot || route?.wholeBody) { + let body: Buffer; + + try { + body = await readBody( + response.body, + PATH_ROUTE_MAX_BYTES, + AbortSignal.timeout(BODY_MAX_MS), + ); + } catch (error) { + const cut = + errorCode(error) === "UND_ERR_BODY_TIMEOUT" + ? STALLED + : error instanceof DOMException && error.name === "TimeoutError" + ? TOO_LONG + : undefined; + + return answer(remote.alias, cut ? 504 : 502, { + ok: false, + error: `${remote.alias} did not answer: ${cut ?? errorText(error)}.`, + }); + } + + if (snapshot) { + const value = parseBody(body); + + if (isSnapshotOf(value, options.reviewId)) { + const stripped = withoutTutorial(value); + + return new Response( + stripped ? JSON.stringify(stripped) : new Uint8Array(body), + { status, headers: out }, + ); + } + + log( + `Refused ${remote.alias}'s answer for ${options.reviewId}: it carried another review.`, + ); + + return answer(remote.alias, 502, { + ok: false, + error: `${remote.alias} answered with another review, so the answer was refused.`, + }); + } + + if (route?.path === "navigator" && status === 200) { + const uris = navigatorUris(body, remote.serverId); + + if (uris) return answer(remote.alias, 200, uris); + log( + `Refused ${remote.alias}'s /navigator answer: a path or its server id is unusable.`, + ); + + return answer(remote.alias, 502, { + ok: false, + error: `${remote.alias} answered with an unusable path, so the answer was refused.`, + }); + } + + const field = pathField(body); + + if (field) { + log( + `Refused ${remote.alias}'s /${routePath} answer: it carried ${field}.`, + ); + + return answer(remote.alias, 502, { + ok: false, + error: `${remote.alias} answered with a path on that machine, so the answer was refused.`, + }); + } + + const unusable = + route?.path === "language-context" && status === 200 + ? unusableLanguageContext(body, remote.serverId) + : undefined; + + if (unusable) { + log(`Refused ${remote.alias}'s /${routePath} answer: ${unusable}.`); + + return answer(remote.alias, 502, { + ok: false, + error: `${remote.alias} answered with an unusable language context, so the answer was refused.`, + }); + } + + return new Response(new Uint8Array(body), { status, headers: out }); + } + + if (request.method === "HEAD" || status === 204 || status === 304) { + await response.body.dump(); + + return new Response(null, { status, headers: out }); + } + + streaming.set(abort, remote); + response.body.on("close", () => streaming.delete(abort)); + + // SAFETY: Node's Response takes its own web stream; the DOM type only + // names the same object. + const body = Readable.toWeb(response.body) as ReadableStream; + + return new Response(body, { status, headers: out }); + } + + async function confirmOwner(remote: GatewayRemote, reviewId: string) { + if ((await ownership(remote, reviewId)) === 404) memory.forget(reviewId); + } + + async function command(request: Request): Promise { + if (!request.body) return input.local(request); + let body: Buffer; + + try { + body = await readBoundedStream(request.body, DEFAULT_MAX_REQUEST_BYTES); + } catch (error) { + if (!(error instanceof StreamLimitError)) throw error; + + return serverJson(413, { + ok: false, + error: "Request body exceeds 1 MiB.", + }); + } + + let reviewId: string | undefined; + + try { + reviewId = commandTarget.safeParse(parseJsonText(body.toString())).data + ?.operation?.reviewId; + } catch {} + + const owner = reviewId === undefined ? undefined : await ownerOf(reviewId); + + if (!owner || reviewId === undefined) + return input.local( + new Request(request.url, { + method: request.method, + headers: request.headers, + body: new Uint8Array(body), + signal: request.signal, + }), + ); + + if ("down" in owner) return unavailable(owner.down); + + return forward(owner.remote, request, { reviewId, body }); + } + + async function review( + request: Request, + reviewId: string, + route?: RemoteRoute | "telemetry", + ): Promise { + if (!reviewId || LAPTOP_ROUTES.has(reviewId)) return input.local(request); + const owner = await ownerOf(reviewId); + + if (!owner) { + const response = await input.local(request); + + if (response.status === 404) laptopIds.delete(reviewId); + + return response; + } + + const alias = "remote" in owner ? owner.remote.alias : owner.down.alias; + + if (route === "telemetry") { + await request.body?.cancel(); + + return answer(alias, 200, { ok: true }); + } + + if (!route) + return answer(alias, 404, { + ok: false, + error: "This route is not available for a review on another machine.", + }); + + if ("down" in owner) return unavailable(owner.down); + + return forward(owner.remote, request, { reviewId, route }); + } + + const app = new Hono({ strict: false }).basePath("/reviews-api"); + + app.post("/commands", (context) => command(context.req.raw)); + app.get("/", (context) => streams.list(context.req.raw)); + app.get("/watch", (context) => streams.watch(context.req.raw)); + app.all("/:id/telemetry/*", (context) => + review(context.req.raw, context.req.param("id"), "telemetry"), + ); + + for (const route of REMOTE_ROUTES) + app.on(route.method, `/:id${route.path && `/${route.path}`}`, (context) => + review(context.req.raw, context.req.param("id") ?? "", route), + ); + + for (const path of ["/:id", "/:id/*"]) + app.all(path, (context) => + review(context.req.raw, context.req.param("id") ?? ""), + ); + + app.all("*", (context) => input.local(context.req.raw)); + + const streams = createGatewayStreams({ + hosts, + memory, + local: input.local, + locate, + lookup: ownerOf, + onLaptop: (reviewId) => laptopIds.add(reviewId), + log, + }); + + async function claim(remote: GatewayRemote, reviewId: string) { + if (await onLaptop(reviewId)) return `${reviewId} belongs to the laptop.`; + const known = memory.owner(reviewId, order()); + + if (known && known.serverId !== remote.serverId) + return `${reviewId} belongs to ${known.alias}.`; + + if (remote.serverId === undefined) + return `${remote.alias} has not reported its server id.`; + + memory.remember( + remote.serverId, + hosts.machineAlias(remote.serverId) ?? remote.alias, + reviewId, + ); + + return undefined; + } + + const pushes = createGatewayPushes({ + hosts, + relay: input.relay, + claim, + log, + }); + + const stopWatchingWindows = input.relay.onAttachedChange?.(() => { + if (!closed) pushes.changed(); + }); + + return { + fetch: (request: Request) => app.fetch(request), + setHosts: (list: ReviewGatewayHost[]) => hosts.set(list), + hosts: () => hosts.states(), + async close() { + closed = true; + stopWatchingWindows?.(); + streams.close(); + pushes.close(); + hosts.close(); + await memory.flush(); + }, + }; +} + +export type ReviewGateway = ReturnType; + +function unusableLanguageContext(body: Buffer, serverId: string | undefined) { + const context = remoteLanguageContext.safeParse( + parseJsonText(body.toString()), + ); + + if (!context.success) return "its remoteRootPath or serverId is malformed"; + + if (context.data.serverId !== serverId) return "it names another server"; + + return undefined; +} + +/** The remote navigator's host paths as `vscode-remote` URIs on its + * `whiteboard+` authority, as the fork's `reviewRemoteAuthority` + * names it. */ +function navigatorUris( + body: Buffer, + serverId: string | undefined, +): ReviewRemoteNavigatorAnswer | undefined { + const parsed = remoteNavigator.safeParse(parseBody(body)); + + if (!parsed.success || !serverId || !/^[0-9a-z-]+$/i.test(serverId)) + return undefined; + const { workspacePath, filePath, emptySide } = parsed.data; + const remoteAuthority = `whiteboard+${serverId.toLowerCase()}`; + + const uri = (host: string) => + `vscode-remote://${remoteAuthority}${uriPath(host)}`; + + return { + workspaceUri: uri(workspacePath), + ...(filePath !== undefined && { fileUri: uri(filePath) }), + remoteAuthority, + ...(emptySide && { emptySide }), + }; +} + +function uriPath(value: string) { + let out = ""; + + for (const char of value) + out += URI_PATH_KEPT.test(char) + ? char + : encodeURIComponent(char).replace( + /[!'()*]/g, + (kept) => `%${kept.charCodeAt(0).toString(16).toUpperCase()}`, + ); + + return out; +} + +function pathField(body: Buffer): string | undefined { + const value = parseBody(body); + + if (value === undefined) return "an unreadable body"; + const pending = [value]; + + for (let next = pending.pop(); next !== undefined; next = pending.pop()) { + if (Array.isArray(next)) pending.push(...next); + else if (isJsonObject(next)) + for (const [key, item] of Object.entries(next)) { + if (PATH_FIELDS.has(key)) return key; + pending.push(item); + } + } + + return undefined; +} + +function parseBody(body: Buffer): JsonValue | undefined { + try { + return parseJsonText(body.toString()); + } catch { + return undefined; + } +} diff --git a/packages/review/src/server/review-server-core.test.ts b/packages/review/src/server/review-server-core.test.ts index b020e9343..197d72550 100644 --- a/packages/review/src/server/review-server-core.test.ts +++ b/packages/review/src/server/review-server-core.test.ts @@ -26,6 +26,7 @@ beforeEach(async () => { root = await mkdtemp(path.join(tmpdir(), "review-server-core-")); vi.stubEnv("DEV_REVIEW_HOME", root); vi.stubEnv("DEV_FAST_REVIEW_TELEMETRY_DISABLED", "1"); + vi.stubEnv("SHELL", ""); }); afterEach(async () => { diff --git a/packages/review/src/server/review-server-core.ts b/packages/review/src/server/review-server-core.ts index 5b0c4cc45..75cde381e 100644 --- a/packages/review/src/server/review-server-core.ts +++ b/packages/review/src/server/review-server-core.ts @@ -4,7 +4,13 @@ import type { ReviewServerHealthWithToken, } from "@dev.fast/review-protocol"; import { traceMachineEnabled } from "@dev.fast/trace-core"; -import { detectAskAgents, launchAskAgent } from "@review/ask/agents.js"; +import { + type AskAgentLauncher, + askAgentTakesMcp, + detectAskAgents, + launchAskAgent, +} from "@review/ask/agents.js"; +import { askThreadLimits } from "@review/ask/thread.js"; import { AskThreads, type AskTools } from "@review/ask/threads.js"; import { readBuildCommit, @@ -94,6 +100,12 @@ export function createReviewServerApp(input: { return serverJson(accepted ? 200 : 404, { ok: accepted }); }); + answerErrorsAsJson(app); + + return app; +} + +export function answerErrorsAsJson(app: Hono) { app.notFound(() => serverJson(404, { ok: false, error: "Not found." })); app.onError((error) => { const serverError = error instanceof ReviewServerError ? error : undefined; @@ -107,8 +119,6 @@ export function createReviewServerApp(input: { : { ok: false, error: message }, ); }); - - return app; } export interface WhiteboardCoreInput { @@ -124,7 +134,15 @@ export interface WhiteboardCoreInput { scratchpad: () => boolean; status: () => JsonObject; hooks?: ReviewApiHooks; - ask?: { tools: AskTools }; + ask?: { + tools: AskTools; + /** Tests only: an in-process agent. */ + launch?: AskAgentLauncher; + /** Where agents are found, and the environment they run in. */ + env?: NodeJS.ProcessEnv; + /** Close a thread nobody has watched for this long. */ + idleCloseMs?: number; + }; } export function createWhiteboardCore(input: WhiteboardCoreInput) { @@ -139,8 +157,17 @@ export function createWhiteboardCore(input: WhiteboardCoreInput) { const callbacks = relayReviewCallbacks(input.relay, input.softwareMapEnabled); + const { ask } = input; + + const launch = ask?.launch ?? launchAskAgent; + const askThreads = - input.ask && new AskThreads(launchAskAgent, input.ask.tools); + ask && + new AskThreads( + (agent, cwd, options) => launch(agent, cwd, { ...options, env: ask.env }), + { takesMcp: (agent) => askAgentTakesMcp(agent, ask.env), ...ask.tools }, + { ...askThreadLimits, idleCloseMs: ask.idleCloseMs }, + ); const api = createReviewApi( store, @@ -152,7 +179,10 @@ export function createWhiteboardCore(input: WhiteboardCoreInput) { () => traceMachineEnabled(), input.status, input.hooks, - askThreads && { threads: askThreads, agents: () => detectAskAgents() }, + askThreads && { + threads: askThreads, + agents: () => detectAskAgents(ask?.env), + }, ); // A shared store mounts the publisher with the rest of sharing. diff --git a/packages/review/src/sharing/export.test.ts b/packages/review/src/sharing/export.test.ts index 9a4c3883d..996526607 100644 --- a/packages/review/src/sharing/export.test.ts +++ b/packages/review/src/sharing/export.test.ts @@ -642,7 +642,7 @@ it("lists and streams shared diff counts with the same mode and persistence as l JSON.parse(new TextDecoder().decode((await reader.read()).value)); try { - expect((await next())[0].value[0].diffStats).toBeNull(); + expect((await next()).reviews[0].diffStats).toBeNull(); expect((await app.request(`/${id}/progress?mode=textual`)).status).toBe( 200, ); @@ -652,7 +652,7 @@ it("lists and streams shared diff counts with the same mode and persistence as l const expected = local.store.list("textual")[0].diffStats; expect(expected).toMatchObject({ fileCount: 4 }); - expect((await next())[0].value[0].diffStats).toEqual(expected); + expect((await next()).reviews[0].diffStats).toEqual(expected); expect((await readCatalog("textual"))[0].diffStats).toEqual(expected); expect((await readCatalog("structural"))[0].diffStats).toBeNull(); diff --git a/packages/review/src/vsix.ts b/packages/review/src/vsix.ts new file mode 100644 index 000000000..46c2da01f --- /dev/null +++ b/packages/review/src/vsix.ts @@ -0,0 +1,122 @@ +// Plain Node and yauzl only: apps/review-desktop/scripts imports this file directly. +import { createWriteStream } from "node:fs"; +import { chmod, mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { Readable } from "node:stream"; +import { pipeline } from "node:stream/promises"; + +import yauzl from "yauzl"; +import { z } from "zod"; + +const manifestSchema = z.looseObject({ + publisher: z.string(), + name: z.string(), + engines: z.looseObject({ vscode: z.string().optional() }).optional(), +}); + +/** Unpacks a VSIX's `extension/` payload into `destination`. */ +export async function extractVsix( + vsix: string, + destination: string, + maxBytes = Number.POSITIVE_INFINITY, +) { + let unpacked = 0; + + const zip = await new Promise((resolve, reject) => + yauzl.open(vsix, { lazyEntries: true }, (error, opened) => + error ? reject(error) : resolve(opened), + ), + ); + + const openReadStream = (entry: yauzl.Entry) => + new Promise((resolve, reject) => + zip.openReadStream(entry, (error, stream) => + error ? reject(error) : resolve(stream), + ), + ); + + try { + await new Promise((resolve, reject) => { + zip.on("error", reject); + zip.on("end", () => resolve()); + zip.on("entry", (entry: yauzl.Entry) => { + void (async () => { + const name = entry.fileName.replaceAll("\\", "/"); + + if (name.endsWith("/") || !name.startsWith("extension/")) return; + + const relative = name.slice("extension/".length); + const parts = relative.split("/"); + const mode = (entry.externalFileAttributes >>> 16) & 0o177777; + + if ( + parts.some( + (part) => + part === ".." || + part === "." || + part === "" || + part.includes(":"), + ) || + path.posix.isAbsolute(relative) + ) + throw new Error(`VSIX contains an unsafe path: ${name}`); + + if ((mode & 0o170000) === 0o120000) + throw new Error(`VSIX contains a symlink: ${name}`); + + unpacked += entry.uncompressedSize; + + if (unpacked > maxBytes) + throw new Error(`VSIX unpacks to more than ${maxBytes} bytes`); + + const output = path.join(destination, ...parts); + await mkdir(path.dirname(output), { recursive: true }); + await pipeline( + await openReadStream(entry), + createWriteStream(output), + ); + + if (process.platform !== "win32" && mode & 0o111) + await chmod(output, mode & 0o777); + })().then(() => zip.readEntry(), reject); + }); + zip.readEntry(); + }); + } finally { + zip.close(); + } +} + +/** + * Drops the `scripts` and `dependencies` a payload ships prebundled, and + * `extensionPack` when Review does not ship the pack's members. + */ +export async function sanitizeVsixManifest( + directory: string, + extension: { id: string; stripExtensionPack: boolean }, +) { + const file = path.join(directory, "package.json"); + + const manifest = manifestSchema.parse( + JSON.parse(await readFile(file, "utf8")), + ); + + const declared = `${manifest.publisher}.${manifest.name}`; + + if (declared.toLowerCase() !== extension.id.toLowerCase()) + throw new Error(`${extension.id}: the VSIX declares ${declared}`); + + const { scripts, dependencies, devDependencies, extensionPack, ...kept } = + manifest; + + await writeFile( + file, + `${JSON.stringify( + extension.stripExtensionPack ? kept : { ...kept, extensionPack }, + undefined, + 2, + )}\n`, + ); + + return manifest.engines?.vscode; +} diff --git a/packages/trace-core/src/index.ts b/packages/trace-core/src/index.ts index 377da1eee..652c4d29f 100644 --- a/packages/trace-core/src/index.ts +++ b/packages/trace-core/src/index.ts @@ -196,6 +196,7 @@ export { } from "./trace-machine-setup"; export { + liveLockOwner, processIsAlive, processStartIdentity, withFileLock, diff --git a/packages/trace-core/src/with-file-lock.ts b/packages/trace-core/src/with-file-lock.ts index 7c010211a..e4789bb62 100644 --- a/packages/trace-core/src/with-file-lock.ts +++ b/packages/trace-core/src/with-file-lock.ts @@ -121,6 +121,16 @@ export async function withFileLock( } } +export async function liveLockOwner(lockPath: string) { + const owner = await readLockOwner(lockPath); + + return owner && + processIsAlive(owner.pid) && + !pidReused(owner.pid, owner.started) + ? owner.pid + : undefined; +} + export function processIsAlive(pid: number): boolean { try { process.kill(pid, 0); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a997ed7c8..5bd9c22b2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -170,6 +170,12 @@ importers: smol-toml: specifier: 1.8.0 version: 1.8.0 + undici: + specifier: 7.29.0 + version: 7.29.0 + yauzl: + specifier: 3.4.0 + version: 3.4.0 zod: specifier: 'catalog:' version: 4.6.5 @@ -198,6 +204,9 @@ importers: '@types/semver': specifier: 7.7.1 version: 7.7.1 + '@types/yauzl': + specifier: 3.4.0 + version: 3.4.0 micromark-extension-math: specifier: 3.1.0 version: 3.1.0 @@ -1435,6 +1444,9 @@ packages: '@types/unist@3.0.3': resolution: {integrity: sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==} + '@types/yauzl@3.4.0': + resolution: {integrity: sha512-NRPn5w6h8dhcnmx3YIRQcqMywY/+nND/uOkJessedcrowO3C0AssHp3tMJpxKAwOhFOo0OV1y9VtsC5hbKKBAw==} + '@typescript/typescript-aix-ppc64@7.0.2': resolution: {integrity: sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==} engines: {node: '>=16.20.0'} @@ -2613,6 +2625,9 @@ packages: pathe@2.0.3: resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} + pend@1.2.0: + resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==} + pi-acp@0.0.34: resolution: {integrity: sha512-MrCp37UbpUzkTkzTIMJQf1RYrT+kr4fgGDGFoLmBN26UupuCZp0XxVIY+7XYsMo3yDnzDeZUJ0jPyM3Si6Bkkg==} engines: {node: '>=20'} @@ -3192,6 +3207,10 @@ packages: yallist@3.1.1: resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==} + yauzl@3.4.0: + resolution: {integrity: sha512-jIH9yLR9wqr0wOS0TpBvo/g/2UgZH5qePVbjgRliiF0BYvOZyaBknKsF+x9Iht0O6sqgnB93rCICdOZFecJuDw==} + engines: {node: '>=12'} + yuku-ast@0.8.4: resolution: {integrity: sha512-s7EWfWIQkaGmsGnyr/BU0jli9YTN5TvrKIsSmALyRD9elumDQInuhv0BrVObENKVCxr9W3Ikmnx5u02KvfuUmw==} @@ -4028,6 +4047,10 @@ snapshots: '@types/unist@3.0.3': {} + '@types/yauzl@3.4.0': + dependencies: + '@types/node': 24.12.2 + '@typescript/typescript-aix-ppc64@7.0.2': optional: true @@ -5277,6 +5300,8 @@ snapshots: pathe@2.0.3: {} + pend@1.2.0: {} + pi-acp@0.0.34: dependencies: '@agentclientprotocol/sdk': 0.26.0(zod@3.25.76) @@ -5853,6 +5878,10 @@ snapshots: yallist@3.1.1: {} + yauzl@3.4.0: + dependencies: + pend: 1.2.0 + yuku-ast@0.8.4: dependencies: '@yuku-toolchain/types': 0.8.4 diff --git a/scripts/pack-review-cli.mjs b/scripts/pack-review-cli.mjs index 11d3d706e..87a2f6175 100644 --- a/scripts/pack-review-cli.mjs +++ b/scripts/pack-review-cli.mjs @@ -1,17 +1,31 @@ import { execFileSync } from "node:child_process"; -import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { + mkdir, + mkdtemp, + readFile, + rm, + stat, + writeFile, +} from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { stageReviewDocs } from "../apps/review-desktop/scripts/stage-review-runtime.mjs"; +import { stageVscodeServer } from "../apps/review-desktop/scripts/stage-vscode-server.mjs"; import { distTag } from "./review-cli-release.mjs"; -/** Pack from the workspace, then add the docs and version metadata shipped by Desktop. */ +export const MAX_TARBALL_BYTES = 60 * 1024 * 1024; + export async function packReviewCli( { version, commit }, outputDirectory, - { packageDirectory = "packages/review", stdio = "inherit" } = {}, + { + packageDirectory = "packages/review", + stdio = "inherit", + vscodeServer = {}, + maxBytes = MAX_TARBALL_BYTES, + } = {}, ) { distTag(version); @@ -46,6 +60,7 @@ export async function packReviewCli( ]); const staged = path.join(scratch, "package"); await stageReviewDocs(staged); + await stageVscodeServer(staged, { ...vscodeServer, commit }); execFileSync( "npm", @@ -53,7 +68,17 @@ export async function packReviewCli( { cwd: staged, stdio }, ); - return path.join(output, tarball); + const packed = path.join(output, tarball); + const { size } = await stat(packed); + + if (size > maxBytes) { + await rm(packed, { force: true }); + throw new Error( + `${tarball} is ${size} bytes, over the ${maxBytes}-byte limit`, + ); + } + + return packed; } finally { await writeFile(manifestPath, original); await rm(scratch, { recursive: true, force: true }); diff --git a/scripts/pack-review-cli.test.mjs b/scripts/pack-review-cli.test.mjs index 60757c1f8..20d32ffab 100644 --- a/scripts/pack-review-cli.test.mjs +++ b/scripts/pack-review-cli.test.mjs @@ -1,45 +1,137 @@ import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; -import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { createServer } from "node:http"; import os from "node:os"; import path from "node:path"; import { test } from "node:test"; +import { REMOTE_BUILTIN_EXTENSIONS } from "../apps/review-desktop/scripts/build-remote-runtime.mjs"; +import { remoteExtensionIds } from "../apps/review-desktop/scripts/curated-extensions.manifest.mjs"; import { packReviewCli } from "./pack-review-cli.mjs"; -test("packs the Desktop version under the whiteboard name with both bins", async (t) => { +const commit = execFileSync("git", ["rev-parse", "HEAD"], { + encoding: "utf8", +}).trim(); + +const version = "1.2.4-preview.20260901.42"; + +async function write(file, content, mode) { + await mkdir(path.dirname(file), { recursive: true }); + await writeFile(file, content, { mode }); +} + +async function fixture(t, { runtimeCommit = commit } = {}) { const root = await mkdtemp(path.join(os.tmpdir(), "whiteboard-pack-test-")); t.after(() => rm(root, { recursive: true, force: true })); - const { scripts, dependencies, devDependencies, ...manifest } = JSON.parse( + const { scripts, devDependencies, ...manifest } = JSON.parse( await readFile("packages/review/package.json", "utf8"), ); + const catalog = Object.fromEntries( + (await readFile("pnpm-workspace.yaml", "utf8")) + .match(/^catalog:\n((?: {2}.*\n)+)/m)[1] + .matchAll(/^ {2}"?([^":]+)"?: "?([^"\n]+)"?$/gm) + .map(([, name, range]) => [name, range]), + ); + + for (const [name, range] of Object.entries(manifest.dependencies)) + if (range === "catalog:") manifest.dependencies[name] = catalog[name]; + const packageDirectory = path.join(root, "package-source"); - await mkdir(path.join(packageDirectory, "dist"), { recursive: true }); - await writeFile( + await write( path.join(packageDirectory, "package.json"), `${JSON.stringify(manifest, null, 2)}\n`, ); - await writeFile(path.join(packageDirectory, "dist/cli.js"), ""); + await write(path.join(packageDirectory, "dist/cli.js"), ""); - const commit = execFileSync("git", ["rev-parse", "HEAD"], { - encoding: "utf8", - }).trim(); + const runtime = path.join(root, "runtime"); + await write( + path.join(runtime, "product.json"), + JSON.stringify({ commit: runtimeCommit }), + ); + await write(path.join(runtime, "out/server-main.js"), ""); + await write( + path.join(runtime, "extensions/node_modules/typescript/lib/tsserver.js"), + "", + ); + + for (const name of REMOTE_BUILTIN_EXTENSIONS) + await write(path.join(runtime, "extensions", name, "package.json"), "{}"); + + const tarballs = path.join(root, "tarballs"); + const downloads = []; + + const server = createServer((request, response) => + readFile(path.join(tarballs, path.basename(request.url))).then( + (data) => response.end(data), + () => response.writeHead(404).end(), + ), + ); - const version = "1.2.4-preview.20260901.42"; + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + t.after(() => new Promise((resolve) => server.close(resolve))); + + for (const [name, files] of [ + ["rg", { "bin/linux-x64/rg": "x64", "bin/linux-arm64/rg": "arm64" }], + ["watcher", { "package.json": "{}" }], + ]) { + for (const [file, content] of Object.entries(files)) + await write(path.join(tarballs, name, "package", file), content, 0o755); + + const tarball = path.join(tarballs, `${name}.tgz`); + execFileSync("tar", ["-czf", tarball, "-C", path.join(tarballs, name), "package"]); + downloads.push({ + url: `http://127.0.0.1:${server.address().port}/${name}.tgz`, + sha256: createHash("sha256") + .update(await readFile(tarball)) + .digest("hex"), + copy: + name === "rg" + ? { + "bin/linux-x64/rg": "out/vs/workbench/api/bin/linux-x64/rg", + "bin/linux-arm64/rg": "out/vs/workbench/api/bin/linux-arm64/rg", + } + : { ".": "node_modules/@parcel/watcher" }, + }); + } + + const cacheDir = path.join(root, "cache"); + await mkdir(cacheDir); + + for (const { url, sha256 } of downloads) + await writeFile( + path.join(cacheDir, `${sha256}-${path.basename(url)}`), + await readFile(path.join(tarballs, path.basename(url))), + ); + + return { + root, + packageDirectory, + downloads, + cacheDir, + vscodeServer: { runtime, cacheDir, downloads }, + }; +} + +test("packs the Desktop version under the whiteboard name with both bins and the VS Code server", async (t) => { + const { root, packageDirectory, vscodeServer } = await fixture(t); const output = path.join(root, "output"); const tarball = await packReviewCli({ version, commit }, output, { packageDirectory, stdio: "pipe", + vscodeServer, }); assert.equal(path.basename(tarball), `dev.fast-whiteboard-${version}.tgz`); execFileSync("tar", ["-xzf", tarball, "-C", output]); + const pkgRoot = path.join(output, "package"); const pkg = JSON.parse( - await readFile(path.join(output, "package/package.json"), "utf8"), + await readFile(path.join(pkgRoot, "package.json"), "utf8"), ); assert.equal(pkg.name, "@dev.fast/whiteboard"); @@ -49,4 +141,98 @@ test("packs the Desktop version under the whiteboard name with both bins", async review: "./dist/cli.js", whiteboard: "./dist/cli.js", }); + + const server = path.join(pkgRoot, "vscode-server"); + + for (const file of [ + "out/server-main.js", + "extensions/node_modules/typescript/lib/tsserver.js", + ...REMOTE_BUILTIN_EXTENSIONS.map((name) => `extensions/${name}/package.json`), + "node_modules/@parcel/watcher/package.json", + ]) + await stat(path.join(server, file)); + + for (const arch of ["x64", "arm64"]) { + const rg = await stat( + path.join(server, `out/vs/workbench/api/bin/linux-${arch}/rg`), + ); + + assert.ok(rg.mode & 0o111, `rg for ${arch} is executable`); + } + + const curated = JSON.parse( + await readFile(path.join(server, "curated.json"), "utf8"), + ); + + assert.deepEqual( + curated.extensions.map((extension) => extension.id), + remoteExtensionIds, + ); + + const go = curated.extensions.find((extension) => extension.id === "golang.go"); + assert.equal(go.tier, "optional"); + assert.equal(go.group, "go"); + + for (const extension of curated.extensions) + for (const target of ["linux-x64", "linux-arm64"]) { + const { url, sha256, size } = extension.targets[target]; + + assert.match(url, /^https:\/\/open-vsx\.org\//); + assert.match(sha256, /^[0-9a-f]{64}$/); + assert.ok(size > 0); + } +}); + +test("refuses a runtime built from another commit", async (t) => { + const { root, packageDirectory, vscodeServer } = await fixture(t, { + runtimeCommit: "0".repeat(40), + }); + + await assert.rejects( + packReviewCli({ version, commit }, path.join(root, "output"), { + packageDirectory, + stdio: "pipe", + vscodeServer, + }), + /the remote runtime was built from 0{40}/, + ); +}); + +test("deletes a download that fails its checksum and packs nothing", async (t) => { + const { root, packageDirectory, vscodeServer, cacheDir, downloads } = + await fixture(t); + + const [rg] = downloads; + rg.sha256 = "0".repeat(64); + const cachedFile = path.join(cacheDir, `${rg.sha256}-${path.basename(rg.url)}`); + + await assert.rejects( + packReviewCli({ version, commit }, path.join(root, "output"), { + packageDirectory, + stdio: "pipe", + vscodeServer, + }), + /Checksum mismatch/, + ); + await assert.rejects(stat(cachedFile)); + await assert.rejects(stat(path.join(root, "output", `dev.fast-whiteboard-${version}.tgz`))); +}); + +test("refuses a tarball over the size limit", async (t) => { + const { root, packageDirectory, vscodeServer } = await fixture(t); + + await assert.rejects( + packReviewCli({ version, commit }, path.join(root, "output"), { + packageDirectory, + stdio: "pipe", + vscodeServer, + maxBytes: 1024, + }), + /over the 1024-byte limit/, + ); + await assert.rejects( + stat( + path.join(root, "output", `dev.fast-whiteboard-${version}.tgz`), + ), + ); });