diff --git a/.github/workflows/fff-platform-validation.yml b/.github/workflows/fff-platform-validation.yml new file mode 100644 index 000000000000..92b245d62267 --- /dev/null +++ b/.github/workflows/fff-platform-validation.yml @@ -0,0 +1,214 @@ +name: FFF native platform validation + +on: + pull_request: + branches: [swxtch] + paths: + - ".github/workflows/fff-platform-validation.yml" + - ".github/actions/setup-bun/action.yml" + - "bun.lock" + - "flake.lock" + - "flake.nix" + - "nix/**" + - "package.json" + - "packages/core/src/filesystem/search.ts" + - "packages/core/src/flag/flag.ts" + - "packages/opencode/script/build.ts" + - "packages/opencode/test/fff-compiled.test.ts" + - "packages/opencode/test/fff-distribution.test.ts" + - "packages/script/src/fff-native.ts" + - "patches/@ff-labs%2Ffff-bun@0.9.4.patch" + +permissions: + contents: read + +concurrency: + group: fff-platform-validation-${{ github.event.pull_request.number }} + cancel-in-progress: true + +env: + TURBO_TELEMETRY_DISABLED: 1 + DO_NOT_TRACK: 1 + +jobs: + windows-fff: + name: Compiled FFF (${{ matrix.target }}) + strategy: + fail-fast: false + matrix: + include: + - target: windows-x64 + runner: windows-latest + - target: windows-arm64 + runner: windows-11-arm + runs-on: ${{ matrix.runner }} + env: + PR_HEAD: ${{ github.event.pull_request.head.sha }} + TARGET: ${{ matrix.target }} + steps: + - name: Check out pull request merge head + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Set up Node + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "24" + + - name: Resolve native Bun archive + id: bun + shell: bash + run: | + VERSION=$(node -p 'require("./package.json").packageManager.split("@")[1]') + case "$TARGET" in + windows-x64) ARCHIVE="bun-windows-x64-baseline.zip" ;; + windows-arm64) ARCHIVE="bun-windows-aarch64.zip" ;; + *) echo "Unsupported validation target: $TARGET" >&2; exit 1 ;; + esac + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "url=https://github.com/oven-sh/bun/releases/download/bun-v${VERSION}/${ARCHIVE}" >> "$GITHUB_OUTPUT" + + - name: Set up native Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: ${{ steps.bun.outputs.version }} + bun-download-url: ${{ steps.bun.outputs.url }} + + - name: Install dependencies + shell: bash + run: bun install --frozen-lockfile + + - name: Record runner and build identity + shell: bash + run: | + { + echo "## Windows compiled FFF validation" + echo "- PR head: $PR_HEAD" + echo "- Build head: $(git rev-parse HEAD)" + echo "- Target: $TARGET" + echo "- Runner: $RUNNER_NAME ($RUNNER_OS/$RUNNER_ARCH)" + echo "- Command: bun test --timeout 300000 test/fff-compiled.test.ts test/fff-distribution.test.ts" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Run compiled FFF distribution tests + working-directory: packages/opencode + shell: bash + run: bun test --timeout 300000 test/fff-compiled.test.ts test/fff-distribution.test.ts + + nix-node-modules: + name: Nix node_modules (${{ matrix.system }}) + strategy: + fail-fast: false + matrix: + include: + - system: x86_64-linux + runner: ubuntu-24.04 + - system: aarch64-linux + runner: ubuntu-24.04-arm + - system: x86_64-darwin + runner: macos-15-intel + - system: aarch64-darwin + runner: macos-15 + runs-on: ${{ matrix.runner }} + env: + PR_HEAD: ${{ github.event.pull_request.head.sha }} + SYSTEM: ${{ matrix.system }} + steps: + - name: Check out pull request merge head + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Initialize validation report + shell: bash + run: | + REPORT="$RUNNER_TEMP/nix-validation.md" + { + echo "## Nix validation" + echo "- PR head: $PR_HEAD" + echo "- Build head: $(git rev-parse HEAD)" + echo "- Nix system: $SYSTEM" + echo "- Runner: $RUNNER_NAME ($RUNNER_OS/$RUNNER_ARCH)" + } > "$REPORT" + echo "REPORT=$REPORT" >> "$GITHUB_ENV" + + - name: Set up Nix + uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34 + + - name: Compute native node_modules hash + shell: bash + run: | + set -euo pipefail + BUILD_LOG="$RUNNER_TEMP/nix-hash-build.log" + HASH_FILE="$RUNNER_TEMP/node-modules-hash.txt" + echo "- Hash command: nix build .#packages.$SYSTEM.node_modules_updater --no-link" >> "$REPORT" + nix build ".#packages.$SYSTEM.node_modules_updater" --no-link 2>&1 | tee "$BUILD_LOG" || true + HASH="$(python3 - "$BUILD_LOG" <<'PY' + import re + import sys + + with open(sys.argv[1], encoding="utf-8", errors="replace") as stream: + values = re.findall(r"got:\s*(sha256-[A-Za-z0-9+/=]+)", stream.read()) + print(values[-1] if values else "") + PY + )" + if [[ -z "$HASH" ]]; then + echo "- Computed hash: unavailable" >> "$REPORT" + echo "- Result: blocked; the Nix build did not report an output hash." >> "$REPORT" + exit 1 + fi + printf '%s\n' "$HASH" > "$HASH_FILE" + echo "- Computed hash: $HASH" >> "$REPORT" + EXPECTED="$(python3 - "$SYSTEM" <<'PY' + import json + import sys + + with open("nix/hashes.json", encoding="utf-8") as stream: + print(json.load(stream)["nodeModules"].get(sys.argv[1], "")) + PY + )" + echo "- Checked-in hash: ${EXPECTED:-missing}" >> "$REPORT" + if [[ "$HASH" != "$EXPECTED" ]]; then + echo "- Result: mismatch; update nix/hashes.json only from these native runner outputs, then rerun." >> "$REPORT" + exit 1 + fi + echo "- Result: hash matches the checked-in value." >> "$REPORT" + + - name: Build OpenCode with Nix + shell: bash + run: | + set -euo pipefail + BUILD_LOG="$RUNNER_TEMP/nix-opencode-build.log" + echo "- App build command: nix build .#packages.$SYSTEM.opencode --no-link --print-out-paths -L" >> "$REPORT" + nix build ".#packages.$SYSTEM.opencode" --no-link --print-out-paths -L 2>&1 | tee "$BUILD_LOG" + STORE_PATH="$(python3 - "$BUILD_LOG" <<'PY' + import re + import sys + + with open(sys.argv[1], encoding="utf-8", errors="replace") as stream: + paths = re.findall(r"/nix/store/[0-9a-z]{32}-[^\s]+", stream.read()) + print(paths[-1] if paths else "unresolved") + PY + )" + echo "- OpenCode output: $STORE_PATH" >> "$REPORT" + echo "- Result: Nix package build passed." >> "$REPORT" + + - name: Publish Nix validation report + if: always() + shell: bash + run: | + if [[ -f "$REPORT" ]]; then cat "$REPORT" >> "$GITHUB_STEP_SUMMARY"; fi + + - name: Retain Nix runner evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: nix-validation-${{ matrix.system }} + path: | + ${{ runner.temp }}/nix-validation.md + ${{ runner.temp }}/nix-hash-build.log + ${{ runner.temp }}/nix-opencode-build.log + ${{ runner.temp }}/node-modules-hash.txt + if-no-files-found: warn + retention-days: 7 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ce72641856c5..b77c4969b2be 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -172,6 +172,19 @@ jobs: exclude-azure-developer-cli-credential: true exclude-interactive-browser-credential: true + - name: Sign Windows CLI native libraries + shell: pwsh + run: | + $files = @() + $files += Get-ChildItem "${{ github.workspace }}\packages\opencode\dist\opencode-windows-arm64\bin\fff_c-*.dll" | Select-Object -ExpandProperty FullName + $files += Get-ChildItem "${{ github.workspace }}\packages\opencode\dist\opencode-windows-x64\bin\fff_c-*.dll" | Select-Object -ExpandProperty FullName + $files += Get-ChildItem "${{ github.workspace }}\packages\opencode\dist\opencode-windows-x64-baseline\bin\fff_c-*.dll" | Select-Object -ExpandProperty FullName + if ($files.Count -ne 3) { + throw "Expected one FFF native library for each Windows target, found $($files.Count)" + } + $signPaths = @($files) + & "${{ github.workspace }}\script\sign-windows.ps1" @signPaths + - name: Verify Windows CLI signatures shell: pwsh run: | @@ -180,6 +193,14 @@ jobs: "${{ github.workspace }}\packages\opencode\dist\opencode-windows-x64\bin\opencode.exe", "${{ github.workspace }}\packages\opencode\dist\opencode-windows-x64-baseline\bin\opencode.exe" ) + $native = @() + $native += Get-ChildItem "${{ github.workspace }}\packages\opencode\dist\opencode-windows-arm64\bin\fff_c-*.dll" | Select-Object -ExpandProperty FullName + $native += Get-ChildItem "${{ github.workspace }}\packages\opencode\dist\opencode-windows-x64\bin\fff_c-*.dll" | Select-Object -ExpandProperty FullName + $native += Get-ChildItem "${{ github.workspace }}\packages\opencode\dist\opencode-windows-x64-baseline\bin\fff_c-*.dll" | Select-Object -ExpandProperty FullName + if ($native.Count -ne 3) { + throw "Expected one FFF native library for each Windows target, found $($native.Count)" + } + $files += $native foreach ($file in $files) { $sig = Get-AuthenticodeSignature $file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9caa6c97eb86..9337f4ca8bf3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,10 +20,7 @@ permissions: env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true - # Turborepo telemetry is on by default and POSTs anonymous usage data to - # Vercel. This fork runs only these two workflows, so opting out here is the - # whole surface. Raised by sol during the #22 review as the one outward-facing - # action still reachable from this fork. + # Turborepo telemetry is enabled by default and sends anonymous usage data to Vercel, so this workflow disables telemetry and tracking (review #22). TURBO_TELEMETRY_DISABLED: 1 DO_NOT_TRACK: 1 diff --git a/CHANGESET.md b/CHANGESET.md index 069e9442b6b5..716d9f591645 100644 --- a/CHANGESET.md +++ b/CHANGESET.md @@ -38,6 +38,7 @@ Newest first. Each entry names the upstream range brought in. ### Fork maintenance and CI +- **#150** The repository-read-only `fff-platform-validation.yml` builds and runs compiled FFF distributions on native Windows x64 and ARM64 runners, computes and checks Nix node_modules hashes on native Linux and macOS runners, retains runner evidence, and builds OpenCode with Nix after each hash matches. _Fork-only._ - **#98** `tests/run.sh` runs the unit suite from the repository root as CI's "Run unit tests" step does (`GITHUB_ACTIONS=false bun turbo test`), returning its exit status, and fails when Bun or the installed workspace is missing. The root `npm test` still refuses. _Fork-only._ @@ -139,6 +140,17 @@ Newest first. Each entry names the upstream range brought in. _Partly upstream:_ upstream #50439 fixed the cycle and its version was taken in the 2026-09-22 sync; the layer-node guard is fork-only. +### Native FFF distribution + +- **#150** Use the root `packageManager` Bun pin for Nix-built OpenCode and desktop binaries (`package.json`, + `nix/bun.nix`, and `flake.nix`). _Fork-only (swxtchio/swx-opencode#97)._ +- **#150** Patch `@ff-labs/fff-bun` (`patches/@ff-labs%2Ffff-bun@0.9.4.patch`) so compiled builds load the + content-addressed, target-specific FFF sidecar beside the executable; carry the sidecar through build, install, + postinstall, local-release, Docker, and generated release-package paths. _Fork-only (swxtchio/swx-opencode#97)._ +- **#150** Enable native FFF by default in packaged Windows builds (`packages/core/src/flag/flag.ts`), preserve + `OPENCODE_DISABLE_FFF=1` as the ripgrep fallback, and report the expected sidecar path when loading is unavailable + (`packages/core/src/filesystem/search.ts`). _Fork-only (swxtchio/swx-opencode#97)._ + ### Features - **#73** V1 server prompt queue core (swxtchio/swx-opencode#68): every V1 prompt except `noReply` is admitted to the durable `session_prompt_queue` table before it becomes a user message. `prompt`, `prompt_async` and `command` take `delivery: "steer" | "queue"`; the default steer reaches the next step as before, while `queue` waits until the run would go idle and then runs as its own turn, one per boundary in admission order. Steers wait behind a pending compaction; an abort or error parks pending prompts until the next admission or wake; `/session/:id/queue` lists, withdraws (404 `QueueItemNotPending` once delivered), restores and re-delivers them, with `session.queue.updated` carrying the full list. A sync prompt answers with the final reply of the turn its prompt joined, recorded for it where the loop ends that turn: steers share the reply of the turn they steered, and a queued prompt gets its own turn's; an editor's withdraw that wins answers it with 409 `PromptWithdrawn`. Deliberately, a compaction whose summary turn stopped on an abort or error is not retried, where upstream V1 retried it on the next prompt with that prompt as its parent (summarising it instead of answering it); if the context is still too large, the next overflow check starts a new compaction. Also closes the V1 lost wakeup where a prompt admitted as a run finished joined it and was never answered. Known ordering-only residuals: a `noReply` prompt and the synthetic message a subtask writes to have its output summarised are stored directly rather than through the queue, so they can land out of admission order relative to pending items. _Fork-only; the lost-wakeup fix is upstreamable._ diff --git a/bun.lock b/bun.lock index c4a7760363ce..992a80eb016c 100644 --- a/bun.lock +++ b/bun.lock @@ -1063,6 +1063,7 @@ "@ai-sdk/xai@3.0.102": "patches/@ai-sdk%2Fxai@3.0.102.patch", "@modelcontextprotocol/sdk@1.29.0": "patches/@modelcontextprotocol%2Fsdk@1.29.0.patch", "gcp-metadata@8.1.2": "patches/gcp-metadata@8.1.2.patch", + "@ff-labs/fff-bun@0.9.4": "patches/@ff-labs%2Ffff-bun@0.9.4.patch", "@ai-sdk/openai@3.0.88": "patches/@ai-sdk%2Fopenai@3.0.88.patch", "@standard-community/standard-openapi@0.2.9": "patches/@standard-community%2Fstandard-openapi@0.2.9.patch", "effect@4.0.0-beta.83": "patches/effect@4.0.0-beta.83.patch", diff --git a/flake.nix b/flake.nix index 8737c3b542f7..5d2fc526d705 100644 --- a/flake.nix +++ b/flake.nix @@ -14,54 +14,63 @@ "aarch64-darwin" "x86_64-darwin" ]; - forEachSystem = f: nixpkgs.lib.genAttrs systems (system: f nixpkgs.legacyPackages.${system}); + forEachSystem = f: + nixpkgs.lib.genAttrs systems ( + system: + let + pkgs = nixpkgs.legacyPackages.${system}; + bun = pkgs.callPackage ./nix/bun.nix { }; + in + f pkgs bun + ); rev = self.shortRev or self.dirtyShortRev or "dirty"; in { - devShells = forEachSystem (pkgs: { + devShells = forEachSystem (pkgs: bun: { default = pkgs.mkShell { - packages = with pkgs; [ - bun - nodejs_20 - pkg-config - openssl - git - ]; + packages = [ bun pkgs.nodejs_20 pkgs.pkg-config pkgs.openssl pkgs.git ]; }; }); overlays = { default = - final: _prev: + final: prev: let + bun = final.callPackage ./nix/bun.nix { bun = prev.bun; }; node_modules = final.callPackage ./nix/node_modules.nix { + # This fixed-output derivation only materializes dependencies; binary builds use the fixed Bun. + bun = prev.bun; inherit rev; }; - in - rec { opencode = final.callPackage ./nix/opencode.nix { - inherit node_modules; + inherit bun node_modules; }; + in + { + inherit bun opencode; opencode-desktop = final.callPackage ./nix/desktop.nix { - inherit opencode; + inherit bun opencode; }; }; }; packages = forEachSystem ( - pkgs: + pkgs: bun: let node_modules = pkgs.callPackage ./nix/node_modules.nix { + # This fixed-output derivation only materializes dependencies; binary builds use the fixed Bun. + bun = pkgs.bun; inherit rev; }; - in - rec { - default = opencode; opencode = pkgs.callPackage ./nix/opencode.nix { - inherit node_modules; + inherit bun node_modules; }; + in + { + default = opencode; + inherit opencode; opencode-desktop = pkgs.callPackage ./nix/desktop.nix { - inherit opencode; + inherit bun opencode; }; # Updater derivation with fakeHash - build fails and reveals correct hash node_modules_updater = node_modules.override { diff --git a/install b/install index b0716d532082..1bed70f449f7 100755 --- a/install +++ b/install @@ -340,13 +340,43 @@ download_and_install() { unzip -q "$tmp_dir/$filename" -d "$tmp_dir" fi + native_library=$(find "$tmp_dir" -maxdepth 1 -type f -name "$(native_library_pattern "$os")" -print -quit) + if [ -n "$native_library" ]; then + native_target="$INSTALL_DIR/$(basename "$native_library")" + if [ ! -f "$native_target" ] || ! cmp -s "$native_library" "$native_target"; then + mv "$native_library" "$native_target" + fi + fi mv "$tmp_dir/opencode" "$INSTALL_DIR" chmod 755 "${INSTALL_DIR}/opencode" rm -rf "$tmp_dir" } +native_library_pattern() { + case "$1" in + darwin) printf 'libfff_c-*.dylib' ;; + linux) printf 'libfff_c-*.so' ;; + windows) printf 'fff_c-*.dll' ;; + *) return 1 ;; + esac +} + install_from_binary() { print_message info "\n${MUTED}Installing ${NC}opencode ${MUTED}from: ${NC}$binary_path" + raw_os=$(uname -s) + local_os=$(echo "$raw_os" | tr '[:upper:]' '[:lower:]') + case "$raw_os" in + Darwin*) local_os="darwin" ;; + Linux*) local_os="linux" ;; + MINGW*|MSYS*|CYGWIN*) local_os="windows" ;; + esac + native_library=$(find "$(dirname "$binary_path")" -maxdepth 1 -type f -name "$(native_library_pattern "$local_os")" -print -quit) + if [ -n "$native_library" ]; then + native_target="$INSTALL_DIR/$(basename "$native_library")" + if [ ! -f "$native_target" ] || ! cmp -s "$native_library" "$native_target"; then + cp "$native_library" "$native_target" + fi + fi cp "$binary_path" "${INSTALL_DIR}/opencode" chmod 755 "${INSTALL_DIR}/opencode" } diff --git a/local-release.sh b/local-release.sh index 74d29d9b0256..8cb756c29be8 100755 --- a/local-release.sh +++ b/local-release.sh @@ -25,7 +25,8 @@ if [[ "$platform" != linux-* && "$platform" != darwin-* ]]; then printf 'Local release installs currently support Linux and macOS only.\n' >&2 exit 1 fi -built="$root/packages/opencode/dist/opencode-$platform/bin/opencode" +build_dir="$root/packages/opencode/dist/opencode-$platform/bin" +built="$build_dir/opencode" target="${OPENCODE_LOCAL_BIN:-$HOME/.local/opencode-swxtch/bin/opencode}" database="${OPENCODE_DB_PATH:-$HOME/.local/share/opencode/opencode.db}" if [[ "$target" != /* || "$database" != /* ]]; then @@ -52,6 +53,11 @@ fi printf 'Building %s with Bun %s\n' "$release" "$bun_version" npx --yes "bun@$bun_version" install --frozen-lockfile OPENCODE_VERSION="$release" npx --yes "bun@$bun_version" run packages/opencode/script/build.ts --single --skip-install +native_library=$(find "$build_dir" -maxdepth 1 -type f \( -name 'libfff_c-*.so' -o -name 'libfff_c-*.dylib' \) -print -quit) +if [[ -z "$native_library" ]]; then + printf 'Built FFF native library was not found beside %s.\n' "$built" >&2 + exit 1 +fi if [[ "$("$built" --version)" != "$release" || "$("$built" db path)" != "$database" ]]; then printf 'Built binary version or database path did not match; install aborted.\n' >&2 @@ -68,7 +74,13 @@ if [[ -f "$target" ]]; then fi temporary="$target.new.$$" -trap 'rm -f "$temporary"' EXIT +native_library_target="$(dirname "$target")/$(basename "$native_library")" +temporary_library="$native_library_target.new.$$" +trap 'rm -f "$temporary" "$temporary_library"' EXIT +if [[ ! -f "$native_library_target" ]] || ! cmp -s "$native_library" "$native_library_target"; then + install -m 644 "$native_library" "$temporary_library" + mv -f "$temporary_library" "$native_library_target" +fi install -m 755 "$built" "$temporary" mv -f "$temporary" "$target" diff --git a/nix/bun.nix b/nix/bun.nix new file mode 100644 index 000000000000..37c17fd917c0 --- /dev/null +++ b/nix/bun.nix @@ -0,0 +1,33 @@ +{ + bun, + fetchurl, + lib, +}: + +let + packageManager = (builtins.fromJSON (builtins.readFile ../package.json)).packageManager; + version = lib.removePrefix "bun@" packageManager; +in +bun.overrideAttrs (old: { + inherit version; + passthru = old.passthru // { + sources = { + "aarch64-darwin" = fetchurl { + url = "https://github.com/oven-sh/bun/releases/download/bun-v${version}/bun-darwin-aarch64.zip"; + hash = "sha256-kJh6OhbX21VtiGrD1VHnttPt8KHPQ6yu1iLoZ2vh0S8="; + }; + "aarch64-linux" = fetchurl { + url = "https://github.com/oven-sh/bun/releases/download/bun-v${version}/bun-linux-aarch64.zip"; + hash = "sha256-VDKLvC2cjgyfiSxUTWbFeoO4QTnjSQnl7oF1jxrI/ac="; + }; + "x86_64-darwin" = fetchurl { + url = "https://github.com/oven-sh/bun/releases/download/bun-v${version}/bun-darwin-x64-baseline.zip"; + hash = "sha256-utW71s8U0JgNEV9ZVMn/kE32GdXplNLaH/zNPzFjALA="; + }; + "x86_64-linux" = fetchurl { + url = "https://github.com/oven-sh/bun/releases/download/bun-v${version}/bun-linux-x64-baseline.zip"; + hash = "sha256-xngEDxT+BEDrg503y9DOTAUaMtpygGrJfeamqra/co8="; + }; + }; + }; +}) diff --git a/nix/desktop.nix b/nix/desktop.nix index 2df62f7a1c76..f2f0a43cc7e3 100644 --- a/nix/desktop.nix +++ b/nix/desktop.nix @@ -57,23 +57,14 @@ stdenv.mkDerivation (finalAttrs: { ELECTRON_SKIP_BINARY_DOWNLOAD = "1"; }; - postPatch = - # NOTE: Relax Bun version check to be a warning instead of an error - '' - substituteInPlace packages/script/src/index.ts \ - --replace-fail 'throw new Error(`This script requires bun@''${expectedBunVersionRange}' \ - 'console.warn(`Warning: This script requires bun@''${expectedBunVersionRange}' - '' - # https://github.com/electron/electron/issues/31121 - # mac builds use a .app bundle which doesnt have this issue - + lib.optionalString stdenv.isLinux '' - BASE_PATH=packages/desktop - FILES=(src/main/windows.ts) - for file in "''${FILES[@]}"; do - substituteInPlace $BASE_PATH/$file \ - --replace-fail "process.resourcesPath" "'$out/opt/opencode-desktop/resources'" - done - ''; + postPatch = lib.optionalString stdenv.isLinux '' + BASE_PATH=packages/desktop + FILES=(src/main/windows.ts) + for file in "''${FILES[@]}"; do + substituteInPlace $BASE_PATH/$file \ + --replace-fail "process.resourcesPath" "'$out/opt/opencode-desktop/resources'" + done + ''; preBuild = '' cp -r "${electron.dist}" $HOME/.electron-dist diff --git a/nix/hashes.json b/nix/hashes.json index 3672243020b4..7cb0d97ad97b 100644 --- a/nix/hashes.json +++ b/nix/hashes.json @@ -1,8 +1,8 @@ { "nodeModules": { - "x86_64-linux": "sha256-RIzpvDBR6GOfU2vjrtgP3LJ/j+gLxkMnJ9VE5Y65Pfo=", - "aarch64-linux": "sha256-seE7v8AqxWCiyxNARMiLJNB1MZv15OwEf6r21k68xqg=", - "aarch64-darwin": "sha256-rbyK+3qz9bGOxg3HlGW6yh3qeazCaxFcuPfdGbXHsIY=", - "x86_64-darwin": "sha256-chSE5WtSQ+qdbj5vTvGmr4rn9gk8eVp3pVeow4gVqYI=" + "x86_64-linux": "sha256-oc1dDVjuqEDE839TRFDUFK11I2Udu1WWgUnB4gLECcM=", + "aarch64-linux": "sha256-+DpJxpVrlNd/vL2TEEUY5/yz+rzN5wfb/zcV5u9sWQc=", + "aarch64-darwin": "sha256-vctJ12gdmM4OhVZ1AS+f3q4WiVWv2RWeraeoODfAy68=", + "x86_64-darwin": "sha256-LCAiY/MP8NW520yNjmQX+dqUUdMD7Vc/bMmHwyrb+sM=" } } diff --git a/nix/opencode.nix b/nix/opencode.nix index a22f7d3d2473..c94935a34b0c 100644 --- a/nix/opencode.nix +++ b/nix/opencode.nix @@ -27,13 +27,6 @@ stdenvNoCC.mkDerivation (finalAttrs: { writableTmpDirAsHomeHook ]; - postPatch = '' - # NOTE: Relax Bun version check to be a warning instead of an error - substituteInPlace packages/script/src/index.ts \ - --replace-fail 'throw new Error(`This script requires bun@''${expectedBunVersionRange}' \ - 'console.warn(`Warning: This script requires bun@''${expectedBunVersionRange}' - ''; - configurePhase = '' runHook preConfigure @@ -63,6 +56,7 @@ stdenvNoCC.mkDerivation (finalAttrs: { runHook preInstall install -Dm755 dist/opencode-*/bin/opencode $out/bin/opencode + install -Dm644 dist/opencode-*/bin/libfff_c-* -t $out/bin install -Dm644 schema.json $out/share/opencode/schema.json wrapProgram $out/bin/opencode \ diff --git a/package.json b/package.json index 95f950cf2e19..8bed53fee178 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "AI-powered development tool", "private": true, "type": "module", - "packageManager": "bun@1.3.14", + "packageManager": "bun@1.4.2", "scripts": { "dev": "bun run --cwd packages/opencode src/index.ts", "dev:desktop": "bun --cwd packages/desktop dev", @@ -166,6 +166,7 @@ "@ai-sdk/groq@3.0.31": "patches/@ai-sdk%2Fgroq@3.0.31.patch", "@ai-sdk/anthropic@3.0.111": "patches/@ai-sdk%2Fanthropic@3.0.111.patch", "@ai-sdk/amazon-bedrock@4.0.166": "patches/@ai-sdk%2Famazon-bedrock@4.0.166.patch", - "@ai-sdk/openai@3.0.88": "patches/@ai-sdk%2Fopenai@3.0.88.patch" + "@ai-sdk/openai@3.0.88": "patches/@ai-sdk%2Fopenai@3.0.88.patch", + "@ff-labs/fff-bun@0.9.4": "patches/@ff-labs%2Ffff-bun@0.9.4.patch" } } diff --git a/packages/core/src/filesystem/search.ts b/packages/core/src/filesystem/search.ts index cdba2be7ec13..766a7a8607db 100644 --- a/packages/core/src/filesystem/search.ts +++ b/packages/core/src/filesystem/search.ts @@ -13,6 +13,13 @@ import { Ripgrep } from "../ripgrep" import { RelativePath } from "../schema" import { Flag } from "../flag/flag" +declare const FFF_BUN_EXTERNAL_LIBRARY: string + +export function packagedFffUnavailableMessage(filename: string) { + const expectedPath = path.join(path.dirname(process.execPath), filename) + return `The packaged FFF native library is unavailable at ${expectedPath}. Set OPENCODE_DISABLE_FFF=1 to use ripgrep instead.` +} + export interface Interface { readonly find: (input: FileSystem.FindInput) => Effect.Effect readonly glob: (input: FileSystem.GlobInput) => Effect.Effect @@ -233,7 +240,16 @@ export const fffLayer = Layer.effect( }), ) -const layer = Layer.unwrap(Effect.sync(() => (Flag.OPENCODE_DISABLE_FFF || !Fff.available() ? ripgrepLayer : fffLayer))) +const layer = Layer.unwrap( + Effect.sync(() => { + if (Flag.OPENCODE_DISABLE_FFF) return ripgrepLayer + if (Fff.available()) return fffLayer + if (typeof FFF_BUN_EXTERNAL_LIBRARY === "string") { + throw new Error(packagedFffUnavailableMessage(FFF_BUN_EXTERNAL_LIBRARY)) + } + return ripgrepLayer + }), +) export const locationLayer = layer diff --git a/packages/core/src/flag/flag.ts b/packages/core/src/flag/flag.ts index d8630508fe94..00b10b682258 100644 --- a/packages/core/src/flag/flag.ts +++ b/packages/core/src/flag/flag.ts @@ -1,10 +1,20 @@ import { Config } from "effect" +declare const FFF_BUN_EXTERNAL_LIBRARY: string + export function truthy(key: string) { const value = process.env[key]?.toLowerCase() return value === "true" || value === "1" } +export function isFffDisabled(configured: string | undefined, platform: string, packagedLibrary: boolean) { + if (configured !== undefined) { + const value = configured.toLowerCase() + return value === "true" || value === "1" + } + return platform === "win32" && !packagedLibrary +} + const copy = process.env["OPENCODE_EXPERIMENTAL_DISABLE_COPY_ON_SELECT"] const fff = process.env["OPENCODE_DISABLE_FFF"] @@ -31,7 +41,11 @@ export const Flag = { OPENCODE_FAKE_VCS: process.env["OPENCODE_FAKE_VCS"], OPENCODE_SERVER_PASSWORD: process.env["OPENCODE_SERVER_PASSWORD"], OPENCODE_SERVER_USERNAME: process.env["OPENCODE_SERVER_USERNAME"], - OPENCODE_DISABLE_FFF: fff === undefined ? process.platform === "win32" : truthy("OPENCODE_DISABLE_FFF"), + OPENCODE_DISABLE_FFF: isFffDisabled( + fff, + process.platform, + typeof FFF_BUN_EXTERNAL_LIBRARY === "string", + ), // Experimental OPENCODE_EXPERIMENTAL_FILEWATCHER: Config.boolean("OPENCODE_EXPERIMENTAL_FILEWATCHER").pipe( diff --git a/packages/core/test/filesystem/search-fff.test.ts b/packages/core/test/filesystem/search-fff.test.ts new file mode 100644 index 000000000000..8a09391589fc --- /dev/null +++ b/packages/core/test/filesystem/search-fff.test.ts @@ -0,0 +1,66 @@ +import { describe, expect } from "bun:test" +import { Effect, Layer } from "effect" +import fs from "node:fs/promises" +import path from "node:path" +import { FileSystemSearch } from "@opencode-ai/core/filesystem/search" +import { Location } from "@opencode-ai/core/location" +import { AbsolutePath, RelativePath } from "@opencode-ai/core/schema" +import { location } from "../fixture/location" +import { tmpdir } from "../fixture/tmpdir" +import { testEffect } from "../lib/effect" + +const it = testEffect(Layer.empty) +const withTmp = (f: (directory: string) => Effect.Effect) => + Effect.acquireRelease( + Effect.promise(() => tmpdir()), + (tmp) => Effect.promise(() => tmp[Symbol.asyncDispose]()), + ).pipe(Effect.flatMap((tmp) => f(tmp.path))) + +describe("FileSystemSearch.fffLayer", () => { + it.live( + "supports native file, directory, glob, and content search", + () => + withTmp((directory) => + Effect.gen(function* () { + yield* Effect.promise(() => fs.mkdir(path.join(directory, "src", "nested"), { recursive: true })) + yield* Effect.promise(() => + fs.writeFile(path.join(directory, "src", "nested", "needle-target.ts"), "picker-content\n"), + ) + + const nativeLayer = Layer.provide( + FileSystemSearch.fffLayer, + Layer.succeed(Location.Service, Location.Service.of(location({ directory: AbsolutePath.make(directory) }))), + ) + + yield* Effect.gen(function* () { + const search = yield* FileSystemSearch.Service + const files = yield* waitForFile(search) + expect(files.map((item) => item.path)).toContain(RelativePath.make("src/nested/needle-target.ts")) + + const directories = yield* search.find({ type: "directory", query: "nested", limit: 10 }) + expect(directories.some((item) => item.path.includes("nested"))).toBe(true) + + const glob = yield* search.glob({ pattern: "src/nested/*.ts", limit: 10 }) + expect(glob.map((item) => item.path)).toContain(RelativePath.make("src/nested/needle-target.ts")) + + const matches = yield* search.grep({ pattern: "picker-content", limit: 10 }) + expect(matches.some((match) => match.text.includes("picker-content"))).toBe(true) + }).pipe(Effect.provide(nativeLayer)) + }), + ), + 15_000, + ) +}) + +function waitForFile( + search: FileSystemSearch.Interface, + started = Date.now(), +): ReturnType { + return Effect.gen(function* () { + const files = yield* search.find({ type: "file", query: "needle-target", limit: 10 }) + if (files.some((item) => item.path === RelativePath.make("src/nested/needle-target.ts"))) return files + if (Date.now() - started > 10_000) throw new Error("FFF did not index the fixture file") + yield* Effect.sleep("2 seconds") + return yield* waitForFile(search, started) + }) +} diff --git a/packages/core/test/flag/flag.test.ts b/packages/core/test/flag/flag.test.ts new file mode 100644 index 000000000000..38fb67ff5194 --- /dev/null +++ b/packages/core/test/flag/flag.test.ts @@ -0,0 +1,11 @@ +import { describe, expect, test } from "bun:test" +import { isFffDisabled } from "@opencode-ai/core/flag/flag" + +describe("FFF platform default", () => { + test("keeps native FFF enabled in compiled Windows distributions", () => { + expect(isFffDisabled(undefined, "win32", true)).toBe(false) + expect(isFffDisabled(undefined, "win32", false)).toBe(true) + expect(isFffDisabled(undefined, "linux", true)).toBe(false) + expect(isFffDisabled("true", "win32", true)).toBe(true) + }) +}) diff --git a/packages/opencode/Dockerfile b/packages/opencode/Dockerfile index f92b48a6d1b5..65a8999d421d 100644 --- a/packages/opencode/Dockerfile +++ b/packages/opencode/Dockerfile @@ -8,9 +8,11 @@ RUN apk add libgcc libstdc++ ripgrep FROM base AS build-amd64 COPY dist/opencode-linux-x64-baseline-musl/bin/opencode /usr/local/bin/opencode +COPY dist/opencode-linux-x64-baseline-musl/bin/libfff_c-*.so /usr/local/bin/ FROM base AS build-arm64 COPY dist/opencode-linux-arm64-musl/bin/opencode /usr/local/bin/opencode +COPY dist/opencode-linux-arm64-musl/bin/libfff_c-*.so /usr/local/bin/ ARG TARGETARCH FROM build-${TARGETARCH} diff --git a/packages/opencode/script/build.ts b/packages/opencode/script/build.ts index e02958b14d6c..4bb1ae3533e1 100755 --- a/packages/opencode/script/build.ts +++ b/packages/opencode/script/build.ts @@ -1,15 +1,20 @@ #!/usr/bin/env bun import { $ } from "bun" +import fs from "node:fs/promises" +import { createRequire } from "node:module" import path from "path" import { fileURLToPath } from "url" import { createSolidTransformPlugin } from "@opentui/solid/bun-plugin" +import { fffBuildDefines, fffLibraryForTarget } from "@opencode-ai/script/fff-native" const __filename = fileURLToPath(import.meta.url) const __dirname = path.dirname(__filename) const dir = path.resolve(__dirname, "..") +const fffRequire = createRequire(import.meta.resolve("@ff-labs/fff-bun")) process.chdir(dir) +const dist = path.resolve(process.env.OPENCODE_BUILD_DIST ?? "dist") const generated = await import("./generate.ts") @@ -54,7 +59,7 @@ const embeddedFileMap = skipEmbedWebUi ? null : await createEmbeddedWebUIBundle( const treeSitterWorker = await Bun.file(fileURLToPath(import.meta.resolve("@opentui/core/parser.worker"))).text() const allTargets: { - os: string + os: "linux" | "darwin" | "win32" arch: "arm64" | "x64" abi?: "musl" avx2?: false @@ -137,7 +142,7 @@ const targets = singleFlag }) : allTargets -await $`rm -rf dist` +await fs.rm(dist, { recursive: true, force: true }) const binaries: Record = {} if (!skipInstall) { @@ -157,7 +162,8 @@ for (const item of targets) { .filter(Boolean) .join("-") console.log(`building ${name}`) - await $`mkdir -p dist/${name}/bin` + await fs.mkdir(path.join(dist, name, "bin"), { recursive: true }) + const nativeLibrary = await fffLibraryForTarget(item, (specifier) => fffRequire.resolve(specifier)) const workerPath = "./src/cli/tui/worker.ts" const watcherWorkerPath = "./src/cli/watcher-worker.ts" @@ -179,7 +185,7 @@ for (const item of targets) { autoloadTsconfig: true, autoloadPackageJson: true, target: name.replace(pkg.name, "bun") as any, - outfile: `dist/${name}/bin/opencode`, + outfile: path.join(dist, name, "bin", "opencode"), execArgv: [`--user-agent=opencode/${Script.version}`, "--use-system-ca", "--"], windows: {}, }, @@ -195,6 +201,7 @@ for (const item of targets) { ...(embeddedFileMap ? ["opencode-web-ui.gen.ts"] : []), ], define: { + ...fffBuildDefines(nativeLibrary), FFF_LIBC: JSON.stringify(item.abi === "musl" ? "musl" : "gnu"), OPENCODE_VERSION: `'${Script.version}'`, // The plugin SDK version THIS TREE was built against, taken from the @@ -213,9 +220,11 @@ for (const item of targets) { }, }) + await Bun.write(path.join(dist, name, "bin", nativeLibrary.filename), Bun.file(nativeLibrary.sourcePath)) + // Smoke test: only run if binary is for current platform if (item.os === process.platform && item.arch === process.arch && !item.abi) { - const binaryPath = `dist/${name}/bin/opencode` + const binaryPath = path.join(dist, name, "bin", "opencode") console.log(`Running smoke test: ${binaryPath} --version`) try { const versionOutput = await $`${binaryPath} --version`.text() @@ -226,8 +235,8 @@ for (const item of targets) { } } - await $`rm -rf ./dist/${name}/bin/tui` - await Bun.file(`dist/${name}/package.json`).write( + await fs.rm(path.join(dist, name, "bin", "tui"), { recursive: true, force: true }) + await Bun.file(path.join(dist, name, "package.json")).write( JSON.stringify( { name, @@ -247,9 +256,9 @@ for (const item of targets) { if (Script.release) { for (const key of Object.keys(binaries)) { if (key.includes("linux")) { - await $`tar -czf ../../${key}.tar.gz *`.cwd(`dist/${key}/bin`) + await $`tar -czf ../../${key}.tar.gz *`.cwd(path.join(dist, key, "bin")) } else { - await $`zip -r ../../${key}.zip *`.cwd(`dist/${key}/bin`) + await $`zip -r ../../${key}.zip *`.cwd(path.join(dist, key, "bin")) } } await $`gh release upload v${Script.version} ./dist/*.zip ./dist/*.tar.gz --clobber --repo ${process.env.GH_REPO}` diff --git a/packages/opencode/script/postinstall.mjs b/packages/opencode/script/postinstall.mjs index fa303b746b8c..4362c3b3d66a 100644 --- a/packages/opencode/script/postinstall.mjs +++ b/packages/opencode/script/postinstall.mjs @@ -27,6 +27,8 @@ const arch = archMap[os.arch()] ?? os.arch() const base = `opencode-${platform}-${arch}` const sourceBinary = platform === "windows" ? "opencode.exe" : "opencode" const targetBinary = path.join(__dirname, "bin", "opencode.exe") +const nativeLibrary = + platform === "windows" ? /^fff_c-.+\.dll$/ : platform === "darwin" ? /^libfff_c-.+\.dylib$/ : /^libfff_c-.+\.so$/ function supportsAvx2() { if (arch !== "x64") return false @@ -118,9 +120,16 @@ function packageNames() { function resolveBinary(name) { const packageJsonPath = require.resolve(`${name}/package.json`) - const binaryPath = path.join(path.dirname(packageJsonPath), "bin", sourceBinary) + const binDirectory = path.join(path.dirname(packageJsonPath), "bin") + const binaryPath = path.join(binDirectory, sourceBinary) if (!fs.existsSync(binaryPath)) throw new Error(`Binary not found at ${binaryPath}`) - return binaryPath + return { binaryPath, libraryPath: resolveNativeLibrary(binDirectory) } +} + +function resolveNativeLibrary(directory) { + const matches = fs.readdirSync(directory).filter((name) => nativeLibrary.test(name)) + if (matches.length !== 1) throw new Error(`Expected one FFF native library in ${directory}, found ${matches.length}`) + return path.join(directory, matches[0]) } function installPackage(name) { @@ -136,7 +145,10 @@ function installPackage(name) { ) if (result.status !== 0) return const packageDir = path.join(temp, "node_modules", name) - copyBinary(path.join(packageDir, "bin", sourceBinary), targetBinary) + const binDirectory = path.join(packageDir, "bin") + const libraryPath = resolveNativeLibrary(binDirectory) + copyNativeLibrary(libraryPath, path.join(__dirname, "bin", path.basename(libraryPath))) + copyBinary(path.join(binDirectory, sourceBinary), targetBinary) return true } finally { fs.rmSync(temp, { recursive: true, force: true }) @@ -155,6 +167,11 @@ function copyBinary(source, target) { fs.chmodSync(target, 0o755) } +function copyNativeLibrary(source, target) { + if (fs.existsSync(target) && fs.readFileSync(source).equals(fs.readFileSync(target))) return + copyBinary(source, target) +} + function verifyBinary() { const result = childProcess.spawnSync(targetBinary, ["--version"], { encoding: "utf8", @@ -167,7 +184,9 @@ function verifyBinary() { function main() { for (const name of packageNames()) { try { - copyBinary(resolveBinary(name), targetBinary) + const binary = resolveBinary(name) + copyNativeLibrary(binary.libraryPath, path.join(__dirname, "bin", path.basename(binary.libraryPath))) + copyBinary(binary.binaryPath, targetBinary) if (verifyBinary()) return } catch { if (installPackage(name) && verifyBinary()) return diff --git a/packages/opencode/script/publish.ts b/packages/opencode/script/publish.ts index ac1cafff56f4..5c1cc86cae52 100755 --- a/packages/opencode/script/publish.ts +++ b/packages/opencode/script/publish.ts @@ -120,6 +120,7 @@ if (!Script.preview) { "", "package() {", ' install -Dm755 ./opencode "${pkgdir}/usr/bin/opencode"', + ' install -Dm644 ./libfff_c-*.so -t "${pkgdir}/usr/bin"', "}", "", ].join("\n") @@ -163,6 +164,7 @@ if (!Script.preview) { "", " def install", ' bin.install "opencode"', + ' bin.install Dir["libfff_c-*.dylib"]', " end", " end", " if Hardware::CPU.arm?", @@ -171,6 +173,7 @@ if (!Script.preview) { "", " def install", ' bin.install "opencode"', + ' bin.install Dir["libfff_c-*.dylib"]', " end", " end", " end", @@ -181,6 +184,7 @@ if (!Script.preview) { ` sha256 "${x64Sha}"`, " def install", ' bin.install "opencode"', + ' bin.install Dir["libfff_c-*.so"]', " end", " end", " if Hardware::CPU.arm? and Hardware::CPU.is_64_bit?", @@ -188,6 +192,7 @@ if (!Script.preview) { ` sha256 "${arm64Sha}"`, " def install", ' bin.install "opencode"', + ' bin.install Dir["libfff_c-*.so"]', " end", " end", " end", diff --git a/packages/opencode/test/cli/db/retention.test.ts b/packages/opencode/test/cli/db/retention.test.ts index 31ec198e2060..117e10123af3 100644 --- a/packages/opencode/test/cli/db/retention.test.ts +++ b/packages/opencode/test/cli/db/retention.test.ts @@ -1307,7 +1307,7 @@ test( seq: 599, }) - // Keep a half-iterated source query cached; pinned Bun 1.3.14 must clear it before the old-inode check. + // Keep a half-iterated source query cached so compaction must release it before checking and replacing the source inode. using cachedSourceStatement = sourceFixture.db.query<{ seq: number }, [string]>( "SELECT seq FROM event_sequence WHERE aggregate_id = ?", ) diff --git a/packages/opencode/test/fff-compiled.test.ts b/packages/opencode/test/fff-compiled.test.ts new file mode 100644 index 000000000000..bfee9f18bf50 --- /dev/null +++ b/packages/opencode/test/fff-compiled.test.ts @@ -0,0 +1,160 @@ +import { describe, expect, test } from "bun:test" +import { getNpmPackageName } from "@ff-labs/fff-bun" +import { fffBuildDefines, fffLibraryForTarget } from "@opencode-ai/script/fff-native" +import fs from "node:fs/promises" +import { createRequire } from "node:module" +import os from "node:os" +import path from "node:path" + +const targetPlatform = process.platform === "win32" ? "windows" : process.platform +const libc = process.platform === "linux" && getNpmPackageName().endsWith("-musl") ? "musl" : "gnu" +const target = + `bun-${targetPlatform}-${process.arch}${process.platform === "linux" && libc === "musl" ? "-musl" : ""}` as Bun.Build.CompileTarget +const require = createRequire(import.meta.resolve("@ff-labs/fff-bun")) +const executablePath = process.env.OPENCODE_FFF_TEST_BUN_EXECUTABLE +const nativeLibrary = await fffLibraryForTarget( + { + os: process.platform === "win32" ? "win32" : process.platform === "darwin" ? "darwin" : "linux", + arch: process.arch as "arm64" | "x64", + ...(process.platform === "linux" && libc === "musl" ? { abi: "musl" as const } : {}), + }, + (specifier) => require.resolve(specifier), +) +const library = Buffer.from(await Bun.file(nativeLibrary.sourcePath).arrayBuffer()) +const extension = process.platform === "darwin" ? ".dylib" : process.platform === "win32" ? ".dll" : ".so" + +describe("compiled FFF extraction", () => { + test("keeps native search usable without temp allocations in the sidecar build", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "opencode-fff-compiled-")) + try { + const temporary = path.join(root, "tmp") + const workspace = path.join(root, "workspace") + const binary = path.join(root, "opencode") + await fs.mkdir(path.join(workspace, "src", "nested"), { recursive: true }) + await fs.mkdir(temporary) + await fs.writeFile(path.join(workspace, "src", "nested", "needle-target.ts"), "picker-content\n") + + const build = async (outfile: string, externalLibrary?: string, executable?: string) => + Bun.build({ + entrypoints: [path.join(import.meta.dir, "fixture/fff-compiled.ts")], + format: "esm", + minify: true, + splitting: true, + define: { + ...(externalLibrary ? fffBuildDefines({ filename: externalLibrary }) : {}), + FFF_LIBC: JSON.stringify(libc), + }, + compile: { + autoloadBunfig: false, + autoloadDotenv: false, + target, + outfile, + ...(executable ? { executablePath: executable } : {}), + }, + }) + + const allocations = async (temporaryDirectory: string) => { + const findLibraries = async (directory: string): Promise => { + const entries = await fs.readdir(directory, { withFileTypes: true }) + return ( + await Promise.all( + entries.map(async (entry) => { + const file = path.join(directory, entry.name) + if (entry.isDirectory()) return findLibraries(file) + return entry.isFile() && entry.name.endsWith(extension) ? [file] : [] + }), + ) + ).flat() + } + + const candidates = await findLibraries(temporaryDirectory) + return Promise.all( + candidates.map(async (file) => { + const content = await fs.readFile(file) + return { file, byteLength: content.byteLength, contentMatches: content.equals(library) } + }), + ) + } + + const launch = (executable: string, temporaryDirectory: string, mode: "complete" | "hold") => + Bun.spawn([executable, workspace, mode], { + cwd: root, + env: { ...process.env, BUN_TMPDIR: temporaryDirectory, TMPDIR: temporaryDirectory }, + stdout: "pipe", + stderr: "pipe", + }) + + const complete = async (executable: string, temporaryDirectory: string) => { + const child = launch(executable, temporaryDirectory, "complete") + const timeout = setTimeout(() => child.kill("SIGKILL"), 15_000) + try { + const [stdout, stderr, status] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]) + expect(status, stderr).toBe(0) + expect(stdout).toContain("ready") + } finally { + clearTimeout(timeout) + } + } + + const forceExit = async ( + executable: string, + temporaryDirectory: string, + expected: Awaited>, + ) => { + const child = launch(executable, temporaryDirectory, "hold") + const reader = child.stdout.getReader() + const timeout = setTimeout(() => child.kill("SIGKILL"), 15_000) + try { + let output = "" + while (!output.includes("\n")) { + const chunk = await reader.read() + if (chunk.done) throw new Error("compiled FFF process exited before its readiness signal") + output += new TextDecoder().decode(chunk.value) + } + expect(output.split("\n", 1)[0]).toBe("ready") + expect(await allocations(temporaryDirectory)).toEqual(expected) + } finally { + clearTimeout(timeout) + child.kill("SIGKILL") + } + expect(await child.exited).not.toBe(0) + expect(await allocations(temporaryDirectory)).toEqual(expected) + } + + const before = await allocations(temporary) + expect(before).toEqual([]) + + const embeddedBuild = await build(binary, undefined, executablePath) + expect(embeddedBuild.success).toBe(true) + await complete(binary, temporary) + const afterFirstNormalExit = await allocations(temporary) + expect(afterFirstNormalExit).toHaveLength(1) + expect(afterFirstNormalExit[0].contentMatches).toBe(true) + await complete(binary, temporary) + const afterNormalExits = await allocations(temporary) + expect(afterNormalExits).toEqual(afterFirstNormalExit) + + const sidecarTemporary = path.join(root, "sidecar-tmp") + const sidecarBinary = path.join(root, "opencode-sidecar") + await fs.mkdir(sidecarTemporary) + await Bun.write(path.join(root, nativeLibrary.filename), Bun.file(nativeLibrary.sourcePath)) + const sidecarBuild = await build(sidecarBinary, nativeLibrary.filename) + expect(sidecarBuild.success).toBe(true) + expect(await allocations(sidecarTemporary)).toEqual([]) + + await complete(sidecarBinary, sidecarTemporary) + expect(await allocations(sidecarTemporary)).toEqual([]) + await complete(sidecarBinary, sidecarTemporary) + expect(await allocations(sidecarTemporary)).toEqual([]) + await forceExit(sidecarBinary, sidecarTemporary, []) + await forceExit(sidecarBinary, sidecarTemporary, []) + expect(await allocations(sidecarTemporary)).toEqual([]) + } finally { + await fs.rm(root, { recursive: true, force: true }) + } + }, 180_000) +}) diff --git a/packages/opencode/test/fff-distribution.test.ts b/packages/opencode/test/fff-distribution.test.ts new file mode 100644 index 000000000000..cad86bfee629 --- /dev/null +++ b/packages/opencode/test/fff-distribution.test.ts @@ -0,0 +1,288 @@ +import { describe, expect, test } from "bun:test" +import { fffLibraryForTarget } from "@opencode-ai/script/fff-native" +import { packagedFffUnavailableMessage } from "@opencode-ai/core/filesystem/search" +import { createRequire } from "node:module" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" + +const targetOS = process.platform as "linux" | "darwin" | "win32" +const targetArch = process.arch as "arm64" | "x64" +const targetName = targetOS === "win32" ? "windows" : targetOS +const require = createRequire(import.meta.resolve("@ff-labs/fff-bun")) +const nativeLibrary = await fffLibraryForTarget({ os: targetOS, arch: targetArch }, (specifier) => + require.resolve(specifier), +) +const nativeBytes = Buffer.from(await Bun.file(nativeLibrary.sourcePath).arrayBuffer()) +const nativeDigest = new Bun.CryptoHasher("sha256").update(nativeBytes).digest("hex") +const fffDownload = await import(new URL("./download.ts", import.meta.resolve("@ff-labs/fff-bun")).href) + +test("names the missing packaged FFF sidecar and its recovery flag", () => { + const expectedPath = path.join(path.dirname(process.execPath), nativeLibrary.filename) + const message = packagedFffUnavailableMessage(nativeLibrary.filename) + + expect(message).toContain(expectedPath) + expect(message).toContain("OPENCODE_DISABLE_FFF=1") +}) + +describe("compiled OpenCode FFF distribution", () => { + test("loads the packaged native sidecar and leaves no temp addon allocations across exits", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "opencode-fff-distribution-")) + const dist = path.join(root, "dist") + const temporaryRoot = path.join(root, "tmp") + const workspace = path.join(root, "workspace") + const home = path.join(root, "home") + const buildTemporary = path.join(root, "build-tmp") + + try { + await Promise.all([ + fs.mkdir(temporaryRoot, { recursive: true }), + fs.mkdir(workspace, { recursive: true }), + fs.mkdir(home, { recursive: true }), + fs.mkdir(buildTemporary, { recursive: true }), + ]) + await fs.mkdir(path.join(workspace, "src", "nested"), { recursive: true }) + await fs.writeFile(path.join(workspace, "src", "nested", "needle-target.ts"), "picker-content\n") + + const build = Bun.spawn( + ["bun", "run", "script/build.ts", "--single", "--skip-install", "--skip-embed-web-ui"], + { + cwd: path.resolve(import.meta.dir, ".."), + env: { + ...process.env, + OPENCODE_BUILD_DIST: dist, + BUN_TMPDIR: buildTemporary, + TMPDIR: buildTemporary, + }, + stdout: "pipe", + stderr: "pipe", + }, + ) + const buildTimeout = setTimeout(() => build.kill("SIGKILL"), 180_000) + const [buildStdout, buildStderr, buildStatus] = await Promise.all([ + new Response(build.stdout).text(), + new Response(build.stderr).text(), + build.exited, + ]) + clearTimeout(buildTimeout) + expect(buildStatus, `${buildStdout}\n${buildStderr}`).toBe(0) + + const output = path.join(dist, `opencode-${targetName}-${targetArch}`, "bin") + const binary = [path.join(output, "opencode"), path.join(output, "opencode.exe")].find((file) => + Bun.file(file).size > 0, + ) + expect(binary).toBeDefined() + const sidecar = path.join(output, nativeLibrary.filename) + expect(await fs.readFile(sidecar)).toEqual(nativeBytes) + expect(nativeLibrary.filename).toContain(nativeDigest) + console.log(`FFF sidecar target=${targetOS}-${targetArch} filename=${nativeLibrary.filename} sha256=${nativeDigest}`) + expect(await addonAllocations(temporaryRoot)).toEqual([]) + + const normalTemporary = path.join(temporaryRoot, "normal") + await fs.mkdir(normalTemporary) + const normal = await run( + [binary!, "debug", "file", "search", "needle-target"], + workspace, + childEnvironment(root, normalTemporary), + ) + expect(normal.status, `${normal.stdout}\n${normal.stderr}`).toBe(0) + expect(normal.stdout).toContain("needle-target.ts") + expect(await addonAllocations(temporaryRoot)).toEqual([]) + + const missingSidecar = `${sidecar}.missing` + await fs.rename(sidecar, missingSidecar) + try { + const missingTemporary = path.join(temporaryRoot, "missing") + await fs.mkdir(missingTemporary) + const missing = await run( + [binary!, "debug", "file", "search", "needle-target"], + workspace, + childEnvironment(root, missingTemporary), + ) + expect(missing.status, `${missing.stdout}\n${missing.stderr}`).not.toBe(0) + } finally { + await fs.rename(missingSidecar, sidecar) + } + expect(await addonAllocations(temporaryRoot)).toEqual([]) + + for (const index of [0, 1]) { + const tmp = path.join(temporaryRoot, `forced-${index}`) + await fs.mkdir(tmp) + const server = Bun.spawn([binary!, "serve", "--hostname", "127.0.0.1", "--port", "0"], { + cwd: workspace, + env: childEnvironment(root, tmp), + stdout: "pipe", + stderr: "pipe", + }) + let serverStopped = false + const stderr = new Response(server.stderr).text() + try { + const address = await waitForServer(server, stderr) + const files = await findFile(address, workspace) + expect(files.some((file) => file.includes("needle-target.ts"))).toBe(true) + expect(await nativeLibraryIsMapped(server.pid, sidecar)).toBe(true) + expect(await addonAllocations(temporaryRoot)).toEqual([]) + server.kill("SIGKILL") + expect(await server.exited).not.toBe(0) + serverStopped = true + expect(await addonAllocations(temporaryRoot)).toEqual([]) + } finally { + if (!serverStopped) { + server.kill("SIGKILL") + await server.exited + } + await stderr + } + } + } finally { + await fs.rm(root, { recursive: true, force: true }) + } + }, 300_000) + + test("resolves sidecars beside Windows compiled roots", () => { + const filename = "fff_c-identity.dll" + const executablePath = path.join("B:/~BUN/root", "opencode.exe") + const expected = path.join("B:/~BUN/root", filename) + const resolved = fffDownload.resolveCompiledLibraryPath( + executablePath, + filename, + (candidate: string) => candidate === expected, + ) + expect(resolved).toBe(expected) + }) +}) + +async function run(command: string[], cwd: string, env: Record) { + const child = Bun.spawn(command, { cwd, env, stdout: "pipe", stderr: "pipe" }) + const timeout = setTimeout(() => child.kill("SIGKILL"), 30_000) + try { + const [stdout, stderr, status] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]) + return { stdout, stderr, status } + } finally { + clearTimeout(timeout) + } +} + +function childEnvironment(root: string, temporary: string) { + const home = path.join(root, "home") + const env: Record = { + ...process.env, + BUN_TMPDIR: temporary, + TMPDIR: temporary, + TMP: temporary, + TEMP: temporary, + HOME: home, + USERPROFILE: home, + APPDATA: path.join(home, "AppData", "Roaming"), + LOCALAPPDATA: path.join(home, "AppData", "Local"), + OPENCODE_TEST_HOME: home, + OPENCODE_CONFIG_DIR: path.join(home, "config"), + XDG_CONFIG_HOME: path.join(home, ".config"), + XDG_DATA_HOME: path.join(home, ".local", "share"), + XDG_CACHE_HOME: path.join(home, ".cache"), + OPENCODE_DISABLE_AUTOUPDATE: "1", + OPENCODE_DISABLE_MODELS_FETCH: "1", + OPENCODE_DISABLE_PROJECT_CONFIG: "1", + OPENCODE_PURE: "1", + } + delete env.OPENCODE_DISABLE_FFF + return env +} + +async function waitForServer(server: Bun.Subprocess, stderr: Promise) { + const stdout = server.stdout + if (!stdout || typeof stdout === "number") throw new Error("server stdout is not piped") + const reader = stdout.getReader() + const timeout = setTimeout(() => server.kill("SIGKILL"), 45_000) + let output = "" + try { + while (true) { + const chunk = await reader.read() + if (chunk.done) { + await server.exited + throw new Error(`server exited before its listening signal: ${output}\n${await stderr}`) + } + output += new TextDecoder().decode(chunk.value) + const match = output.match(/opencode server listening on (https?:\/\/\S+)/) + if (!match) continue + void drain(reader) + return match[1] + } + } finally { + clearTimeout(timeout) + } +} + +async function drain(reader: ReadableStreamDefaultReader) { + while (!(await reader.read()).done) {} +} + +async function findFile(address: string, directory: string) { + const url = new URL("/find/file", address) + url.searchParams.set("query", "needle-target") + url.searchParams.set("directory", directory) + url.searchParams.set("limit", "10") + const deadline = Date.now() + 30_000 + while (Date.now() < deadline) { + const response = await fetch(url, { signal: AbortSignal.timeout(5_000) }) + if (!response.ok) throw new Error(`file search returned HTTP ${response.status}`) + const files = (await response.json()) as string[] + if (files.some((file) => file.includes("needle-target.ts"))) return files + await Bun.sleep(50) + } + throw new Error("compiled file search did not find its fixture before the failure deadline") +} + +async function nativeLibraryIsMapped(pid: number, sidecar: string) { + const name = path.basename(sidecar).toLowerCase() + if (process.platform === "linux") { + return (await fs.readFile(`/proc/${pid}/maps`, "utf8")).toLowerCase().includes(name) + } + const command = + process.platform === "win32" + ? [ + "powershell.exe", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-Command", + `[Console]::OutputEncoding = [System.Text.UTF8Encoding]::new(); (Get-Process -Id ${pid}).Modules | ForEach-Object { $_.FileName }`, + ] + : ["vmmap", String(pid)] + const result = await run(command, process.cwd(), process.env) + expect(result.status, result.stderr).toBe(0) + return result.stdout.toLowerCase().includes(name) +} + +async function addonAllocations(directory: string) { + const nativeExtensions = new Set([".so", ".dylib", ".dll"]) + const found: { path: string; byteLength: number; digest: string; matchesFFF: boolean }[] = [] + const visit = async (current: string): Promise => { + for (const entry of await fs.readdir(current, { withFileTypes: true })) { + const file = path.join(current, entry.name) + if (entry.isDirectory()) { + await visit(file) + continue + } + if (!entry.isFile() || !nativeExtensions.has(path.extname(entry.name).toLowerCase())) continue + const contents = await fs.readFile(file) + const matchesFFF = + contents.byteLength === 0 || + (contents.byteLength <= nativeBytes.byteLength && nativeBytes.subarray(0, contents.byteLength).equals(contents)) + if (!matchesFFF) continue + found.push({ + path: file, + byteLength: contents.byteLength, + digest: new Bun.CryptoHasher("sha256").update(contents).digest("hex"), + matchesFFF: contents.equals(nativeBytes), + }) + } + } + await visit(directory) + return found +} diff --git a/packages/opencode/test/fixture/fff-compiled.ts b/packages/opencode/test/fixture/fff-compiled.ts new file mode 100644 index 000000000000..aa80a8a79e72 --- /dev/null +++ b/packages/opencode/test/fixture/fff-compiled.ts @@ -0,0 +1,46 @@ +import { FileFinder } from "@ff-labs/fff-bun" + +const basePath = process.argv[2] +const mode = process.argv[3] +if (!basePath || (mode !== "complete" && mode !== "hold")) throw new Error("expected base path and mode") + +const created = FileFinder.create({ + basePath, + aiMode: true, + disableMmapCache: true, + disableContentIndexing: true, + disableWatch: true, +}) +if (!created.ok) throw new Error(created.error) + +const scanned = await created.value.waitForScan(15_000) +if (!scanned.ok || !scanned.value) throw new Error("FFF scan did not finish") + +const file = created.value.fileSearch("needle-target", { pageSize: 10 }) +if (!file.ok || !file.value.items.some((item) => item.relativePath.endsWith("needle-target.ts"))) { + throw new Error("FFF file search did not find the indexed file") +} + +const directory = created.value.directorySearch("nested", { pageSize: 10 }) +if (!directory.ok || !directory.value.items.some((item) => item.relativePath.includes("nested"))) { + throw new Error("FFF directory search did not find the indexed directory") +} + +const glob = created.value.glob("src/nested/*.ts", { pageSize: 10 }) +if (!glob.ok || !glob.value.items.some((item) => item.relativePath.endsWith("needle-target.ts"))) { + throw new Error("FFF glob did not find the indexed file") +} + +const grep = created.value.grep("picker-content", { mode: "plain", pageSize: 10 }) +if (!grep.ok || !grep.value.items.some((item) => item.lineContent.includes("picker-content"))) { + throw new Error("FFF grep did not find the indexed content") +} + +if (mode === "hold") { + process.stdout.write("ready\n") + setInterval(() => {}, 1_000) + await new Promise(() => {}) +} + +created.value.destroy() +process.stdout.write("ready\n") diff --git a/packages/script/package.json b/packages/script/package.json index aa0d2f563f51..ed627b77b363 100644 --- a/packages/script/package.json +++ b/packages/script/package.json @@ -10,6 +10,7 @@ "@types/semver": "^7.5.8" }, "exports": { - ".": "./src/index.ts" + ".": "./src/index.ts", + "./fff-native": "./src/fff-native.ts" } } diff --git a/packages/script/src/fff-native.ts b/packages/script/src/fff-native.ts new file mode 100644 index 000000000000..74b6fb70ced4 --- /dev/null +++ b/packages/script/src/fff-native.ts @@ -0,0 +1,23 @@ +import path from "node:path" + +export async function fffLibraryForTarget( + target: { os: "linux" | "darwin" | "win32"; arch: "arm64" | "x64"; abi?: "musl" }, + resolvePackage: (specifier: string) => string, +) { + const libc = target.os === "linux" ? `-${target.abi ?? "gnu"}` : "" + const name = `@ff-labs/fff-bin-${target.os}-${target.arch}${libc}` + const packageJsonPath = resolvePackage(`${name}/package.json`) + const library = target.os === "win32" ? "fff_c" : "libfff_c" + const extension = target.os === "win32" ? "dll" : target.os === "darwin" ? "dylib" : "so" + const directory = path.dirname(packageJsonPath) + const sourcePath = path.join(directory, `${library}.${extension}`) + const digest = new Bun.CryptoHasher("sha256").update(await Bun.file(sourcePath).arrayBuffer()).digest("hex") + return { + filename: `${library}-${digest}.${extension}`, + sourcePath, + } +} + +export function fffBuildDefines(nativeLibrary: { filename: string }) { + return { FFF_BUN_EXTERNAL_LIBRARY: JSON.stringify(nativeLibrary.filename) } +} diff --git a/patches/@ff-labs%2Ffff-bun@0.9.4.patch b/patches/@ff-labs%2Ffff-bun@0.9.4.patch new file mode 100644 index 000000000000..305631d1d2dd --- /dev/null +++ b/patches/@ff-labs%2Ffff-bun@0.9.4.patch @@ -0,0 +1,39 @@ +diff --git a/src/download.ts b/src/download.ts +--- a/src/download.ts ++++ b/src/download.ts +@@ -14,0 +15,2 @@ ++declare const FFF_BUN_EXTERNAL_LIBRARY: string; ++ +@@ -103,0 +106,11 @@ ++export function resolveCompiledLibraryPath( ++ executablePath = process.execPath, ++ filename = typeof FFF_BUN_EXTERNAL_LIBRARY === "string" ? FFF_BUN_EXTERNAL_LIBRARY : undefined, ++ exists = existsSync, ++) { ++ if (!filename) return null; ++ ++ const libraryPath = join(dirname(executablePath), filename); ++ return exists(libraryPath) ? libraryPath : null; ++} ++ +@@ -113,0 +127,4 @@ ++ const compiledLibraryPath = resolveCompiledLibraryPath(); ++ if (compiledLibraryPath) return compiledLibraryPath; ++ if (typeof FFF_BUN_EXTERNAL_LIBRARY === "string") return null; ++ +diff --git a/src/embedded.ts b/src/embedded.ts +--- a/src/embedded.ts ++++ b/src/embedded.ts +@@ -2 +2 @@ +- * Tools for standalone-executable native library embedding. ++ * Default embedded path for standalone-executable native libraries. +@@ -12,0 +13,2 @@ ++ * OpenCode uses a packaged native sidecar so compiled addons stay out of Bun's ++ * scratch-file extraction path. +@@ -14,0 +17,2 @@ ++declare const FFF_BUN_EXTERNAL_LIBRARY: string; ++ +@@ -53 +57,2 @@ +-export const embeddedLibPath: string | null = await resolveEmbeddedLibPath(); ++export const embeddedLibPath: string | null = ++ typeof FFF_BUN_EXTERNAL_LIBRARY === "string" ? null : await resolveEmbeddedLibPath(); diff --git a/script/check-workflow-guards.ts b/script/check-workflow-guards.ts index 6628aead63e7..51f01e4ac717 100644 --- a/script/check-workflow-guards.ts +++ b/script/check-workflow-guards.ts @@ -33,6 +33,8 @@ const GUARD = "github.repository == 'anomalyco/opencode'" * guarded on four jobs out of five. */ const ALLOWED = new Map([ + ["fff-platform-validation.yml::windows-fff", "f04bf8aa6b7a6845"], + ["fff-platform-validation.yml::nix-node-modules", "454365e67a9a2ffe"], ["test.yml::unit", "fd0a927305590ae6"], ["test.yml::e2e", "9f89116be327fa02"], ["typecheck.yml::typecheck", "276be38badce0802"], @@ -59,6 +61,7 @@ const ALLOWED_ACTIONS = new Map([["setup-bun", "761c61a6a87fd34a"]]) * RUNS is part of what is being trusted. */ const ALLOWED_ENVELOPES = new Map([ + ["fff-platform-validation.yml", "7c3c7fc7328f12b6"], ["test.yml", "0e4560dca190d1b3"], ["typecheck.yml", "783bae590157a011"], ])