From bce57531b758675fa34d944e49a8c3b3d399374b Mon Sep 17 00:00:00 2001 From: Quentin Date: Tue, 6 Oct 2026 09:41:03 +0200 Subject: [PATCH 1/5] Run every CI check as its own job, and start the release builds beside the gate The Rust and JS checks each run on their own runner, gathered under the two names the main ruleset requires, so a PR waits for the slowest check instead of their sum. On main the binaries, the .vsix and the browser zips build while the suite runs; the images, the release and every store publish still wait for it to pass. --- .claude/skills/releasing/SKILL.md | 21 +++-- .github/workflows/ci.yml | 146 +++++++++++++++++++++++++++--- .github/workflows/release.yml | 73 ++++++++++----- 3 files changed, 200 insertions(+), 40 deletions(-) diff --git a/.claude/skills/releasing/SKILL.md b/.claude/skills/releasing/SKILL.md index f9cf7d93..3b692bbe 100644 --- a/.claude/skills/releasing/SKILL.md +++ b/.claude/skills/releasing/SKILL.md @@ -38,10 +38,16 @@ into `main` through a PR, and the merge is what publishes. CLI binaries, the `.vsix` and the browser zips, updates the rolling `latest` release, and then publishes to each store. -CI runs on `pull_request` into `main` only. Six checks: `extension -versions bumped` (PRs only), `build`, `public proxy routes`, -`cargo test + clippy`, `cargo test --features db-tests`, and -`graph + editor`. +CI runs on `pull_request` into `main` only. Every check is its own +job, so they run side by side. The main ruleset requires four names: +`extension versions bumped` (PRs only), `cargo test --features +db-tests`, and two gates that are green only when every job under them +is: `cargo test + clippy` (nextest, doc tests, clippy, the parse-only +build, the examples) and `graph + editor` (each JS package, the VS Code +extension, the WhatsApp bridge, the browser extension). The docs book +builds beside them. On `main`, `release.yml` runs the same suite as its +gate, and starts the builds (binaries, .vsix, browser zips) beside it; +only what publishes waits for the gate. ## Before every push to a PR, run what the PR runs @@ -50,8 +56,11 @@ opens it, and every fix pushed after), run locally everything the PR's checks run, not only the tests your change reaches. This is the one place the [test scope] does not apply: a red PR costs a full CI round trip, and twice a PR has gone red on a test in a crate the change only -touched indirectly. Background each one, and fix everything before the -push, not after. +touched indirectly. Background the run, and fix everything before the +push, not after. Locally the cargo commands go one after another (they +share one build directory, which cargo locks, and overlapping cargo runs +have crashed this machine); the JS checks and the docs build run beside +them. The PR runs `.github/workflows/ci.yml` and `.github/workflows/docs.yml`; read them for the current list. At the time of writing: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c60308d3..697e81fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,8 +48,14 @@ jobs: fetch-depth: 0 - run: scripts/check-extension-bump.sh "origin/${{ github.base_ref }}" - rust: - name: cargo test + clippy + # The Rust checks, each on its own runner so they run side by side: + # the wall time is the slowest one instead of their sum. Each builds + # what it needs from the tests job's cache (`shared-key`), which only + # that job saves, on main: one warm copy in the repo's cache space + # instead of one per job. The `rust` job below gathers them under the + # one name the main ruleset requires. + test: + name: cargo nextest runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -61,29 +67,89 @@ jobs: - run: rustup toolchain install # Restores everywhere, saves only on main (release.yml's call of # this file): a PR reads main's copy and leaves nothing behind, and - # release.yml's prune-caches keeps one copy per job. + # release.yml's prune-caches keeps one copy per cache. - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 with: + shared-key: workspace save-if: ${{ github.ref == 'refs/heads/main' }} - # No compile-time query macros anywhere, so the workspace suite needs no # database. The tests that do want one are behind the `db-tests` feature. # Under cargo-nextest, as developers run it: each test in its own - # process, which tests that set the process's env vars rely on. Doc - # tests are the one kind nextest does not run. + # process, which tests that set the process's env vars rely on. - run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin" - run: cargo nextest run --workspace --locked + + doc: + name: cargo doc tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: rustup toolchain install + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + with: + shared-key: workspace + save-if: false + # Doc tests are the one kind nextest does not run. - run: cargo test --workspace --locked --doc + + clippy: + name: cargo clippy + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: rustup toolchain install + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + with: + shared-key: workspace + save-if: false - run: cargo clippy --workspace --all-targets --locked -- -D warnings + + parse-only: + name: parse-only build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: rustup toolchain install + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + with: + shared-key: workspace + save-if: false # The parse-only build (weft-core without its `runtime` layer, what a # WASM parser links). A workspace build always turns `runtime` on, so # nothing else compiles this shape. - run: cargo check -p weft-compiler --no-default-features --locked + + examples: + name: examples compile + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: rustup toolchain install + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + with: + shared-key: workspace + save-if: false # The examples are the first thing a new user copies, and nothing # else compiles them. The script tolerates only the # connect-a-credential diagnostics a fresh checkout produces. - run: scripts/validate-examples.sh + # The name the main ruleset requires: green only when every Rust check + # above is. It runs whatever happened above (unless the run was + # cancelled), so a failure there shows here as a failure, never as a + # skip the ruleset would wait on. + rust: + name: cargo test + clippy + needs: [test, doc, clippy, parse-only, examples] + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + env: + NEEDS: ${{ toJSON(needs) }} + steps: + - run: | + jq -e 'all(.[]; .result == "success")' <<< "$NEEDS" >/dev/null \ + || { jq -r 'to_entries[] | "\(.key): \(.value.result)"' <<< "$NEEDS"; exit 1; } + db: name: cargo test --features db-tests runs-on: ubuntu-latest @@ -115,8 +181,10 @@ jobs: # container, so the crate list lives in exactly one place. - run: scripts/run-db-tests.sh - editor: - name: graph + editor + # The JS checks, side by side like the Rust ones, gathered under the + # name the main ruleset requires by the `editor` job below. + graph: + name: weft-graph tests runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -125,17 +193,71 @@ jobs: workspace: extension-vscode borrow-packages: packages/weft-graph packages/weft-syntax packages/weft-connect - run: pnpm -C packages/weft-graph test + + connect: + name: weft-connect tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-node-workspace + with: + workspace: extension-vscode + borrow-packages: packages/weft-graph packages/weft-syntax packages/weft-connect - run: pnpm -C packages/weft-connect test + + syntax: + name: weft-syntax tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-node-workspace + with: + workspace: extension-vscode + borrow-packages: packages/weft-graph packages/weft-syntax packages/weft-connect - run: pnpm -C packages/weft-syntax test + + vscode: + name: VS Code extension + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-node-workspace + with: + workspace: extension-vscode + borrow-packages: packages/weft-graph packages/weft-syntax packages/weft-connect - run: pnpm -C extension-vscode test - run: pnpm -C extension-vscode run compile - # The WhatsApp bridge's pure parts; `node --test` needs no install. + + bridge: + name: WhatsApp bridge tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + # The bridge's pure parts; `node --test` needs no install. - run: node --test catalog/bailey/bridge/images/bridge/src/*.test.js - # The browser extension ships to three stores gated only on a - # version bump, so its checks belong in the same gate as - # everything else JS. + + # The browser extension ships to three stores gated only on a version + # bump, so its checks belong in the same gate as everything else JS. + browser: + name: browser extension + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 - uses: ./.github/actions/setup-node-workspace with: workspace: extension-browser - run: pnpm -C extension-browser exec tsc --noEmit - run: pnpm -C extension-browser check + + # The name the main ruleset requires (see the `rust` job). + editor: + name: graph + editor + needs: [graph, connect, syntax, vscode, bridge, browser] + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + env: + NEEDS: ${{ toJSON(needs) }} + steps: + - run: | + jq -e 'all(.[]; .result == "success")' <<< "$NEEDS" >/dev/null \ + || { jq -r 'to_entries[] | "\(.key): \(.value.result)"' <<< "$NEEDS"; exit 1; } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e860845d..db1c769a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -47,9 +47,11 @@ jobs: # ---- the gate -------------------------------------------------------- # # The full CI suite (tests, clippy, schema agreement, editor checks) - # runs first and every publishing job below waits on it. A commit - # that fails the checks publishes NOTHING: no images, no binaries, no - # store uploads. Whoever pulls such a commit builds from source and + # runs and every publishing job below waits on it. What only builds + # (the binaries, the .vsix, the browser zips) starts at once beside it, + # so the slow compiles overlap the checks; nothing it makes leaves the + # run's own artifacts until the gate is green. A commit that fails the + # checks publishes NOTHING: no images, no binaries, no store uploads. Whoever pulls such a commit builds from source and # hits the same loud local failure the dev did; their data is safe # either way (the boot refuses on schema drift instead of guessing). # @@ -76,9 +78,9 @@ jobs: # it a second time inside Docker. That works because these legs run on # Ubuntu 22.04, whose glibc is older than the image's Debian bookworm # (`weft build-images` starts the binary in the image to make sure). - linux: - name: ${{ matrix.asset }} + images - needs: [ci] + linux-binaries: + name: ${{ matrix.asset }} + if: github.repository == 'WeaveMindAI/weft' strategy: # One architecture failing must not kill the other's build # mid-way; the stitch below requires both, so nothing partial is @@ -89,23 +91,50 @@ jobs: # ubuntu-22.04: oldest supported runner glibc, so the binary # runs on every distro at least that new, and in the image. # SYNC: CLI asset names <-> setup.sh (prebuilt artifacts - # block: the uname case), the macos job's matrix + # block: the uname case), the macos job's matrix, the linux job's matrix - { runner: ubuntu-22.04, asset: weft-x86_64-linux, arch: amd64 } - { runner: ubuntu-22.04-arm, asset: weft-aarch64-linux, arch: arm64 } runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/build-binaries + with: + asset: ${{ matrix.asset }} + + # The images push to ghcr, so they wait on the gate; the binaries they + # carry come from the leg above. + linux: + name: ${{ matrix.asset }} images + needs: [ci, linux-binaries] + strategy: + fail-fast: false + matrix: + include: + # SYNC: the matrix <-> the linux-binaries job's matrix + - { runner: ubuntu-22.04, asset: weft-x86_64-linux, runtime: weft-runtime-x86_64-linux, arch: amd64 } + - { runner: ubuntu-22.04-arm, asset: weft-aarch64-linux, runtime: weft-runtime-aarch64-linux, arch: arm64 } + runs-on: ${{ matrix.runner }} permissions: contents: read packages: write steps: - uses: actions/checkout@v4 # The builder base alone is >1GB and the builds run through - # BuildKit on the same disk as the cargo cache; the preinstalled - # Android/.NET/GHC toolchains are dead weight here. + # BuildKit on this disk; the preinstalled Android/.NET/GHC + # toolchains are dead weight here. - name: free disk space run: sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/.ghcup - - uses: ./.github/actions/build-binaries + - uses: actions/download-artifact@v4 with: - asset: ${{ matrix.asset }} + name: ${{ matrix.asset }} + path: ${{ runner.temp }}/bin + - uses: actions/download-artifact@v4 + with: + name: ${{ matrix.runtime }} + path: ${{ runner.temp }}/bin + # Outside the checkout: the image refs hash the source tree, and the + # two legs' binaries differ. + - run: chmod +x "$RUNNER_TEMP"/bin/* - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 with: registry: ghcr.io @@ -119,9 +148,9 @@ jobs: # SYNC: `weft build-images` stdout <-> crates/weft-cli/src/images.rs (SharedImages::bare_refs), .github/workflows/install-gcp.yml (build the runtime's images) - name: build + push shared images run: | - ./target/release/weft build-images \ + "$RUNNER_TEMP/bin/${{ matrix.asset }}" build-images \ --push-suffix "-${{ matrix.arch }}" \ - --runtime-binary target/release/weft-runtime > image-refs.txt + --runtime-binary "$RUNNER_TEMP/bin/${{ matrix.runtime }}" > image-refs.txt - uses: actions/upload-artifact@v4 with: name: image-refs-${{ matrix.arch }} @@ -136,7 +165,7 @@ jobs: # whole release by far. macos: name: ${{ matrix.asset }} - needs: [ci] + if: github.repository == 'WeaveMindAI/weft' strategy: fail-fast: false matrix: @@ -222,7 +251,7 @@ jobs: # saves any more goes on its own after seven days unused. prune-caches: name: keep one copy of each build cache - needs: [ci, linux, macos] + needs: [ci, linux-binaries, macos] if: ${{ !cancelled() }} runs-on: ubuntu-latest permissions: @@ -248,7 +277,7 @@ jobs: # ---- VS Code extension ---------------------------------------------- vsix: name: package .vsix - needs: [ci] + if: github.repository == 'WeaveMindAI/weft' runs-on: ubuntu-latest outputs: version: ${{ steps.tags.outputs.version }} @@ -291,7 +320,7 @@ jobs: # ---- browser extension ---------------------------------------------- browser: name: package browser zips - needs: [ci] + if: github.repository == 'WeaveMindAI/weft' runs-on: ubuntu-latest outputs: version: ${{ steps.tags.outputs.version }} @@ -325,7 +354,7 @@ jobs: release: name: update latest release runs-on: ubuntu-latest - needs: [linux, vsix, browser, images-manifest] + needs: [ci, linux, vsix, browser, images-manifest] # Runs even when the browser packaging failed (one unbuildable # artifact must not strand every other's assets), but never without # the vsix job (it supplies the version the manifest needs) and never @@ -335,7 +364,7 @@ jobs: # on, and publishing it while no image is pullable would hand a # toolchain-less user a prebuilt CLI whose first `daemon start` # falls back to a multi-GB local docker build. - if: ${{ !cancelled() && needs.vsix.result == 'success' && needs['images-manifest'].result == 'success' }} + if: ${{ !cancelled() && needs.ci.result == 'success' && needs.vsix.result == 'success' && needs['images-manifest'].result == 'success' }} permissions: contents: write env: @@ -468,7 +497,7 @@ jobs: publish-marketplace: name: publish to VS Code Marketplace runs-on: ubuntu-latest - needs: [vsix] + needs: [ci, vsix] # `contains` on the pending ARRAY fails closed: an unknown store # name simply is not in it. if: ${{ contains(fromJSON(needs.vsix.outputs.pending), 'marketplace') }} @@ -512,7 +541,7 @@ jobs: publish-openvsx: name: publish to Open VSX runs-on: ubuntu-latest - needs: [vsix] + needs: [ci, vsix] if: ${{ contains(fromJSON(needs.vsix.outputs.pending), 'openvsx') }} permissions: contents: write @@ -548,7 +577,7 @@ jobs: publish-browser: name: publish to ${{ matrix.store }} runs-on: ubuntu-latest - needs: [browser] + needs: [ci, browser] # The matrix IS the gate: it is built from the probe's `pending` # array, so a store that already carries this version has no leg # at all, and adding a store means touching only the probe's From 18219d657b33d2d6351b349391b6b26f33c247cf Mon Sep 17 00:00:00 2001 From: Quentin Date: Tue, 6 Oct 2026 09:41:38 +0200 Subject: [PATCH 2/5] Carry the releasing skill's run-what-the-PR-runs rule and the parallel checks to the Kilo and agents copies --- .agents/skills/releasing/SKILL.md | 55 ++++++++++++++++++++++++------- .kilo/skills/releasing/SKILL.md | 55 ++++++++++++++++++++++++------- 2 files changed, 88 insertions(+), 22 deletions(-) diff --git a/.agents/skills/releasing/SKILL.md b/.agents/skills/releasing/SKILL.md index 956ddc1a..3b692bbe 100644 --- a/.agents/skills/releasing/SKILL.md +++ b/.agents/skills/releasing/SKILL.md @@ -38,17 +38,50 @@ into `main` through a PR, and the merge is what publishes. CLI binaries, the `.vsix` and the browser zips, updates the rolling `latest` release, and then publishes to each store. -CI runs on `pull_request` into `main` only. Six checks: `extension -versions bumped` (PRs only), `build`, `public proxy routes`, -`cargo test + clippy`, `cargo test --features db-tests`, and -`graph + editor`. - -## Run the checks CI runs, not the ones you touched - -`cargo clippy --workspace --all-targets --locked -- -D warnings` is -the CI line. Running clippy on the crates you edited is not the same -thing and has already sent a PR red on a lint in a crate the change -only touched indirectly. +CI runs on `pull_request` into `main` only. Every check is its own +job, so they run side by side. The main ruleset requires four names: +`extension versions bumped` (PRs only), `cargo test --features +db-tests`, and two gates that are green only when every job under them +is: `cargo test + clippy` (nextest, doc tests, clippy, the parse-only +build, the examples) and `graph + editor` (each JS package, the VS Code +extension, the WhatsApp bridge, the browser extension). The docs book +builds beside them. On `main`, `release.yml` runs the same suite as its +gate, and starts the builds (binaries, .vsix, browser zips) beside it; +only what publishes waits for the gate. + +## Before every push to a PR, run what the PR runs + +Before you push a commit to a branch with a PR (the first push that +opens it, and every fix pushed after), run locally everything the PR's +checks run, not only the tests your change reaches. This is the one +place the [test scope] does not apply: a red PR costs a full CI round +trip, and twice a PR has gone red on a test in a crate the change only +touched indirectly. Background the run, and fix everything before the +push, not after. Locally the cargo commands go one after another (they +share one build directory, which cargo locks, and overlapping cargo runs +have crashed this machine); the JS checks and the docs build run beside +them. + +The PR runs `.github/workflows/ci.yml` and `.github/workflows/docs.yml`; +read them for the current list. At the time of writing: + +- `scripts/check-extension-bump.sh origin/main` +- `cargo nextest run --workspace --locked` +- `cargo test --workspace --locked --doc` +- `cargo clippy --workspace --all-targets --locked -- -D warnings` +- `cargo check -p weft-compiler --no-default-features --locked` +- `scripts/validate-examples.sh` +- `scripts/run-db-tests.sh` (every crate, no filter) +- `pnpm -C packages/weft-graph test`, `pnpm -C packages/weft-connect test`, + `pnpm -C packages/weft-syntax test`, `pnpm -C extension-vscode test`, + `pnpm -C extension-vscode run compile` +- `node --test catalog/bailey/bridge/images/bridge/src/*.test.js` +- `pnpm -C extension-browser exec tsc --noEmit`, `pnpm -C extension-browser check` +- the docs build: `cd docs && mdbook-mermaid install . && mdbook build` + +Clippy on the crates you edited is not the same as the workspace line +above, and has sent a PR red on a lint in a crate the change only +touched indirectly. `scripts/validate-examples.sh` compiles every example. It fails on ERRORS only, so a warning is advice an example may carry. If an diff --git a/.kilo/skills/releasing/SKILL.md b/.kilo/skills/releasing/SKILL.md index 956ddc1a..3b692bbe 100644 --- a/.kilo/skills/releasing/SKILL.md +++ b/.kilo/skills/releasing/SKILL.md @@ -38,17 +38,50 @@ into `main` through a PR, and the merge is what publishes. CLI binaries, the `.vsix` and the browser zips, updates the rolling `latest` release, and then publishes to each store. -CI runs on `pull_request` into `main` only. Six checks: `extension -versions bumped` (PRs only), `build`, `public proxy routes`, -`cargo test + clippy`, `cargo test --features db-tests`, and -`graph + editor`. - -## Run the checks CI runs, not the ones you touched - -`cargo clippy --workspace --all-targets --locked -- -D warnings` is -the CI line. Running clippy on the crates you edited is not the same -thing and has already sent a PR red on a lint in a crate the change -only touched indirectly. +CI runs on `pull_request` into `main` only. Every check is its own +job, so they run side by side. The main ruleset requires four names: +`extension versions bumped` (PRs only), `cargo test --features +db-tests`, and two gates that are green only when every job under them +is: `cargo test + clippy` (nextest, doc tests, clippy, the parse-only +build, the examples) and `graph + editor` (each JS package, the VS Code +extension, the WhatsApp bridge, the browser extension). The docs book +builds beside them. On `main`, `release.yml` runs the same suite as its +gate, and starts the builds (binaries, .vsix, browser zips) beside it; +only what publishes waits for the gate. + +## Before every push to a PR, run what the PR runs + +Before you push a commit to a branch with a PR (the first push that +opens it, and every fix pushed after), run locally everything the PR's +checks run, not only the tests your change reaches. This is the one +place the [test scope] does not apply: a red PR costs a full CI round +trip, and twice a PR has gone red on a test in a crate the change only +touched indirectly. Background the run, and fix everything before the +push, not after. Locally the cargo commands go one after another (they +share one build directory, which cargo locks, and overlapping cargo runs +have crashed this machine); the JS checks and the docs build run beside +them. + +The PR runs `.github/workflows/ci.yml` and `.github/workflows/docs.yml`; +read them for the current list. At the time of writing: + +- `scripts/check-extension-bump.sh origin/main` +- `cargo nextest run --workspace --locked` +- `cargo test --workspace --locked --doc` +- `cargo clippy --workspace --all-targets --locked -- -D warnings` +- `cargo check -p weft-compiler --no-default-features --locked` +- `scripts/validate-examples.sh` +- `scripts/run-db-tests.sh` (every crate, no filter) +- `pnpm -C packages/weft-graph test`, `pnpm -C packages/weft-connect test`, + `pnpm -C packages/weft-syntax test`, `pnpm -C extension-vscode test`, + `pnpm -C extension-vscode run compile` +- `node --test catalog/bailey/bridge/images/bridge/src/*.test.js` +- `pnpm -C extension-browser exec tsc --noEmit`, `pnpm -C extension-browser check` +- the docs build: `cd docs && mdbook-mermaid install . && mdbook build` + +Clippy on the crates you edited is not the same as the workspace line +above, and has sent a PR red on a lint in a crate the change only +touched indirectly. `scripts/validate-examples.sh` compiles every example. It fails on ERRORS only, so a warning is advice an example may carry. If an From d580ec4953cf2fcce3e7f158411c48a7cc8f51f0 Mon Sep 17 00:00:00 2001 From: Quentin Date: Thu, 8 Oct 2026 14:31:29 +0200 Subject: [PATCH 3/5] Hold a wait in its worker when a run cannot pause, write runs as rows the worker owns, and review fixes A run that cannot pause (a caller still on the line, recorded off, or a bus open) holds ctx.await_signal in the node's call instead of failing: the hold clock runs only while every step waits, `holdSecs` (default 60, 0 fails at once, at most 30 days) gives the wait up as a catchable error, recorded so a replay fails the same, and the wait is withdrawn. Unrecorded runs hold their waits; Route's defaultHoldSecs is gone. Review fixes: the node's call records the answer it took and early answers wait for their step; the last word never overtakes a chunk and is queued under the lock it is taken under; a run let go of is claimed again once its last drive here left; a queued answer stays the wait's one answer; a wait needs its run's row; run endings page past failures; the search index pauses on one full broker; a run that cannot pause carries on through a worker's shutdown until the process goes. ExecPython documents what its shared processes keep. --- .github/ISSUE_TEMPLATE/provider_request.yml | 6 +- Cargo.lock | 400 ++- Cargo.toml | 19 +- README.md | 4 +- TODO.md | 183 ++ catalog/api/close/metadata.json | 1 + catalog/api/reply/metadata.json | 1 + catalog/api/route/metadata.json | 25 +- catalog/api/route/tests.rs | 2 +- catalog/api/socket/metadata.json | 17 - catalog/api/stream/metadata.json | 1 + catalog/api/testing.rs | 4 +- .../bridge/images/bridge/src/actions.js | 2 +- catalog/bailey/bridge/metadata.json | 7 +- catalog/basic/cast/metadata.json | 1 + catalog/basic/debug/metadata.json | 1 + catalog/basic/exec_python/build.rs | 11 - catalog/basic/exec_python/deps.toml | 48 +- catalog/basic/exec_python/metadata.json | 2 +- catalog/basic/exec_python/mod.rs | 504 ++-- catalog/basic/exec_python/tests.rs | 95 +- catalog/basic/format/metadata.json | 1 + catalog/basic/json_object/metadata.json | 1 + catalog/basic/list/metadata.json | 1 + catalog/basic/range/metadata.json | 5 +- catalog/basic/text/metadata.json | 5 +- catalog/basic/timers/wait/metadata.json | 2 +- catalog/basic/timers/wait_until/metadata.json | 2 +- catalog/human/query/metadata.json | 2 +- .../metadata.json | 2 +- .../stop_instance_infra/metadata.json | 2 +- .../terminate_instance_infra/metadata.json | 2 +- catalog/logic/all/metadata.json | 1 + catalog/logic/first_in_order/metadata.json | 1 + catalog/logic/not/metadata.json | 5 +- catalog/logic/switch/metadata.json | 5 +- .../database/images/credential/bootstrap.py | 58 +- catalog/postgres/database/metadata.json | 7 +- catalog/postgres/database/mod.rs | 11 + catalog/postgres/execute_query/mod.rs | 3 +- catalog/postgres/insert_row/mod.rs | 3 +- catalog/postgres/postgres.rs | 138 +- catalog/postgres/update_rows/mod.rs | 3 +- catalog/postgres/watch_query/mod.rs | 4 +- crates/weft-access-store/src/flows.rs | 100 +- crates/weft-access-store/src/lib.rs | 2 +- crates/weft-access-store/src/resolve.rs | 16 +- crates/weft-assets/Cargo.toml | 4 +- crates/weft-assets/src/lib.rs | 29 +- crates/weft-assets/tests/sync.rs | 39 + crates/weft-broker-client/src/activation.rs | 9 + crates/weft-broker-client/src/client.rs | 468 ++-- crates/weft-broker-client/src/lib.rs | 6 +- crates/weft-broker-client/src/line.rs | 68 +- crates/weft-broker-client/src/protocol.rs | 651 +++-- crates/weft-broker/src/auth.rs | 52 +- crates/weft-broker/src/caller_auth.rs | 42 +- crates/weft-broker/src/credential.rs | 3 +- crates/weft-broker/src/door.rs | 398 +++ crates/weft-broker/src/handlers.rs | 729 ++--- crates/weft-broker/src/held_signals.rs | 8 +- crates/weft-broker/src/lib.rs | 52 +- crates/weft-broker/src/lifecycle_writes.rs | 64 +- crates/weft-broker/src/line.rs | 17 +- crates/weft-broker/src/notices.rs | 90 + crates/weft-broker/src/program_tokens.rs | 2 +- crates/weft-broker/src/records.rs | 162 ++ crates/weft-broker/src/runtime_storage.rs | 10 +- crates/weft-broker/src/scope.rs | 36 +- crates/weft-broker/src/state.rs | 26 +- crates/weft-broker/tests/caller_auth.rs | 16 +- crates/weft-broker/tests/door_let_go.rs | 153 + crates/weft-broker/tests/held_signals.rs | 25 +- crates/weft-broker/tests/journal_writes.rs | 92 - crates/weft-broker/tests/lifecycle_writes.rs | 69 +- crates/weft-broker/tests/record_batch.rs | 277 ++ crates/weft-catalog/src/lib.rs | 18 +- crates/weft-cli/src/commands/activate.rs | 29 +- crates/weft-cli/src/commands/assets.rs | 7 + crates/weft-cli/src/commands/branch.rs | 17 +- crates/weft-cli/src/commands/connect.rs | 65 +- crates/weft-cli/src/commands/daemon.rs | 88 +- crates/weft-cli/src/commands/deactivate.rs | 14 +- crates/weft-cli/src/commands/domain.rs | 10 +- crates/weft-cli/src/commands/ensure.rs | 24 +- crates/weft-cli/src/commands/files.rs | 2 +- crates/weft-cli/src/commands/follow.rs | 38 +- crates/weft-cli/src/commands/freeze.rs | 9 +- crates/weft-cli/src/commands/frontend.rs | 5 +- crates/weft-cli/src/commands/infra.rs | 30 +- crates/weft-cli/src/commands/infra_card.rs | 11 + crates/weft-cli/src/commands/infra_env.rs | 11 +- .../weft-cli/src/commands/instance_values.rs | 15 +- crates/weft-cli/src/commands/mod.rs | 15 +- crates/weft-cli/src/commands/options.rs | 24 +- crates/weft-cli/src/commands/parse.rs | 2 +- crates/weft-cli/src/commands/rm.rs | 9 +- crates/weft-cli/src/commands/run.rs | 4 +- crates/weft-cli/src/commands/status.rs | 33 +- crates/weft-cli/src/commands/target.rs | 1 + crates/weft-cli/src/commands/tree.rs | 32 +- crates/weft-cli/src/commands/versions.rs | 79 +- crates/weft-cli/src/commands/workers.rs | 10 +- crates/weft-cli/src/main.rs | 136 +- crates/weft-compiler/src/build.rs | 5 +- crates/weft-compiler/src/codegen.rs | 106 +- crates/weft-compiler/src/enrich.rs | 1 + crates/weft-compiler/src/hash.rs | 5 +- crates/weft-compiler/src/lib.rs | 1 + crates/weft-compiler/src/project.rs | 78 + crates/weft-compiler/src/validate.rs | 20 + crates/weft-compiler/src/weft_compiler.rs | 7 +- crates/weft-compiler/tests/start_feeders.rs | 26 +- crates/weft-compiler/tests/test_crate_emit.rs | 3 +- crates/weft-compiler/tests/validate_tests.rs | 28 +- crates/weft-core/Cargo.toml | 8 +- crates/weft-core/src/access/events.rs | 13 + crates/weft-core/src/access/mod.rs | 6 + crates/weft-core/src/activation.rs | 12 + .../src/arrival.rs | 38 +- crates/weft-core/src/caller.rs | 234 +- crates/weft-core/src/caller_token.rs | 264 +- crates/weft-core/src/context.rs | 94 +- crates/weft-core/src/context/pure.rs | 292 ++ crates/weft-core/src/door_fire.rs | 209 ++ crates/weft-core/src/error.rs | 7 + crates/weft-core/src/exec/boundary.rs | 249 +- crates/weft-core/src/exec/cancel.rs | 14 + crates/weft-core/src/exec/loop_runtime.rs | 38 +- crates/weft-core/src/exec/mod.rs | 2 +- crates/weft-core/src/exec/postprocess.rs | 93 +- crates/weft-core/src/exec/ready.rs | 72 +- crates/weft-core/src/exec/run_kind.rs | 76 +- crates/weft-core/src/exec/stuck.rs | 12 +- crates/weft-core/src/infra/action.rs | 1 + crates/weft-core/src/infra/bake.rs | 216 ++ crates/weft-core/src/infra/mod.rs | 4 +- crates/weft-core/src/infra/resolve.rs | 12 + crates/weft-core/src/infra/run_gate.rs | 114 + crates/weft-core/src/lib.rs | 18 +- crates/weft-core/src/live_event.rs | 18 +- crates/weft-core/src/net.rs | 88 +- crates/weft-core/src/node.rs | 117 +- crates/weft-core/src/node_test.rs | 48 +- crates/weft-core/src/ports.rs | 6 + crates/weft-core/src/primitive.rs | 63 +- crates/weft-core/src/program.rs | 128 +- crates/weft-core/src/project.rs | 163 +- crates/weft-core/src/project/selection.rs | 176 +- crates/weft-core/src/projects.rs | 36 +- crates/weft-core/src/route.rs | 14 +- crates/weft-core/src/run_class.rs | 111 - crates/weft-core/src/run_settings.rs | 598 ++++ crates/weft-core/src/run_spec.rs | 197 +- crates/weft-core/src/seeding.rs | 18 +- crates/weft-core/src/shared.rs | 399 +++ crates/weft-core/src/signal/auth.rs | 11 +- crates/weft-core/src/signal/limits.rs | 113 +- .../weft-core/src/signal/listener_protocol.rs | 21 +- .../weft-core/src/signal/live_connection.rs | 204 +- crates/weft-core/src/signal/mod.rs | 19 +- crates/weft-core/src/signed_token.rs | 9 + crates/weft-core/src/versions.rs | 8 +- crates/weft-core/src/wait.rs | 199 -- crates/weft-core/src/weft_type.rs | 8 +- .../weft-dispatcher/src/activation_store.rs | 72 +- crates/weft-dispatcher/src/api/events.rs | 6 +- crates/weft-dispatcher/src/api/execution.rs | 263 +- crates/weft-dispatcher/src/api/infra.rs | 103 +- crates/weft-dispatcher/src/api/mod.rs | 95 +- crates/weft-dispatcher/src/api/node_tests.rs | 3 - crates/weft-dispatcher/src/api/project.rs | 1565 +++-------- crates/weft-dispatcher/src/api/signal.rs | 1617 ++--------- crates/weft-dispatcher/src/api/versions.rs | 323 +-- crates/weft-dispatcher/src/api/workers.rs | 11 +- crates/weft-dispatcher/src/app.rs | 74 +- crates/weft-dispatcher/src/authenticator.rs | 2 +- crates/weft-dispatcher/src/delivery.rs | 220 +- crates/weft-dispatcher/src/domains.rs | 37 +- crates/weft-dispatcher/src/door.rs | 28 +- crates/weft-dispatcher/src/drain.rs | 290 ++ crates/weft-dispatcher/src/entry_limits.rs | 536 +--- crates/weft-dispatcher/src/events.rs | 34 +- crates/weft-dispatcher/src/front.rs | 198 ++ crates/weft-dispatcher/src/held.rs | 6 +- crates/weft-dispatcher/src/infra_bake.rs | 55 + crates/weft-dispatcher/src/infra_door.rs | 2 +- crates/weft-dispatcher/src/infra_event.rs | 79 +- .../weft-dispatcher/src/infra_event_bridge.rs | 158 +- .../src/infra_lifecycle_command.rs | 78 +- crates/weft-dispatcher/src/infra_node.rs | 107 +- crates/weft-dispatcher/src/install_picks.rs | 4 +- crates/weft-dispatcher/src/instance_values.rs | 4 +- crates/weft-dispatcher/src/journal/fake.rs | 1014 +++---- crates/weft-dispatcher/src/journal/mod.rs | 522 ++-- .../weft-dispatcher/src/journal/postgres.rs | 2114 ++++++-------- crates/weft-dispatcher/src/journal_bridge.rs | 600 ---- crates/weft-dispatcher/src/lib.rs | 19 +- .../weft-dispatcher/src/lifecycle_claimer.rs | 93 +- crates/weft-dispatcher/src/live_relay.rs | 250 +- crates/weft-dispatcher/src/live_view.rs | 236 ++ crates/weft-dispatcher/src/parked_drain.rs | 243 ++ crates/weft-dispatcher/src/project_ports.rs | 144 + crates/weft-dispatcher/src/project_store.rs | 65 +- crates/weft-dispatcher/src/projection.rs | 71 +- crates/weft-dispatcher/src/proxy.rs | 133 +- crates/weft-dispatcher/src/reaper.rs | 275 +- crates/weft-dispatcher/src/retention.rs | 42 + crates/weft-dispatcher/src/run_ends.rs | 157 ++ crates/weft-dispatcher/src/run_search.rs | 102 - crates/weft-dispatcher/src/search_index.rs | 213 ++ crates/weft-dispatcher/src/settled.rs | 377 --- crates/weft-dispatcher/src/state.rs | 30 +- crates/weft-dispatcher/src/storage.rs | 55 +- crates/weft-dispatcher/src/take_down.rs | 104 +- .../weft-dispatcher/src/task_kinds/execute.rs | 174 -- .../src/task_kinds/fire_signal.rs | 85 +- crates/weft-dispatcher/src/task_kinds/mod.rs | 9 +- .../src/task_kinds/program_call.rs | 44 +- .../src/task_kinds/record_cost.rs | 63 - .../src/task_kinds/record_log.rs | 52 - .../src/task_kinds/register_signal.rs | 136 +- .../src/task_kinds/route_entry.rs | 550 ---- .../src/task_kinds/run_node_test.rs | 106 +- .../src/task_kinds/stop_tagged.rs | 6 +- .../src/task_kinds/withdraw_signal.rs | 37 + crates/weft-dispatcher/src/tenant.rs | 7 +- crates/weft-dispatcher/src/versions.rs | 442 ++- crates/weft-dispatcher/src/worker_door.rs | 87 + crates/weft-dispatcher/src/worker_fire.rs | 46 + .../weft-dispatcher/tests/db_entry_limits.rs | 285 +- .../weft-dispatcher/tests/db_install_state.rs | 6 +- crates/weft-dispatcher/tests/db_lifecycle.rs | 1521 ++++------ .../weft-dispatcher/tests/db_project_ports.rs | 60 + crates/weft-dispatcher/tests/db_tags.rs | 208 +- crates/weft-dispatcher/tests/db_versions.rs | 177 +- .../tests/member_keys_migration.rs | 174 +- .../weft-dispatcher/tests/tenant_isolation.rs | 9 +- .../weft-e2e/fixtures/api_reply/src/main.weft | 8 + .../openrouter_custom/nodes/ask/mod.rs | 3 - .../fixtures/port_types/src/main.weft | 4 +- .../fixtures/route_hold/src/main.weft | 27 + crates/weft-e2e/fixtures/route_hold/weft.toml | 4 + .../fixtures/step_crash/src/main.weft | 5 +- .../web_trigger/nodes/http_responder/mod.rs | 3 +- crates/weft-e2e/src/live.rs | 18 +- crates/weft-e2e/src/platform.rs | 38 +- crates/weft-e2e/src/run.rs | 8 +- crates/weft-e2e/tests/api_reply.rs | 91 +- crates/weft-e2e/tests/api_ticket.rs | 18 +- crates/weft-e2e/tests/api_unrecorded.rs | 4 +- crates/weft-e2e/tests/include_twice.rs | 8 + crates/weft-e2e/tests/infra_feed.rs | 10 +- crates/weft-e2e/tests/long_run.rs | 51 - crates/weft-e2e/tests/metering_edges.rs | 6 +- crates/weft-e2e/tests/port_types.rs | 7 +- crates/weft-e2e/tests/route_hold.rs | 58 + crates/weft-e2e/tests/step_crash.rs | 75 +- crates/weft-engine/Cargo.toml | 8 + crates/weft-engine/src/busy.rs | 221 ++ crates/weft-engine/src/caller_conn.rs | 2023 ++++++-------- crates/weft-engine/src/context.rs | 1823 +++++------- crates/weft-engine/src/door/broker.rs | 270 ++ crates/weft-engine/src/door/limits.rs | 292 ++ crates/weft-engine/src/door/mod.rs | 975 +++++++ crates/weft-engine/src/door/permits.rs | 143 + crates/weft-engine/src/execution_driver.rs | 2377 +++++++++------- .../src/execution_driver_tests/baked.rs | 209 ++ .../src/execution_driver_tests/born.rs | 416 +++ .../src/execution_driver_tests/branching.rs | 27 - .../src/execution_driver_tests/bus_comm.rs | 658 ++--- .../execution_driver_tests/catch_errors.rs | 113 +- .../src/execution_driver_tests/hand_back.rs | 286 ++ .../src/execution_driver_tests/loop_rig.rs | 61 +- .../execution_driver_tests/phase_routing.rs | 21 +- .../src/execution_driver_tests/resume.rs | 13 +- .../src/execution_driver_tests/rig.rs | 455 +-- .../src/execution_driver_tests/seeded.rs | 27 +- .../src/execution_driver_tests/stream.rs | 6 +- .../src/execution_driver_tests/unrecorded.rs | 172 +- crates/weft-engine/src/fired_caller.rs | 9 +- crates/weft-engine/src/held.rs | 55 +- crates/weft-engine/src/held_waits.rs | 374 +++ crates/weft-engine/src/journal_writer.rs | 1455 ++++++++++ crates/weft-engine/src/lib.rs | 18 +- crates/weft-engine/src/memory_guard.rs | 136 - crates/weft-engine/src/metering.rs | 410 ++- crates/weft-engine/src/plan.rs | 617 +++++ crates/weft-engine/src/profile.rs | 86 + crates/weft-engine/src/record_first.rs | 290 ++ crates/weft-engine/src/socket.rs | 129 +- crates/weft-engine/src/test_record.rs | 186 ++ crates/weft-engine/src/test_rig.rs | 145 +- crates/weft-engine/src/test_runner.rs | 33 +- crates/weft-engine/src/wait_tracker.rs | 9 + crates/weft-engine/src/worker.rs | 2455 +++++++++-------- .../weft-infra-supervisor/src/broker_ops.rs | 6 +- .../src/broker_ops/fake.rs | 76 +- crates/weft-infra-supervisor/src/lifecycle.rs | 64 +- .../tests/lifecycle_loop.rs | 188 +- .../tests/ownership_loop.rs | 4 +- crates/weft-journal/Cargo.toml | 2 + crates/weft-journal/src/birth.rs | 224 ++ crates/weft-journal/src/events.rs | 439 ++- crates/weft-journal/src/fold.rs | 289 +- crates/weft-journal/src/frame.rs | 149 + crates/weft-journal/src/lib.rs | 148 +- crates/weft-journal/src/record.rs | 923 +++++++ crates/weft-journal/src/search.rs | 107 + crates/weft-journal/src/seed.rs | 103 +- crates/weft-journal/src/stored.rs | 98 + crates/weft-journal/src/tags.rs | 124 +- crates/weft-journal/src/traits.rs | 295 +- crates/weft-journal/src/unrecorded.rs | 685 +---- crates/weft-journal/src/write.rs | 154 -- crates/weft-listener/src/event_context.rs | 12 +- crates/weft-listener/src/fire_sink.rs | 220 +- crates/weft-listener/src/kinds/form.rs | 2 +- .../src/kinds/live_connection.rs | 19 +- crates/weft-listener/src/kinds/mod.rs | 34 +- .../weft-listener/src/kinds/sse_subscribe.rs | 7 +- crates/weft-listener/src/kinds/timer.rs | 3 +- crates/weft-listener/src/lib.rs | 11 +- crates/weft-listener/src/registry.rs | 13 +- crates/weft-listener/tests/provider_events.rs | 99 +- crates/weft-listener/tests/stream_listen.rs | 86 +- crates/weft-listener/tests/wakes.rs | 185 +- crates/weft-platform-gcp/src/accounts.rs | 9 +- crates/weft-platform-gcp/src/domains.rs | 128 +- crates/weft-platform-gcp/src/identity.rs | 86 +- crates/weft-platform-gcp/src/lib.rs | 2 +- crates/weft-platform-gcp/src/metadata.rs | 19 + crates/weft-platform-gcp/src/names.rs | 38 +- crates/weft-platform-gcp/src/runner.rs | 1042 ++++--- crates/weft-platform-local/src/docker.rs | 49 +- crates/weft-platform-local/src/domains.rs | 4 +- crates/weft-platform-local/src/infra_host.rs | 138 +- crates/weft-platform-local/src/lib.rs | 2 +- crates/weft-platform-local/src/runner.rs | 683 +++-- crates/weft-platform-traits/src/config.rs | 39 +- crates/weft-platform-traits/src/domains.rs | 27 +- crates/weft-platform-traits/src/drain.rs | 134 - crates/weft-platform-traits/src/identity.rs | 6 + crates/weft-platform-traits/src/lib.rs | 6 +- crates/weft-platform-traits/src/runner.rs | 396 ++- crates/weft-platform-traits/src/unit_agent.rs | 19 + crates/weft-providers/Cargo.toml | 12 +- crates/weft-providers/src/lib.rs | 95 +- .../src/providers/elevenlabs.rs | 153 +- crates/weft-providers/src/providers/exa.rs | 68 - crates/weft-providers/src/providers/fal.rs | 274 +- .../weft-providers/src/providers/firecrawl.rs | 45 +- .../weft-providers/src/providers/mistral.rs | 60 +- .../src/providers/openrouter.rs | 270 +- .../tests/project_defined_meter.rs | 9 - .../tests/skill_example_meter.rs | 13 +- crates/weft-runtime/src/lib.rs | 41 +- crates/weft-runtime/src/platform.rs | 21 +- crates/weft-runtime/src/role_waker.rs | 3 +- crates/weft-runtime/src/server.rs | 12 + crates/weft-runtime/src/unit_agent.rs | 71 +- crates/weft-task-store/Cargo.toml | 2 - ...08T020205_dispatcher_cursor_run_record.sql | 2 + ...41910_entry_rate_fast_and_durable_runs.sql | 11 + ...ry_rate_unrecorded_seeds_and_lost_runs.sql | 51 + ...07T090220_entry_rate_long_runs_removed.sql | 28 + ...21_entry_rate_callers_reach_the_worker.sql | 1 + ...00_exec_event_journal_rows_hold_writes.sql | 64 + ...41910_exec_event_fast_and_durable_runs.sql | 208 ++ ...c_event_unrecorded_seeds_and_lost_runs.sql | 59 + ..._exec_event_parked_runs_hold_no_worker.sql | 12 + ...07T090220_exec_event_long_runs_removed.sql | 46 + ...0420_exec_event_execution_driven_index.sql | 1 + .../20261008T020205_exec_event_run_record.sql | 318 +++ ...105548_exec_event_waits_name_their_run.sql | 132 + ...244_exec_event_born_runs_hold_no_waits.sql | 126 + ...execution_search_fast_and_durable_runs.sql | 39 + ...008T020205_execution_search_run_record.sql | 30 + ...20261008T020205_infra_event_run_record.sql | 16 + ...205_infra_lifecycle_command_run_record.sql | 31 + ...41910_infra_node_fast_and_durable_runs.sql | 6 + ...05548_parked_fire_waits_name_their_run.sql | 12 + .../migrations/parked_fire/origin.sql | 57 + ...6T141910_project_fast_and_durable_runs.sql | 3 + ...25821_project_callers_reach_the_worker.sql | 1 + ...4433_project_asked_port_and_live_lease.sql | 3 + ...209_project_project_port_asked_removed.sql | 3 + ...10_storage_sweep_fast_and_durable_runs.sql | 10 + ...261008T020205_storage_sweep_run_record.sql | 3 + ...T130000_task_births_carry_run_settings.sql | 41 + ...1006T141910_task_fast_and_durable_runs.sql | 1 + .../task/20261008T020205_task_run_record.sql | 72 + ...8T020205_trigger_activation_run_record.sql | 26 + ..._versions_run_specs_carry_run_settings.sql | 6 + .../20261008T020205_versions_run_record.sql | 22 + ..._worker_door_asked_port_and_live_lease.sql | 27 + ...20261008T020205_worker_door_run_record.sql | 36 + .../migrations/worker_door/origin.sql | 42 + crates/weft-task-store/src/announce.rs | 10 +- crates/weft-task-store/src/db.rs | 26 +- crates/weft-task-store/src/drain.rs | 28 +- crates/weft-task-store/src/executor.rs | 144 +- crates/weft-task-store/src/held_copy.rs | 20 +- crates/weft-task-store/src/infra_copies.rs | 56 + crates/weft-task-store/src/journal_rows.rs | 23 - crates/weft-task-store/src/kinds.rs | 405 +-- crates/weft-task-store/src/lib.rs | 35 +- crates/weft-task-store/src/locks.rs | 6 +- crates/weft-task-store/src/parked_fires.rs | 393 +++ crates/weft-task-store/src/runs.rs | 76 + crates/weft-task-store/src/schema_guard.rs | 47 +- crates/weft-task-store/src/tasks.rs | 1054 +------ crates/weft-task-store/src/traits.rs | 86 +- crates/weft-task-store/src/worker_door.rs | 97 + crates/weft-task-store/tests/claims.rs | 72 + crates/weft-task-store/tests/rerun.rs | 80 - crates/weft-task-store/tests/signals.rs | 63 +- crates/weft-task-store/tests/support/mod.rs | 41 +- crates/weft-task-store/tests/terminal_wait.rs | 47 +- crates/weft-task-store/tests/worker_claims.rs | 450 --- deploy/terraform/gcp/secrets.tf | 7 - deploy/terraform/gcp/serverless.tf | 18 +- docs/src/appendix/glossary.md | 19 +- docs/src/appendix/roadmap.md | 2 +- docs/src/build/public-address.md | 2 +- docs/src/connections/meters.md | 3 +- docs/src/language/diagnostics.md | 1 + docs/src/language/how-a-program-runs.md | 11 +- docs/src/language/triggers-and-routes.md | 170 +- docs/src/nodes/ctx.md | 79 +- docs/src/nodes/durable-execution.md | 80 +- docs/src/nodes/infrastructure.md | 132 +- docs/src/nodes/metadata.md | 6 +- docs/src/nodes/testing.md | 1 + docs/src/nodes/values-and-emission.md | 4 +- docs/src/running/architecture.md | 356 ++- docs/src/running/cli.md | 37 +- docs/src/running/cloud.md | 33 +- docs/src/running/instances.md | 8 +- docs/src/running/lifecycles.md | 26 +- docs/src/running/the-journal.md | 159 +- docs/src/thinking/plumbing.md | 16 +- examples/README.md | 2 +- extension-browser/package.json | 2 +- extension-vscode/package.json | 2 +- extension-vscode/src/sidebar/executions.ts | 16 +- extension-vscode/src/sidebar/outcome.ts | 2 - .../src/sidebar/version-tree.test.ts | 7 + extension-vscode/src/sidebar/version-tree.ts | 10 +- packages/weft-connect/src/core/socket.test.ts | 4 +- packages/weft-connect/src/core/socket.ts | 2 +- packages/weft-graph/src/protocol.ts | 5 +- packages/weft-graph/src/run-spec.test.ts | 28 + packages/weft-graph/src/run-spec.ts | 57 +- scripts/lib/weft-cleanup.sh | 45 + setup.sh | 73 +- .../claude-code/.claude/agents/node-smith.md | 6 +- .../.claude/commands/weft-debug.md | 2 +- tangle/claude-code/.claude/settings.json | 1 + .../.claude/skills/weft-api/SKILL.md | 44 +- .../.claude/skills/weft-catalog/SKILL.md | 5 +- .../.claude/skills/weft-debugging/SKILL.md | 19 +- .../.claude/skills/weft-deploying/SKILL.md | 45 +- .../.claude/skills/weft-gaps/SKILL.md | 2 +- .../.claude/skills/weft-language/SKILL.md | 36 +- .../.claude/skills/weft-metering/SKILL.md | 4 +- .../skills/weft-node-authoring/SKILL.md | 37 +- .../.claude/skills/weft-onboarding/SKILL.md | 2 +- .../.claude/skills/weft-running/SKILL.md | 11 +- .../.claude/skills/weft-sdp/SKILL.md | 17 +- tangle/claude-code/CLAUDE.md | 6 +- .../cline/.cline/skills/node-smith/SKILL.md | 6 +- tangle/cline/.cline/skills/weft-api/SKILL.md | 44 +- .../cline/.cline/skills/weft-catalog/SKILL.md | 5 +- .../cline/.cline/skills/weft-debug/SKILL.md | 2 +- .../.cline/skills/weft-debugging/SKILL.md | 19 +- .../.cline/skills/weft-deploying/SKILL.md | 45 +- tangle/cline/.cline/skills/weft-gaps/SKILL.md | 2 +- .../.cline/skills/weft-language/SKILL.md | 36 +- .../.cline/skills/weft-metering/SKILL.md | 4 +- .../skills/weft-node-authoring/SKILL.md | 37 +- .../.cline/skills/weft-onboarding/SKILL.md | 2 +- .../cline/.cline/skills/weft-running/SKILL.md | 11 +- tangle/cline/.cline/skills/weft-sdp/SKILL.md | 17 +- tangle/cline/.clinerules/tangle.md | 6 +- tangle/codex/.agents/skills/weft-api/SKILL.md | 44 +- .../.agents/skills/weft-catalog/SKILL.md | 5 +- .../codex/.agents/skills/weft-debug/SKILL.md | 2 +- .../.agents/skills/weft-debugging/SKILL.md | 19 +- .../.agents/skills/weft-deploying/SKILL.md | 45 +- .../codex/.agents/skills/weft-gaps/SKILL.md | 2 +- .../.agents/skills/weft-language/SKILL.md | 36 +- .../.agents/skills/weft-metering/SKILL.md | 4 +- .../skills/weft-node-authoring/SKILL.md | 37 +- .../.agents/skills/weft-onboarding/SKILL.md | 2 +- .../.agents/skills/weft-running/SKILL.md | 11 +- tangle/codex/.agents/skills/weft-sdp/SKILL.md | 17 +- tangle/codex/.codex/agents/node-smith.toml | 6 +- tangle/codex/AGENTS.md | 6 +- tangle/cursor/.cursor/agents/node-smith.md | 6 +- tangle/cursor/.cursor/rules/tangle.mdc | 6 +- .../cursor/.cursor/skills/weft-api/SKILL.md | 44 +- .../.cursor/skills/weft-catalog/SKILL.md | 5 +- .../cursor/.cursor/skills/weft-debug/SKILL.md | 2 +- .../.cursor/skills/weft-debugging/SKILL.md | 19 +- .../.cursor/skills/weft-deploying/SKILL.md | 45 +- .../cursor/.cursor/skills/weft-gaps/SKILL.md | 2 +- .../.cursor/skills/weft-language/SKILL.md | 36 +- .../.cursor/skills/weft-metering/SKILL.md | 4 +- .../skills/weft-node-authoring/SKILL.md | 37 +- .../.cursor/skills/weft-onboarding/SKILL.md | 2 +- .../.cursor/skills/weft-running/SKILL.md | 11 +- .../cursor/.cursor/skills/weft-sdp/SKILL.md | 17 +- tangle/devin-desktop/.devin/rules/tangle-1.md | 2 +- tangle/devin-desktop/.devin/rules/tangle-4.md | 4 +- .../.windsurf/skills/node-smith/SKILL.md | 6 +- .../.windsurf/skills/weft-api/SKILL.md | 44 +- .../.windsurf/skills/weft-catalog/SKILL.md | 5 +- .../.windsurf/skills/weft-debugging/SKILL.md | 19 +- .../.windsurf/skills/weft-deploying/SKILL.md | 45 +- .../.windsurf/skills/weft-gaps/SKILL.md | 2 +- .../.windsurf/skills/weft-language/SKILL.md | 36 +- .../.windsurf/skills/weft-metering/SKILL.md | 4 +- .../skills/weft-node-authoring/SKILL.md | 37 +- .../.windsurf/skills/weft-onboarding/SKILL.md | 2 +- .../.windsurf/skills/weft-running/SKILL.md | 11 +- .../.windsurf/skills/weft-sdp/SKILL.md | 17 +- .../.windsurf/workflows/weft-debug.md | 2 +- .../.agents/skills/node-smith/SKILL.md | 6 +- .../fallback/.agents/skills/weft-api/SKILL.md | 44 +- .../.agents/skills/weft-catalog/SKILL.md | 5 +- .../.agents/skills/weft-debug/SKILL.md | 2 +- .../.agents/skills/weft-debugging/SKILL.md | 19 +- .../.agents/skills/weft-deploying/SKILL.md | 45 +- .../.agents/skills/weft-gaps/SKILL.md | 2 +- .../.agents/skills/weft-language/SKILL.md | 36 +- .../.agents/skills/weft-metering/SKILL.md | 4 +- .../skills/weft-node-authoring/SKILL.md | 37 +- .../.agents/skills/weft-onboarding/SKILL.md | 2 +- .../.agents/skills/weft-running/SKILL.md | 11 +- .../fallback/.agents/skills/weft-sdp/SKILL.md | 17 +- tangle/fallback/AGENTS.md | 6 +- .../gemini-cli/.gemini/agents/node-smith.md | 6 +- .../.gemini/commands/weft-debug.toml | 2 +- .../.gemini/skills/weft-api/SKILL.md | 44 +- .../.gemini/skills/weft-catalog/SKILL.md | 5 +- .../.gemini/skills/weft-debugging/SKILL.md | 19 +- .../.gemini/skills/weft-deploying/SKILL.md | 45 +- .../.gemini/skills/weft-gaps/SKILL.md | 2 +- .../.gemini/skills/weft-language/SKILL.md | 36 +- .../.gemini/skills/weft-metering/SKILL.md | 4 +- .../skills/weft-node-authoring/SKILL.md | 37 +- .../.gemini/skills/weft-onboarding/SKILL.md | 2 +- .../.gemini/skills/weft-running/SKILL.md | 11 +- .../.gemini/skills/weft-sdp/SKILL.md | 17 +- tangle/gemini-cli/GEMINI.md | 6 +- .../.github/agents/node-smith.agent.md | 6 +- .../.github/copilot-instructions.md | 6 +- .../.github/prompts/weft-debug.prompt.md | 2 +- .../.github/skills/weft-api/SKILL.md | 44 +- .../.github/skills/weft-catalog/SKILL.md | 5 +- .../.github/skills/weft-debugging/SKILL.md | 19 +- .../.github/skills/weft-deploying/SKILL.md | 45 +- .../.github/skills/weft-gaps/SKILL.md | 2 +- .../.github/skills/weft-language/SKILL.md | 36 +- .../.github/skills/weft-metering/SKILL.md | 4 +- .../skills/weft-node-authoring/SKILL.md | 37 +- .../.github/skills/weft-onboarding/SKILL.md | 2 +- .../.github/skills/weft-running/SKILL.md | 11 +- .../.github/skills/weft-sdp/SKILL.md | 17 +- tangle/junie/.junie/AGENTS.md | 6 +- tangle/junie/.junie/agents/node-smith.md | 6 +- tangle/junie/.junie/commands/weft-debug.md | 2 +- tangle/junie/.junie/skills/weft-api/SKILL.md | 44 +- .../junie/.junie/skills/weft-catalog/SKILL.md | 5 +- .../.junie/skills/weft-debugging/SKILL.md | 19 +- .../.junie/skills/weft-deploying/SKILL.md | 45 +- tangle/junie/.junie/skills/weft-gaps/SKILL.md | 2 +- .../.junie/skills/weft-language/SKILL.md | 36 +- .../.junie/skills/weft-metering/SKILL.md | 4 +- .../skills/weft-node-authoring/SKILL.md | 37 +- .../.junie/skills/weft-onboarding/SKILL.md | 2 +- .../junie/.junie/skills/weft-running/SKILL.md | 11 +- tangle/junie/.junie/skills/weft-sdp/SKILL.md | 17 +- tangle/kilo-code/.kilo/agent/node-smith.md | 6 +- tangle/kilo-code/.kilo/agent/tangle.md | 6 +- tangle/kilo-code/.kilo/command/weft-debug.md | 2 +- .../kilo-code/.kilo/skills/weft-api/SKILL.md | 44 +- .../.kilo/skills/weft-catalog/SKILL.md | 5 +- .../.kilo/skills/weft-debugging/SKILL.md | 19 +- .../.kilo/skills/weft-deploying/SKILL.md | 45 +- .../kilo-code/.kilo/skills/weft-gaps/SKILL.md | 2 +- .../.kilo/skills/weft-language/SKILL.md | 36 +- .../.kilo/skills/weft-metering/SKILL.md | 4 +- .../.kilo/skills/weft-node-authoring/SKILL.md | 37 +- .../.kilo/skills/weft-onboarding/SKILL.md | 2 +- .../.kilo/skills/weft-running/SKILL.md | 11 +- .../kilo-code/.kilo/skills/weft-sdp/SKILL.md | 17 +- tangle/kilo-code/kilo.json | 1 + .../opencode/.opencode/agents/node-smith.md | 6 +- tangle/opencode/.opencode/agents/tangle.md | 6 +- .../opencode/.opencode/commands/weft-debug.md | 2 +- .../.opencode/skills/weft-api/SKILL.md | 44 +- .../.opencode/skills/weft-catalog/SKILL.md | 5 +- .../.opencode/skills/weft-debugging/SKILL.md | 19 +- .../.opencode/skills/weft-deploying/SKILL.md | 45 +- .../.opencode/skills/weft-gaps/SKILL.md | 2 +- .../.opencode/skills/weft-language/SKILL.md | 36 +- .../.opencode/skills/weft-metering/SKILL.md | 4 +- .../skills/weft-node-authoring/SKILL.md | 37 +- .../.opencode/skills/weft-onboarding/SKILL.md | 2 +- .../.opencode/skills/weft-running/SKILL.md | 11 +- .../.opencode/skills/weft-sdp/SKILL.md | 17 +- tangle/opencode/AGENTS.md | 4 +- tangle/opencode/opencode.json | 1 + 615 files changed, 35685 insertions(+), 27656 deletions(-) delete mode 100644 catalog/basic/exec_python/build.rs create mode 100644 crates/weft-broker/src/door.rs create mode 100644 crates/weft-broker/src/notices.rs create mode 100644 crates/weft-broker/src/records.rs create mode 100644 crates/weft-broker/tests/door_let_go.rs delete mode 100644 crates/weft-broker/tests/journal_writes.rs create mode 100644 crates/weft-broker/tests/record_batch.rs rename crates/{weft-dispatcher => weft-core}/src/arrival.rs (80%) create mode 100644 crates/weft-core/src/context/pure.rs create mode 100644 crates/weft-core/src/door_fire.rs create mode 100644 crates/weft-core/src/infra/bake.rs create mode 100644 crates/weft-core/src/infra/run_gate.rs delete mode 100644 crates/weft-core/src/run_class.rs create mode 100644 crates/weft-core/src/run_settings.rs create mode 100644 crates/weft-core/src/shared.rs delete mode 100644 crates/weft-core/src/wait.rs create mode 100644 crates/weft-dispatcher/src/drain.rs create mode 100644 crates/weft-dispatcher/src/front.rs create mode 100644 crates/weft-dispatcher/src/infra_bake.rs delete mode 100644 crates/weft-dispatcher/src/journal_bridge.rs create mode 100644 crates/weft-dispatcher/src/live_view.rs create mode 100644 crates/weft-dispatcher/src/parked_drain.rs create mode 100644 crates/weft-dispatcher/src/project_ports.rs create mode 100644 crates/weft-dispatcher/src/retention.rs create mode 100644 crates/weft-dispatcher/src/run_ends.rs delete mode 100644 crates/weft-dispatcher/src/run_search.rs create mode 100644 crates/weft-dispatcher/src/search_index.rs delete mode 100644 crates/weft-dispatcher/src/settled.rs delete mode 100644 crates/weft-dispatcher/src/task_kinds/execute.rs delete mode 100644 crates/weft-dispatcher/src/task_kinds/record_cost.rs delete mode 100644 crates/weft-dispatcher/src/task_kinds/record_log.rs delete mode 100644 crates/weft-dispatcher/src/task_kinds/route_entry.rs create mode 100644 crates/weft-dispatcher/src/task_kinds/withdraw_signal.rs create mode 100644 crates/weft-dispatcher/src/worker_door.rs create mode 100644 crates/weft-dispatcher/src/worker_fire.rs create mode 100644 crates/weft-dispatcher/tests/db_project_ports.rs create mode 100644 crates/weft-e2e/fixtures/route_hold/src/main.weft create mode 100644 crates/weft-e2e/fixtures/route_hold/weft.toml delete mode 100644 crates/weft-e2e/tests/long_run.rs create mode 100644 crates/weft-e2e/tests/route_hold.rs create mode 100644 crates/weft-engine/src/busy.rs create mode 100644 crates/weft-engine/src/door/broker.rs create mode 100644 crates/weft-engine/src/door/limits.rs create mode 100644 crates/weft-engine/src/door/mod.rs create mode 100644 crates/weft-engine/src/door/permits.rs create mode 100644 crates/weft-engine/src/execution_driver_tests/baked.rs create mode 100644 crates/weft-engine/src/execution_driver_tests/born.rs create mode 100644 crates/weft-engine/src/execution_driver_tests/hand_back.rs create mode 100644 crates/weft-engine/src/held_waits.rs create mode 100644 crates/weft-engine/src/journal_writer.rs delete mode 100644 crates/weft-engine/src/memory_guard.rs create mode 100644 crates/weft-engine/src/plan.rs create mode 100644 crates/weft-engine/src/profile.rs create mode 100644 crates/weft-engine/src/record_first.rs create mode 100644 crates/weft-engine/src/test_record.rs create mode 100644 crates/weft-journal/src/birth.rs create mode 100644 crates/weft-journal/src/frame.rs create mode 100644 crates/weft-journal/src/record.rs create mode 100644 crates/weft-journal/src/search.rs create mode 100644 crates/weft-journal/src/stored.rs delete mode 100644 crates/weft-journal/src/write.rs delete mode 100644 crates/weft-platform-traits/src/drain.rs create mode 100644 crates/weft-task-store/migrations/dispatcher_cursor/20261008T020205_dispatcher_cursor_run_record.sql create mode 100644 crates/weft-task-store/migrations/entry_rate/20261006T141910_entry_rate_fast_and_durable_runs.sql create mode 100644 crates/weft-task-store/migrations/entry_rate/20261006T152323_entry_rate_unrecorded_seeds_and_lost_runs.sql create mode 100644 crates/weft-task-store/migrations/entry_rate/20261007T090220_entry_rate_long_runs_removed.sql create mode 100644 crates/weft-task-store/migrations/entry_rate/20261007T125821_entry_rate_callers_reach_the_worker.sql create mode 100644 crates/weft-task-store/migrations/exec_event/20261006T130000_exec_event_journal_rows_hold_writes.sql create mode 100644 crates/weft-task-store/migrations/exec_event/20261006T141910_exec_event_fast_and_durable_runs.sql create mode 100644 crates/weft-task-store/migrations/exec_event/20261006T152323_exec_event_unrecorded_seeds_and_lost_runs.sql create mode 100644 crates/weft-task-store/migrations/exec_event/20261006T160000_exec_event_parked_runs_hold_no_worker.sql create mode 100644 crates/weft-task-store/migrations/exec_event/20261007T090220_exec_event_long_runs_removed.sql create mode 100644 crates/weft-task-store/migrations/exec_event/20261007T150420_exec_event_execution_driven_index.sql create mode 100644 crates/weft-task-store/migrations/exec_event/20261008T020205_exec_event_run_record.sql create mode 100644 crates/weft-task-store/migrations/exec_event/20261008T105548_exec_event_waits_name_their_run.sql create mode 100644 crates/weft-task-store/migrations/exec_event/20261008T112244_exec_event_born_runs_hold_no_waits.sql create mode 100644 crates/weft-task-store/migrations/execution_search/20261006T141910_execution_search_fast_and_durable_runs.sql create mode 100644 crates/weft-task-store/migrations/execution_search/20261008T020205_execution_search_run_record.sql create mode 100644 crates/weft-task-store/migrations/infra_event/20261008T020205_infra_event_run_record.sql create mode 100644 crates/weft-task-store/migrations/infra_lifecycle_command/20261008T020205_infra_lifecycle_command_run_record.sql create mode 100644 crates/weft-task-store/migrations/infra_node/20261006T141910_infra_node_fast_and_durable_runs.sql create mode 100644 crates/weft-task-store/migrations/parked_fire/20261008T105548_parked_fire_waits_name_their_run.sql create mode 100644 crates/weft-task-store/migrations/parked_fire/origin.sql create mode 100644 crates/weft-task-store/migrations/project/20261006T141910_project_fast_and_durable_runs.sql create mode 100644 crates/weft-task-store/migrations/project/20261007T125821_project_callers_reach_the_worker.sql create mode 100644 crates/weft-task-store/migrations/project/20261007T154433_project_asked_port_and_live_lease.sql create mode 100644 crates/weft-task-store/migrations/project/20261008T074209_project_project_port_asked_removed.sql create mode 100644 crates/weft-task-store/migrations/storage_sweep/20261006T141910_storage_sweep_fast_and_durable_runs.sql create mode 100644 crates/weft-task-store/migrations/storage_sweep/20261008T020205_storage_sweep_run_record.sql create mode 100644 crates/weft-task-store/migrations/task/20261006T130000_task_births_carry_run_settings.sql create mode 100644 crates/weft-task-store/migrations/task/20261006T141910_task_fast_and_durable_runs.sql create mode 100644 crates/weft-task-store/migrations/task/20261008T020205_task_run_record.sql create mode 100644 crates/weft-task-store/migrations/trigger_activation/20261008T020205_trigger_activation_run_record.sql create mode 100644 crates/weft-task-store/migrations/versions/20261006T130000_versions_run_specs_carry_run_settings.sql create mode 100644 crates/weft-task-store/migrations/versions/20261008T020205_versions_run_record.sql create mode 100644 crates/weft-task-store/migrations/worker_door/20261007T154433_worker_door_asked_port_and_live_lease.sql create mode 100644 crates/weft-task-store/migrations/worker_door/20261008T020205_worker_door_run_record.sql create mode 100644 crates/weft-task-store/migrations/worker_door/origin.sql create mode 100644 crates/weft-task-store/src/infra_copies.rs delete mode 100644 crates/weft-task-store/src/journal_rows.rs create mode 100644 crates/weft-task-store/src/parked_fires.rs create mode 100644 crates/weft-task-store/src/runs.rs create mode 100644 crates/weft-task-store/src/worker_door.rs create mode 100644 crates/weft-task-store/tests/claims.rs delete mode 100644 crates/weft-task-store/tests/rerun.rs delete mode 100644 crates/weft-task-store/tests/worker_claims.rs diff --git a/.github/ISSUE_TEMPLATE/provider_request.yml b/.github/ISSUE_TEMPLATE/provider_request.yml index 6061fae9..26b1a883 100644 --- a/.github/ISSUE_TEMPLATE/provider_request.yml +++ b/.github/ISSUE_TEMPLATE/provider_request.yml @@ -1,13 +1,13 @@ name: Provider request -description: Ask for a paid provider to be usable on the shared keys in app.weavemind.ai. +description: Ask for a paid provider's meter to ship with weft, so its calls are measured. labels: ["provider-request", "triage"] body: - type: markdown attributes: value: | - Ask here for a provider whose calls people can make without bringing their own key, billed to app.weavemind.ai instead. + Ask here for a provider whose meter should ship with weft: the code that records what each of its calls cost. Without one, its calls go through unmeasured, and an install cannot offer its own key for it, because that key only travels on routes a meter knows. - You are not blocked waiting on us. A provider defined inside your own project works today with your own key: write its meter as a shared `.rs` file in your package ([how meters work](https://weavemindai.github.io/weft/connections/meters.html)) and set the key on the node. This request is only about the shared keys, which need a reviewed meter shipped in weft itself. + You are not blocked waiting on us. A provider defined inside your own project works today with your own key: write its meter as a shared `.rs` file in your package ([how meters work](https://weavemindai.github.io/weft/connections/meters.html)) and set the key on the node. This request is about a meter shipped in weft itself, which every install gets. - type: input id: name diff --git a/Cargo.lock b/Cargo.lock index 3a87c2cb..95c4e868 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "addr2line" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" +dependencies = [ + "gimli", +] + [[package]] name = "adler2" version = "2.0.1" @@ -43,6 +52,19 @@ dependencies = [ "subtle", ] +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -52,6 +74,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "aligned-vec" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b" +dependencies = [ + "equator", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -132,6 +163,12 @@ dependencies = [ "rustversion", ] +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + [[package]] name = "async-trait" version = "0.1.89" @@ -620,6 +657,21 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "backtrace" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide 0.8.9", + "object", + "rustc-demangle", + "windows-link", +] + [[package]] name = "base16ct" version = "0.2.0" @@ -654,6 +706,12 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.11.1" @@ -687,6 +745,12 @@ version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + [[package]] name = "byteorder" version = "1.5.0" @@ -716,6 +780,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex 1.3.0", ] @@ -893,6 +959,15 @@ version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7704b5fdd17b18ae31c4c1da5a2e0305a2bf17b5249300a9ee9ed7b72114c636" +[[package]] +name = "cpp_demangle" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2bb79cb74d735044c972aae58ed0aaa9a837e85b01106a54c39e42e97f62253" +dependencies = [ + "cfg-if", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -1068,6 +1143,15 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +[[package]] +name = "debugid" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" +dependencies = [ + "uuid", +] + [[package]] name = "der" version = "0.7.10" @@ -1203,6 +1287,26 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "equator" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" +dependencies = [ + "equator-macro", +] + +[[package]] +name = "equator-macro" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1282,7 +1386,7 @@ checksum = "369995dae0733f1fe5ab0e3f345f6503a5f384179df5d8da333702031a131cf9" dependencies = [ "chrono", "mediatype", - "quick-xml", + "quick-xml 0.41.0", "regex", "serde", "serde_json", @@ -1323,6 +1427,18 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "findshlibs" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40b9e59cd0f7e0806cca4be089683ecb6434e602038df21fe6bf6711b2f07f64" +dependencies = [ + "cc", + "lazy_static", + "libc", + "winapi", +] + [[package]] name = "flate2" version = "1.1.10" @@ -1330,7 +1446,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" dependencies = [ "crc32fast", - "miniz_oxide", + "miniz_oxide 0.9.1", "zlib-rs", ] @@ -1533,6 +1649,12 @@ dependencies = [ "polyval", ] +[[package]] +name = "gimli" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" + [[package]] name = "group" version = "0.13.0" @@ -1631,6 +1753,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" + [[package]] name = "hex" version = "0.4.3" @@ -2008,6 +2136,24 @@ dependencies = [ "serde_core", ] +[[package]] +name = "inferno" +version = "0.11.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" +dependencies = [ + "ahash", + "indexmap", + "is-terminal", + "itoa", + "log", + "num-format", + "once_cell", + "quick-xml 0.26.0", + "rgb", + "str_stack", +] + [[package]] name = "inout" version = "0.1.4" @@ -2051,6 +2197,17 @@ dependencies = [ "once_cell", ] +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "is-wsl" version = "0.4.0" @@ -2082,6 +2239,16 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.2", + "libc", +] + [[package]] name = "js-sys" version = "0.3.97" @@ -2151,7 +2318,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" dependencies = [ - "bitflags", + "bitflags 2.11.1", "libc", "plain", "redox_syscall 0.7.4", @@ -2259,6 +2426,15 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + [[package]] name = "mime" version = "0.3.17" @@ -2275,7 +2451,6 @@ dependencies = [ "dotenvy", "eventsource-stream", "futures", - "regex", "reqwest", "reqwest-middleware", "secrecy", @@ -2295,6 +2470,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + [[package]] name = "miniz_oxide" version = "0.9.1" @@ -2316,6 +2500,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "nix" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -2367,6 +2562,16 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +[[package]] +name = "num-format" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" +dependencies = [ + "arrayvec", + "itoa", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -2397,6 +2602,15 @@ dependencies = [ "libm", ] +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -2631,6 +2845,28 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "pprof" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38a01da47675efa7673b032bf8efd8214f1917d89685e07e395ab125ea42b187" +dependencies = [ + "aligned-vec", + "backtrace", + "cfg-if", + "findshlibs", + "inferno", + "libc", + "log", + "nix", + "once_cell", + "smallvec", + "spin 0.10.0", + "symbolic-demangle", + "tempfile", + "thiserror 2.0.18", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -2677,6 +2913,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quick-xml" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" +dependencies = [ + "memchr", +] + [[package]] name = "quick-xml" version = "0.41.0" @@ -2855,7 +3100,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags", + "bitflags 2.11.1", ] [[package]] @@ -2864,7 +3109,7 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a" dependencies = [ - "bitflags", + "bitflags 2.11.1", ] [[package]] @@ -2971,6 +3216,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "rgb" +version = "0.8.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4" +dependencies = [ + "bytemuck", +] + [[package]] name = "ring" version = "0.17.14" @@ -3038,6 +3292,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustc-demangle" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" + [[package]] name = "rustc-hash" version = "1.1.0" @@ -3065,7 +3325,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys", @@ -3229,7 +3489,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags", + "bitflags 2.11.1", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -3242,7 +3502,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags", + "bitflags 2.11.1", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -3511,6 +3771,9 @@ name = "spin" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" +dependencies = [ + "lock_api", +] [[package]] name = "spki" @@ -3619,7 +3882,7 @@ checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" dependencies = [ "atoi", "base64 0.22.1", - "bitflags", + "bitflags 2.11.1", "byteorder", "bytes", "chrono", @@ -3663,7 +3926,7 @@ checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" dependencies = [ "atoi", "base64 0.22.1", - "bitflags", + "bitflags 2.11.1", "byteorder", "chrono", "crc", @@ -3726,6 +3989,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "str_stack" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f446288b699d66d0fd2e30d1cfe7869194312524b3b9252594868ed26ef056a" + [[package]] name = "stringprep" version = "0.1.5" @@ -3749,6 +4018,29 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "symbolic-common" +version = "12.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332615d90111d8eeaf86a84dc9bbe9f65d0d8c5cf11b4caccedc37754eb0dcfd" +dependencies = [ + "debugid", + "memmap2", + "stable_deref_trait", + "uuid", +] + +[[package]] +name = "symbolic-demangle" +version = "12.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "912017718eb4d21930546245af9a3475c9dccf15675a5c215664e76621afc471" +dependencies = [ + "cpp_demangle", + "rustc-demangle", + "symbolic-common", +] + [[package]] name = "syn" version = "2.0.117" @@ -3880,6 +4172,26 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tikv-jemalloc-sys" +version = "0.7.1+5.3.1-0-g81034ce1f1373e37dc865038e1bc8eeecf559ce8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2825c78386b4ae0314074867860ba9577875de945f05992c38815cbec327f0" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "249f09e49ab1609436f34c776e84231bead18d6a955f119f939bdc1d847561bd" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + [[package]] name = "time" version = "0.3.51" @@ -4139,7 +4451,7 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ - "bitflags", + "bitflags 2.11.1", "bytes", "futures-util", "http 1.4.0", @@ -4381,6 +4693,7 @@ checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ "getrandom 0.4.2", "js-sys", + "rand 0.10.3", "serde_core", "sha1_smol", "wasm-bindgen", @@ -4545,7 +4858,7 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags", + "bitflags 2.11.1", "hashbrown 0.15.5", "indexmap", "semver", @@ -4624,6 +4937,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "tokio", + "tracing", "weft-core", ] @@ -4879,15 +5193,20 @@ dependencies = [ "hex", "http 1.4.0", "parking_lot", + "percent-encoding", + "pprof", "reqwest", "reqwest-middleware", "serde", "serde_json", "socket2 0.6.3", + "tempfile", "thiserror 2.0.18", + "tikv-jemallocator", "tokio", "tokio-stream", "tokio-tungstenite 0.24.0", + "tokio-util", "tower", "tracing", "tracing-subscriber", @@ -4929,6 +5248,7 @@ version = "0.2.0" dependencies = [ "anyhow", "async-trait", + "base64 0.22.1", "chrono", "serde", "serde_json", @@ -4939,6 +5259,7 @@ dependencies = [ "uuid", "weft-core", "weft-task-store", + "zstd", ] [[package]] @@ -5069,7 +5390,6 @@ dependencies = [ "base64 0.22.1", "http 1.4.0", "inventory", - "minillmlib", "reqwest", "reqwest-middleware", "serde", @@ -5141,6 +5461,28 @@ dependencies = [ "wasite", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.62.2" @@ -5504,7 +5846,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags", + "bitflags 2.11.1", "indexmap", "log", "serde", @@ -5684,3 +6026,31 @@ name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.1.0+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 7274e864..89c4c620 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,11 @@ members = [ doc_lazy_continuation = "allow" doc_overindented_list_items = "allow" too_many_arguments = "allow" +# An HTTP handler's refusal is the response itself (axum's `Response`, 128 +# bytes), returned as the `Err` of the helpers that build it: boxing every +# refusal to please the size lint buys nothing on a path that answers a +# request. +result_large_err = "allow" # Dev/test binaries carry line tables only, not full variable # debuginfo. This workspace's test binaries are enormous with full @@ -113,6 +118,9 @@ toml_edit = "0.22" # (weft-platform-gcp `images`). tar = "0.4" flate2 = "1" +# A run's record, compressed by whoever writes it (`weft_journal::stored`): +# a ping's events are about 2 KB of JSON, which Postgres leaves as it is. +zstd = "0.13" # lossless concrete syntax tree (the .weft CST: parse, edit, byte-exact round-trip) rowan = "0.16" @@ -131,7 +139,7 @@ tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } # utils -uuid = { version = "1", features = ["v4", "v5", "serde"] } +uuid = { version = "1", features = ["v4", "v5", "v7", "fast-rng", "serde"] } chrono = { version = "0.4", features = ["serde"] } # IANA time zones, for a cron schedule that means "nine in Paris". chrono-tz = "0.10" @@ -163,6 +171,15 @@ tempfile = "3" parking_lot = "0.12" filetime = "0.2" base64 = "0.22" +# The project worker's allocator (`weft_engine::Jemalloc`, declared by the +# generated worker `main`): glibc's kept a burst's freed memory for good, +# so a worker past one read as full while idle. Background threads purge +# what was freed even while nothing allocates (an idle worker), the way +# Restate's server runs it. +tikv-jemallocator = { version = "0.7", features = ["background_threads"] } +# The worker's own CPU profile (`weft_engine::profile`, `WEFT_PROFILE`): +# sampled with signals, so a machine without `perf` profiles too. +pprof = { version = "0.15", features = ["flamegraph"] } # internal workspace crates weft-core = { path = "crates/weft-core" } diff --git a/README.md b/README.md index d6f5be7d..7d4ac51f 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Running an agent puts one model in charge of the whole job, and you spend time a In Weft, you break down any task to a graph of scoped nodes. Every node (LLM, agents, human in the loop, API, database, ...) gets exactly the context it needs to run, and the graph takes on the task of coordinating each component properly without any nodes having to spend compute on it. -The compiler checks every connection before the run to make sure your orchestration will run properly, and every value that passes between nodes is recorded in a journal you can watch afterwards. +The compiler checks every connection before the run to make sure your orchestration will run properly, and every value that passes between nodes is recorded in a journal you can watch afterwards (unless its trigger turns `recorded` off). ## Meet Tangle @@ -42,7 +42,7 @@ Tangle loads automatically when you open a weft project, and it works with all m - **Readable.** The language describes how components interact, not what happens inside them, so a whole system fits in a few lines of code that are fast to produce and hard to get wrong. The internal representation of the program is natively a graph, so even a non-developer can follow what is happening and fully interact with it through a GUI. - **Compiled.** The compiler proves things about your orchestration before it runs: today it checks port types, required connections and graph structure; it will grow toward compiler flags that pin down what the system is allowed to do at runtime. The compiler emits a Rust crate around the graph and builds it into a native binary, so you get Rust's memory safety and speed. - **Alive.** Execution is not one pass through a graph. Parts can stay up, wait for answers, and keep talking to each other while the rest of the program carries on. -- **Durable.** A program can save a wait for a human or an external event, release its worker, and continue when the answer arrives. Every run is written down as it happens, so you can open any of them and see what happened, or is happening, interactively through the graph. +- **Durable.** A program can save a wait for a human or an external event, release its worker, and continue when the answer arrives. Every run is written down as it happens (unless its trigger turns `recorded` off), so you can open any of them and see what happened, or is happening, interactively through the graph. - **Tests itself.** Any piece of the program can run on its own, with values you make up or against a real service you connected. Tangle uses that to build without you: it runs one step, reads what came out, fixes it, then re-runs only what changed. When a case comes out right, Tangle freezes it and runs it again after later changes to see what moved. You hand it a job and find the program working when you come back. For how to build this way, go and read [Sequential Diffusion Programming](https://weavemindai.github.io/weft/thinking/sdp.html). - **Infrastructure included.** Write `pg = PostgresDatabase` and the program gets a Postgres of its own. The node says which container it needs; the runtime starts that container with a disk that survives restarts, and the rest of the program reaches it through one output, `pg.access`. Anything that runs in a container can be an infrastructure node; if you want to write your own, go and read [infrastructure nodes](https://weavemindai.github.io/weft/nodes/infrastructure.html). - **Dynamic vocabulary.** A node starts with two files: its declaration and its Rust code. The ctx supplies the shared machinery for credentials, storage, signals and execution records. It is opinionated on purpose: there is one way a file is stored and one way a credential is held, and you get every parameter but never the mechanism, so each piece is hardened once instead of half-written again in every node. Your node implements its own job without rebuilding those mechanisms. Where weft's job stops and yours begins is written down in [the commandments of plumbing](https://weavemindai.github.io/weft/thinking/plumbing.html). diff --git a/TODO.md b/TODO.md index 50fc2be9..3e19d383 100644 --- a/TODO.md +++ b/TODO.md @@ -518,3 +518,186 @@ line rather than a failed read; and whether a rate limit belongs per token or per (token, node), given that one client legitimately watches one bridge closely and has no reason to sweep every display it can see. + +## A stream writes two journal rows per item + +**Problem.** Every item a stream produces writes one `PortEmitted` row +and one `PulsesConsumed` row (`crates/weft-journal/src/events.rs`, the +TODO on `PulsesConsumed`). That is fine for the streams people run +today, but a stream of ten million items would write about twenty +million rows. + +**Direction.** Write stream items in windows, the way a bus already +batches its appends. This belongs with the packed record format of the +current performance work (one write per step for a durable run, one +compact row per run for a fast one), which cuts the rows per item for +the same reason it cuts them per run. + +[Update Notice Warning] If we touch `PulsesConsumed`, the stream +runtime (`crates/weft-engine/src/stream_runtime.rs`) or the journal's +record format, revisit this entry. + +## The engine interprets the wiring: precompile its plan, or compile the wiring + +**When.** After the current round of performance work (fast and durable +runs, callers straight to the worker, one journal writer per worker), +with numbers measured then. + +**Problem.** Node bodies are compiled Rust, but the wiring between them +is interpreted at run time, the way Python is. The program is a data +structure (`ProjectDefinition`), and the execution loop +(`crates/weft-engine/src/execution_driver.rs`) walks it: + +``` +TODAY: node bodies are compiled Rust, the wiring is interpreted +─────────────────────────────────────────────────────────────────────── +loop: scan every node of the program ─► "who has all its inputs?" + (weft_core::exec::ready::find_ready_nodes) + └► for each ready node: build a context, start a task for it, + wait for its message back, file its outputs in a table keyed + by node and port NAMES (text), write its records ─► scan again + +COMPILED: the wiring becomes Rust code too +─────────────────────────────────────────────────────────────────────── +async fn ping(request) { + let fired = route.run(request).await?; // a wire is a variable + let body = json_object.run(fired).await?; + reply.run(body).await +} +``` + +**Measured on 2026-10-08**, after the performance round (fast and durable +runs, callers straight to the worker, one journal writer per worker, the +program's facts worked out once per program, a step that ends at once run on +the drive's own task). The bench project (40 nodes, `bench/weft-project`), +its unrecorded ping (`Route` to `JsonObject` to `Reply`): + +- In a test process (release build, records in memory, no network or + database), one run costs about 73 µs of CPU, down from 172 µs before the + round. +- In the local install under 50 callers, the worker spends about 365 µs of + CPU per call (axum in Docker: about 50 µs). A profile of that worker, + taken before unrecorded runs stopped keeping their history, put about + half of its CPU in the drive loop (readiness, settling boundaries, + building each step's input bag, applying emissions) and the rest around + it: serving the call, the door, the run's birth, the record. +- Throughput: 23,181 unrecorded and 12,092 fast pings a second, against + axum's 36,996 behind Docker's port relay; 0.6 ms and 0.7 ms at one caller + against axum's 0.3 ms. + +So the loop is now the largest single cost, about half of the worker's own, +and it grows with the size of the graph. The rest (the record, the door, +the call) is ordinary code that can be trimmed on its own. The probe was a +throwaway test driving `run_one_execution_observed` over `engine_test_rig` +with the bench project's definition and stand-in nodes; rebuild it to +measure again (`bench/README.md` has the worker-side profile). + +**Direction, two levels.** + +1. **Keep the interpreter, precompile its plan.** Everything the loop + works out at run time (who feeds whom, which inputs are required, + which ports are wired) becomes plain numbers decided at compile time: + no name lookups, no scan of every node each step, and a straight + chain runs in place instead of starting a task per node. Pausing and + resuming do not change. +2. **Compile the wiring fully**, as in the sketch: a wire costs what a + Rust variable costs, and the compiler, which already produces a + native binary per program, generates it. The hard part is pausing + and resuming. Today a paused run is rebuilt by folding its records + back into the tables; compiled code would resume the way Restate and + Temporal do, by running the function again from the top while every + node that already finished hands back its recorded output instead of + running again. Loops, streams, buses, `_should_flow` skip cascades, + `catchErrors` and suspensions all have to be expressed in that model, + so this is a rewrite of the engine and of how runs resume. + +**Why deferred.** The language work comes first. Choose between the two +levels with numbers measured then, and before choosing the second, list +what it needs feature by feature. + +[Update Notice Warning] If we touch `crates/weft-engine/src/execution_driver.rs`'s +dispatch loop or `weft_core::exec::ready`, revisit this entry. + +## "At once" is a share per copy, never the project's true count + +**Problem.** An entry's `callsAtOnce` is held at each worker's door, and +each copy takes its share: the limit divided by the copies alive at the +last tick, at least one (`crates/weft-engine/src/door/limits.rs`). Two +things follow. A copy whose share is full refuses a caller while another +copy has room, so under uneven routing the entry runs fewer at once than +its limit. And with more copies than the limit, every copy still gets +one, so together they run more at once than the limit allows. + +**Direction.** Count what is going across copies the way the per-minute +counts are: each copy states how many of an entry's runs it has going on +its tick, and admits against the copies' total. That is exact to about a +second, like the minute counts. A run born on a dead copy has to stop +counting, which the tick's "copies alive" already tells. + +**Why deferred.** Low impact for now: the share is wrong only with +several copies under uneven load, and the per-minute limits stay exact +to a second. + +[Update Notice Warning] If we touch `crates/weft-engine/src/door/limits.rs` +or `/v1/door/tick`, revisit this entry. + +## Every copy sends every caller's count every second + +**Problem.** A worker's tick (`/v1/door/tick`, once a second) states its +counts of the minute, and the per-caller ones are keyed by entry and +caller. An entry called by many different callers in one minute makes +every copy send one count per caller, every second, and hear every other +copy's. The tick grows with the number of distinct callers, not with the +traffic that matters for the limit. + +**Direction.** Send what changed since the last tick instead of the whole +minute, or keep per-caller counts on the copy the caller reached (one +caller mostly lands on one copy) and only exchange the entry-wide counts. + +**Why deferred.** Only matters with thousands of distinct callers a +minute on one project. + +[Update Notice Warning] If we touch `crates/weft-engine/src/door/limits.rs` +or `/v1/door/tick`, revisit this entry. + +## Several workers of a project can open more Postgres connections than the database takes + +**Problem.** The Postgres node keeps one pool per worker and database +(`catalog/postgres/postgres.rs`, `MOST_AT_ONCE`), held to 20 connections +so a few workers together stay under Postgres's default limit of 100. +That is a guess about how many workers there are. A project scaled to +many workers can still open more connections than the database accepts, +and the runs past it fail to connect. + +**Direction.** Put a connection pooler (PgBouncer, in transaction mode) +in front of the database inside the Postgres node's own infra, so every +worker talks to the pooler and the pooler holds the few real +connections. The worker's own limit can then be generous again. The open +question is a database the user brings, which has no infra of ours to +put the pooler in. + +[Update Notice Warning] If we touch `catalog/postgres/postgres.rs`'s +pool or the Postgres node's infra, revisit this entry. + +## A Python process runs one script at a time + +**Problem.** `ExecPython` keeps as many `python3` processes as the worker +has CPUs (`catalog/basic/exec_python/mod.rs`), and each runs one script at +a time. A script that mostly waits (an HTTP call, a `sleep`) holds its +process while it uses no CPU, so with every process busy waiting, the next +script queues for nothing. + +**Direction.** Each process runs several scripts at once on threads: a +waiting script releases Python's lock, so the others go on, and scripts +that compute still share the CPUs. Cap the scripts at once per process (a +setting) and benchmark `burn` and a waiting script at a few values to +find the best. On a Python built without the lock (supported since 3.14, +not the default build), the same threads compute in parallel too. Scripts +on threads of one process share its module state more directly than +today, so the node's docs would say so. + +**Why deferred.** A node's own optimisation, apart from the language +work around it. + +[Update Notice Warning] If we touch `catalog/basic/exec_python/mod.rs`'s +process pool, revisit this entry. diff --git a/catalog/api/close/metadata.json b/catalog/api/close/metadata.json index 29e8f917..6ab37513 100644 --- a/catalog/api/close/metadata.json +++ b/catalog/api/close/metadata.json @@ -56,6 +56,7 @@ } ], "features": { + "pure": true, "answersCaller": "end" }, "validate": [ diff --git a/catalog/api/reply/metadata.json b/catalog/api/reply/metadata.json index bbb77469..d78b7254 100644 --- a/catalog/api/reply/metadata.json +++ b/catalog/api/reply/metadata.json @@ -60,6 +60,7 @@ } ], "features": { + "pure": true, "answersCaller": "whole" }, "validate": [ diff --git a/catalog/api/route/metadata.json b/catalog/api/route/metadata.json index 2e050fde..ed04df25 100644 --- a/catalog/api/route/metadata.json +++ b/catalog/api/route/metadata.json @@ -60,23 +60,6 @@ ], "description": "Auth access node (ApiKeyAuth, JwtAuth, HmacAuth). Unwired, anyone with the URL may call. The established identity comes out on `caller`." }, - { - "name": "outlivesCaller", - "type": "Boolean", - "label": "May outlive the caller", - "description": "Off (default): the caller leaving cancels the run, and a long durable wait is killed instead of parked. On: the run is its own (answer early, keep working), so whatever runs after the answer must end by itself; anything that watches or loops needs `maxSessionSecs` or a TagRun/StopTagged pair." - }, - { - "name": "defaultHoldSecs", - "type": "Number", - "widget": { - "kind": "number", - "min": 1, - "step": 1 - }, - "label": "Hold seconds before giving up on a wait", - "description": "Caller-tied run only: how long a node's wait holds the worker (and the connection) before giving up. Per-wait overrides win. Never unbounded." - }, { "name": "maxSessionSecs", "type": "Number", @@ -117,13 +100,6 @@ "type": "Boolean", "label": "Keep the exchange out of the journal", "description": "On: the journal keeps only sizes and counts, never the request or answer content (also hidden from you when debugging). Off (default): both are recorded, trimmed above 100 KB; raw bytes are never recorded." - }, - { - "name": "recorded", - "type": "Boolean", - "default": true, - "label": "Record each run", - "description": "Off: a successful or cancelled run leaves nothing in `weft executions` but its cost; a failed one is still written down whole. For cheap, frequently polled routes. An unrecorded run cannot wait (timer, form), cannot outlive its caller, and is lost if the worker dies." } ], "outputs": [ @@ -178,6 +154,7 @@ "body?": "JsonDict | String" }, "features": { + "pure": true, "isTrigger": true, "liveConnection": "http", "canAddOutputPorts": true diff --git a/catalog/api/route/tests.rs b/catalog/api/route/tests.rs index 2131e383..9cea6a13 100644 --- a/catalog/api/route/tests.rs +++ b/catalog/api/route/tests.rs @@ -224,7 +224,7 @@ async fn setup_registers(rig: FakeRig) -> WeftResult<()> { assert_eq!(spec["config"]["path"], "chat/{room}"); assert_eq!(spec["config"]["methods"], json!(["POST"])); assert_eq!(spec["config"]["data_type"], "text"); - assert_eq!(spec["config"]["can_suspend"], true); + assert_eq!(spec["config"]["outlives_caller"], true); assert_eq!(spec["config"]["auth"]["kind"], "none"); Ok(()) } diff --git a/catalog/api/socket/metadata.json b/catalog/api/socket/metadata.json index 3b2b5f50..1e8a0d09 100644 --- a/catalog/api/socket/metadata.json +++ b/catalog/api/socket/metadata.json @@ -41,23 +41,6 @@ ], "description": "Auth access node (ApiKeyAuth, JwtAuth, HmacAuth), checked on the opening request. Unwired, anyone with the URL may connect. The identity comes out on `caller`." }, - { - "name": "outlivesCaller", - "type": "Boolean", - "label": "May outlive the caller", - "description": "Off (default): the caller leaving cancels the run, and a long durable wait is killed instead of parked. On: the run is its own, so whatever it still does must end by itself; anything that watches or loops needs `maxSessionSecs` or a TagRun/StopTagged pair." - }, - { - "name": "defaultHoldSecs", - "type": "Number", - "widget": { - "kind": "number", - "min": 1, - "step": 1 - }, - "label": "Hold seconds before giving up on a wait", - "description": "Caller-tied run only: how long a node's wait holds the worker (and the connection) before giving up. Per-wait overrides win. Never unbounded." - }, { "name": "maxSessionSecs", "type": "Number", diff --git a/catalog/api/stream/metadata.json b/catalog/api/stream/metadata.json index b41a5920..03e6cefb 100644 --- a/catalog/api/stream/metadata.json +++ b/catalog/api/stream/metadata.json @@ -83,6 +83,7 @@ } ], "features": { + "pure": true, "answersCaller": "stream" }, "validate": [ diff --git a/catalog/api/testing.rs b/catalog/api/testing.rs index f2e5a5a7..7e8af855 100644 --- a/catalog/api/testing.rs +++ b/catalog/api/testing.rs @@ -11,7 +11,6 @@ use weft::caller::{ CallerRuntimeConfig, FakeCallerConnection, InboundMessage, LiveRequest, DEFAULT_INBOUND_WINDOW, }; use weft::signal::{Backpressure, DataType, ErrorMode, Protocol}; -use weft::wait::SuspendPolicy; fn config(protocol: Protocol, data_type: DataType) -> CallerRuntimeConfig { CallerRuntimeConfig { @@ -19,13 +18,12 @@ fn config(protocol: Protocol, data_type: DataType) -> CallerRuntimeConfig { data_type, backpressure: Backpressure::Block, error_mode: ErrorMode::Surface, - connect_timeout_secs: 5, max_inbound_bytes: 1 << 20, // The fake caller is never a socket, so nothing here reads this; // the default keeps it honest against the real config. caller_silence_secs: weft::signal::DEFAULT_CALLER_SILENCE_SECS, max_session_secs: 0, - suspend: SuspendPolicy::default(), + outlives_caller: false, inbound_window: DEFAULT_INBOUND_WINDOW, // A fake run writes no journal, so the default is here to keep // this config the same shape as the real one. diff --git a/catalog/bailey/bridge/images/bridge/src/actions.js b/catalog/bailey/bridge/images/bridge/src/actions.js index 455cd817..327e23cc 100644 --- a/catalog/bailey/bridge/images/bridge/src/actions.js +++ b/catalog/bailey/bridge/images/bridge/src/actions.js @@ -363,7 +363,7 @@ export function createActionRouter(bridge, messageStore) { } else { console.log(`[action] ${action} ok`); } - // SYNC: the /action envelope <-> crates/weft-dispatcher/src/api/infra.rs (infra_action_result), catalog/postgres/database/images/credential/bootstrap.py + // SYNC: the /action envelope <-> crates/weft-core/src/infra/action.rs, catalog/postgres/database/images/credential/bootstrap.py res.json({ result }); } catch (err) { // The action name is caller-supplied, so it goes in as an argument, diff --git a/catalog/bailey/bridge/metadata.json b/catalog/bailey/bridge/metadata.json index aacc31e2..930637a8 100644 --- a/catalog/bailey/bridge/metadata.json +++ b/catalog/bailey/bridge/metadata.json @@ -16,14 +16,14 @@ "type": "String", "default": "0.25", "label": "CPUs", - "description": "CPUs the machine sustains (`0.25`, `1`, `250m`). A cloud install picks its cheapest machine holding this and `memory`: a quarter CPU with 1Gi is Compute Engine's `e2-micro`, about $6 a month. A local install caps the container at it." + "description": "CPUs the machine sustains (`0.25`, `1`, `250m`). A cloud install picks its cheapest machine holding this and `memory`: a quarter CPU with 1Gi is Compute Engine's `e2-micro`, about $6 a month. A local install does not cap the container." }, { "name": "memory", "type": "String", "default": "1Gi", "label": "Memory", - "description": "Memory, a number followed by Gi, Mi, G or M (`1Gi`). Sized with `cpu`; a local install caps the container at it." + "description": "Memory, a number followed by Gi, Mi, G or M (`1Gi`). Sized with `cpu`; a local install does not cap the container." }, { "name": "machineType", @@ -37,7 +37,8 @@ { "name": "bridge", "type": "Infra", - "description": "The bridge's handle. Wire it into the `bridge` input of every other WhatsApp node." + "description": "The bridge's handle. Wire it into the `bridge` input of every other WhatsApp node. Baked: made when the bridge's infra is applied, so a run that reads only this output uses the saved handle and does not run this node.", + "baked": true }, { "name": "status", diff --git a/catalog/basic/cast/metadata.json b/catalog/basic/cast/metadata.json index cdcd3070..bf587ef7 100644 --- a/catalog/basic/cast/metadata.json +++ b/catalog/basic/cast/metadata.json @@ -27,6 +27,7 @@ } ], "features": { + "pure": true, "castPorts": { "input": "value", "output": "value" diff --git a/catalog/basic/debug/metadata.json b/catalog/basic/debug/metadata.json index 5943c7c6..9a245e29 100644 --- a/catalog/basic/debug/metadata.json +++ b/catalog/basic/debug/metadata.json @@ -19,6 +19,7 @@ ], "outputs": [], "features": { + "pure": true, "showDebugPreview": true } } diff --git a/catalog/basic/exec_python/build.rs b/catalog/basic/exec_python/build.rs deleted file mode 100644 index 0e5f338a..00000000 --- a/catalog/basic/exec_python/build.rs +++ /dev/null @@ -1,11 +0,0 @@ -// Stamp an rpath to the probed interpreter's lib directory into any -// binary linking this package. In a container both stages share the -// distro python, whose lib dir is already on the loader path (the -// extra rpath is inert); on a host build the probed python is whatever -// `python3` resolves to (conda, pyenv, ...), whose libpython the -// loader cannot find without this. -pub fn main() { - if let Some(lib_dir) = pyo3_build_config::get().lib_dir.as_ref() { - println!("cargo:rustc-link-arg=-Wl,-rpath,{lib_dir}"); - } -} diff --git a/catalog/basic/exec_python/deps.toml b/catalog/basic/exec_python/deps.toml index 34c49618..eba517a2 100644 --- a/catalog/basic/exec_python/deps.toml +++ b/catalog/basic/exec_python/deps.toml @@ -1,51 +1,11 @@ -# ExecPython embeds CPython via PyO3. -# -# Build stage: needs libpython3-dev and pkg-config so PyO3's build -# script can link against the interpreter's shared library. The -# linker writes a NEEDED entry for whichever libpython3.X is -# available at build time. -# -# Runtime stage: needs the same libpython3.X the builder linked -# against. Because the builder and runtime containers are -# derived from the SAME base image, the installed Python version -# matches by construction. No stable-ABI config shim, no manual -# version pinning. -# -# See docs/build-system.md for the per-distro matrix pattern. +# ExecPython runs each script in a `python3` process the worker starts +# (see mod.rs), so the worker image needs the interpreter and its +# standard library at runtime, and nothing at build time. [dependencies] -pyo3 = { version = "0.22", features = ["auto-initialize"] } - -[build-dependencies] -# The package's build.rs stamps an rpath to the probed interpreter's -# lib dir, so a host-built test binary finds the same libpython the -# link chose (conda/pyenv interpreters live outside the loader path). -pyo3-build-config = "0.22" - -[system.build.apt] -# python3 at build time: PyO3's build script runs it to probe -# the interpreter's sysconfig (version, lib dir, lib name). Kept -# separate from python3-dev even though dev depends on python3 -# on Debian; being explicit is cheaper than surprise. -default = ["python3", "libpython3-dev", "pkg-config"] - -[system.build.apk] -default = ["python3", "python3-dev", "pkgconfig"] - -[system.build.yum] -default = ["python3", "python3-devel", "pkg-config"] [system.runtime.apt] -# `libpython3.X` (no suffix) is the package that ships -# `libpython3.X.so.1.0`, which the PyO3 build links against by -# soname at build time. `python3` pulls the stdlib alongside so -# `import json` etc work at runtime. Keep per-distro entries -# because the libpython minor version is distro-specific. -default = ["python3", "libpython3.11"] -debian_12 = ["python3", "libpython3.11"] -debian_13 = ["python3", "libpython3.13"] -ubuntu_22_04 = ["python3", "libpython3.10"] -ubuntu_24_04 = ["python3", "libpython3.12"] +default = ["python3"] [system.runtime.apk] default = ["python3"] diff --git a/catalog/basic/exec_python/metadata.json b/catalog/basic/exec_python/metadata.json index 21f85791..92ab749e 100644 --- a/catalog/basic/exec_python/metadata.json +++ b/catalog/basic/exec_python/metadata.json @@ -1,7 +1,7 @@ { "type": "ExecPython", "label": "Python", - "description": "Run a Python snippet. Declare the node's inputs and outputs inline: `ExecPython(x: Number) -> (result: String) { code: \"...\" }`. Every declared input is a Python variable of the same name; one that received nothing is `None`, so an optional input reads as `if problem is None`. A file input (Image, Audio, Video, Blob, File) arrives as a plain dict with `url` (a download link minted for this firing, never written to the journal, so do not look for it there), `filename`, `mimeType`, `sizeBytes` and `key`: read `photo['url']` and fetch it. Return a file on a file-typed output by returning that same dict. The code must return a dict keyed by output port name; a missing key emits no pulse on that port, and `return {}` emits nothing. A key set to None sends a real null on a port whose type takes it (`last: JsonDict | Null`), and emits no pulse on any other port. An exception the script raises (a request it makes failing, a `raise` of its own) fails the node with its traceback, or, with `error` wired, comes out on `error` instead. Mistakes in the program always fail the run, `error` wired or not: code that does not compile, no dict returned, a key that is not a declared output (`error` included: raise to fail on purpose), a value of the wrong type for its port.", + "description": "Run a Python snippet. Declare the node's inputs and outputs inline: `ExecPython(x: Number) -> (result: String) { code: \"...\" }`. Every declared input is a Python variable of the same name; one that received nothing is `None`, so an optional input reads as `if problem is None`. A file input (Image, Audio, Video, Blob, File) arrives as a plain dict with `url` (a download link minted for this firing, never written to the journal, so do not look for it there), `filename`, `mimeType`, `sizeBytes` and `key`: read `photo['url']` and fetch it. Return a file on a file-typed output by returning that same dict. The code must return a dict keyed by output port name; a missing key emits no pulse on that port, and `return {}` emits nothing. A key set to None sends a real null on a port whose type takes it (`last: JsonDict | Null`), and emits no pulse on any other port. An exception the script raises (a request it makes failing, a `raise` of its own) fails the node with its traceback, or, with `error` wired, comes out on `error` instead. Mistakes in the program always fail the run, `error` wired or not: code that does not compile, no dict returned, a key that is not a declared output (`error` included: raise to fail on purpose), a value of the wrong type for its port. Scripts run in a few Python processes the worker keeps and every run on it shares: a script's own variables start fresh each time, but a module it imported, anything it sets on a module, environment variables, the working directory and files it writes stay for the next script on that process, so never keep a secret or one run's data there.", "tags": [ "python", "script", diff --git a/catalog/basic/exec_python/mod.rs b/catalog/basic/exec_python/mod.rs index b18b0c64..ad3f9cc2 100644 --- a/catalog/basic/exec_python/mod.rs +++ b/catalog/basic/exec_python/mod.rs @@ -1,5 +1,5 @@ -//! ExecPython: run a user-supplied Python snippet inside the -//! worker using an embedded CPython interpreter (PyO3). +//! ExecPython: run a user-supplied Python snippet in a `python3` +//! process beside the worker, from a pool the worker keeps. //! //! Wiring: //! @@ -17,12 +17,17 @@ //! optional input with a plain `if problem is None`. `return //! ` in the user code supplies the output pulses. //! -//! - Inputs are converted serde_json → Python via `json_to_py`. A -//! recursive walk over the JSON tree keeps types straightforward: -//! strings to `str`, numbers to `int`/`float`, nulls to `None`, -//! arrays to `list`, objects to `dict`. No custom classes leak -//! across the boundary; we round-trip through JSON twice per -//! call but the shape is simple and predictable. +//! - The script runs in a Python process of its own, one of a few the +//! worker holds for its runs to share (`ctx.shared`), as many as +//! the worker has CPUs, started the first time a script needs one: +//! scripts run side by side. Each script's own names start fresh (its +//! own scope), but the process is shared on purpose: a module a script +//! imported stays imported for the next, and so does whatever it set on +//! a module, the environment, the working directory and files it +//! wrote. Inputs go to it as JSON and the answer comes +//! back as JSON: strings to `str`, numbers to `int`/`float`, nulls to +//! `None`, arrays to `list`, objects to `dict`, and back. No custom +//! class leaks across. What a script prints goes to the worker's log. //! //! - A file arrives UNWRAPPED on a port DECLARED as one (the same //! question the way out asks, so a dict that happens to carry a @@ -58,37 +63,30 @@ //! keys, a string on a number port). `error` is weft's, so a script //! that wants to fail on purpose raises. //! -//! - A cancelled run stops its script. Python offers one way to stop -//! code running on another thread: `PyThreadState_SetAsyncExc` -//! raises an exception in that thread at its next bytecode. The -//! script's thread records its id in an [`Interrupt`] while it runs, -//! and the guard the node body holds raises `KeyboardInterrupt` there -//! when the body ends early (the run was cancelled, or the engine -//! dropped the body). A script inside one long C call (a `time.sleep`, -//! a blocking socket read) stops when that call returns, the earliest -//! moment Python checks. `KeyboardInterrupt` is outside `Exception`, -//! so a script's `except Exception` does not swallow it. +//! - A cancelled run stops its script at once: its process is killed +//! (a script inside one long C call, a `time.sleep`, a blocking read, +//! included), and the next script that needs one starts a fresh one. +//! The same happens when the engine drops the node body. //! //! Isolation: the worker the node runs in IS the isolation boundary; //! the Python executes there with the same access that worker already -//! has, and is not sandboxed further. Running a project therefore runs +//! has, and is not sandboxed further (a process of its own keeps a +//! script that crashes Python from taking the worker with it, nothing +//! more). Running a project therefore runs //! its ExecPython code with that worker's privileges, the same trust //! model as running the project's own program. use async_trait::async_trait; -use pyo3::prelude::*; -use pyo3::types::{PyBool, PyDict, PyFloat, PyList, PyString}; -use pyo3::ToPyObject; -use serde_json::{Map, Number, Value}; +use serde_json::Value; use std::collections::HashMap; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; +use tokio::io::{AsyncBufReadExt, AsyncWriteExt}; use weft::node::NodeOutput; use weft::storage::media::{media_slots, substitute_media}; use weft::weft_type::FileKind; use weft::context::ERROR_PORT; -use weft::{node_error, ExecutionContext, Node, NodeErrExt, NodeManifest, StoredFile, WeftError, WeftResult, WeftType}; +use weft::{node_error, ExecutionContext, Node, NodeManifest, StoredFile, WeftError, WeftResult, WeftType}; #[derive(NodeManifest)] pub struct ExecPythonNode; @@ -134,19 +132,8 @@ impl Node for ExecPythonNode { }) .collect(); - // PyO3 needs the GIL which it acquires on whatever sync - // thread we call from. Hop off the async executor for the - // blocking call so we don't stall other node invocations. - // `_stop` interrupts the script when this body ends before it - // does (see the module doc). - let interrupt = Arc::new(Interrupt::default()); - let _stop = StopOnDrop(interrupt.clone()); - let task = tokio::task::spawn_blocking(move || run_python(&code, inputs, &interrupt)); - let cancel = ctx.cancellation(); - let result = tokio::select! { - err = cancel.cancelled_err() => return Err(err), - joined = task => joined.node_err("ExecPython blocking task panicked")??, - }; + let interpreters = ctx.shared("exec_python", |()| async { Ok(Interpreters::new()) }).await?; + let result = run_python(&interpreters, &code, inputs, &ctx.cancellation()).await?; // Check the whole answer before anything goes out, so a wrong // key or type refuses the firing as the program mistake it is @@ -264,285 +251,210 @@ fn wrap_files(value: &Value, ty: &WeftType) -> WeftResult { Ok(substitute_media(value, ty, &replacements)) } -/// Execute `code` with the given input bindings and return the raw -/// key-value pairs the script returned. Each way it can go wrong has -/// its own kind: code that does not compile is an input error, an -/// answer that is not a dict of plain values is a type error (both are -/// the program's own mistakes), and an exception the script raises while -/// it runs is a node failure, the one kind `error` catches. -fn run_python(code: &str, inputs: Vec<(String, Value)>, interrupt: &Interrupt) -> WeftResult> { - Python::with_gil(|py| -> WeftResult> { - // Build the wrapper source once per call. Wrapping in a - // function lets the user write `return {...}` naturally. - // The signature lists every input port so Python's scope - // rules do the right thing (closures, shadowing, etc). - let param_names: Vec = - inputs.iter().map(|(k, _)| k.clone()).collect(); - let wrapper_source = format!( - "def __weft_user_fn({params}):\n{body}\n", - params = param_names.join(", "), - body = indent_block(code, " "), - ); +/// The script's process: what it reads, one request a line on the stdin +/// it keeps for that, and what it answers, one a line on the stdout it +/// keeps for that. A script never touches either: what it prints goes to +/// stderr, the worker's log, and what it reads from stdin is empty, so an +/// `input()` cannot eat the next request. Each +/// request is `{"names", "body", "args"}`: the input names, the script +/// indented under the function it becomes, and the inputs in that order. +const SERVER: &str = r#" +import json, os, sys, traceback +_answers = os.fdopen(os.dup(1), "w", buffering=1) +os.dup2(2, 1) +_requests = os.fdopen(os.dup(0), "r") +os.dup2(os.open(os.devnull, os.O_RDONLY), 0) +sys.stdin = open(os.devnull) - // Running the wrapper only defines the function, so what fails - // here is the code not compiling (a syntax or indentation error). - let globals = PyDict::new_bound(py); - py.run_bound(&wrapper_source, Some(&globals), None) - .map_err(|err| WeftError::Input(format!("the code does not compile: {}", python_error(py, &err))))?; - let user_fn = globals - .get_item("__weft_user_fn") - .map_err(|err| node_error(format!("locating __weft_user_fn: {}", python_error(py, &err))))? - .node_err("internal: ExecPython wrapper did not define __weft_user_fn")?; +def _described(e): + tb = "".join(traceback.format_exception(type(e), e, e.__traceback__)).rstrip() + return f"{type(e).__name__}: {e}\n{tb}" - // Convert each input into a Python value and call the - // wrapper as a positional-arg tuple matching the signature. - let args = PyList::empty_bound(py); - for (_, v) in &inputs { - let py_val = json_to_py(py, v) - .map_err(|err| node_error(format!("converting an input to Python: {}", python_error(py, &err))))?; - args.append(py_val) - .map_err(|err| node_error(format!("building the argument list: {}", python_error(py, &err))))?; - } - let ret = interrupt - .run(py, || user_fn.call1(args.to_tuple()))? - .map_err(|err| node_error(format!("the script raised {}", python_error(py, &err))))?; +def _plain(v): + if v is None or isinstance(v, (bool, str, int)): + return v + if isinstance(v, float): + if v != v or v in (float("inf"), float("-inf")): + raise TypeError(f"{v}, a float JSON cannot carry") + return v + if isinstance(v, list): + return [_plain(x) for x in v] + if isinstance(v, dict): + out = {} + for k, x in v.items(): + if not isinstance(k, str): + raise TypeError(f"a dict with a `{type(k).__name__}` key, which has no value on a port (its keys must be strings)") + out[k] = _plain(x) + return out + raise TypeError(f"a `{type(v).__name__}`, which has no value on a port (return None, bool, int, float, str, list or dict)") - // Falling off the end, a bare `return` and `return None` are one - // thing to Python, and none of them says which ports get what. - if ret.is_none() { - return Err(WeftError::Type( - "the script ended without returning a dict; end it with `return {...}` keyed by \ - output port (`return {}` emits nothing)" - .into(), - )); - } +def _answer(request): + source = "def __weft_user_fn(" + ", ".join(request["names"]) + "):\n" + request["body"] + "\n" + scope = {} + try: + exec(compile(source, "