From 61e10406d35bfde9c316ba04ea06f4290d1d1175 Mon Sep 17 00:00:00 2001 From: Quentin Date: Sun, 4 Oct 2026 11:05:05 +0200 Subject: [PATCH] Publish a release without waiting for macOS, compile weft once per release, and let a fresh install's first upload wait for its grant - The release's Linux legs build the CLI, the runtime and the images in one job; the runtime image takes the runtime binary that job built (`weft build-images --runtime-binary`, checked by starting it in the image) instead of compiling it again in Docker. - The macOS binaries are their own job and join the latest release afterwards, so the release, the manifest and every fork waiting on it no longer wait for the Intel runner. - The manifest names the source tree; setup.sh, the GCP install and the project deploy workflow match on it, so a fork that merged weft unchanged downloads instead of compiling. - scripts/release-cli.sh fetches the release's CLI for a checkout, waiting while upstream's release for that source is still building; a fork that has to build keeps its dependencies cached between runs. - Only main saves Rust caches; after each release one copy per job is kept and the rest deleted. - The AWS SDK uses ring like every other TLS client; aws-lc is gone. - Signing a storage link waits up to ten minutes for a new install's grant to take effect, through the IAM settle helper, now given a patience per change. --- .github/actions/build-binaries/action.yml | 41 ++++ .github/workflows/ci.yml | 8 + .github/workflows/install-gcp.yml | 51 ++-- .github/workflows/release.yml | 224 ++++++++++++------ Cargo.lock | 65 +---- Cargo.toml | 9 +- crates/weft-broker/src/runtime_storage.rs | 2 +- crates/weft-cli/src/commands/build.rs | 12 +- crates/weft-cli/src/commands/daemon.rs | 2 +- crates/weft-cli/src/images.rs | 69 +++++- crates/weft-cli/src/main.rs | 13 +- crates/weft-cli/templates/ci/gcp.yml | 52 ++-- crates/weft-core/src/net.rs | 19 +- crates/weft-platform-gcp/src/accounts.rs | 86 ++++++- crates/weft-platform-gcp/src/storage.rs | 32 ++- crates/weft-platform-traits/Cargo.toml | 1 + .../weft-platform-traits/src/object_store.rs | 8 + deploy/docker/runtime.Dockerfile | 23 +- docs/src/running/cloud.md | 14 +- scripts/release-cli.sh | 108 +++++++++ setup.sh | 27 ++- 21 files changed, 603 insertions(+), 263 deletions(-) create mode 100644 .github/actions/build-binaries/action.yml create mode 100755 scripts/release-cli.sh diff --git a/.github/actions/build-binaries/action.yml b/.github/actions/build-binaries/action.yml new file mode 100644 index 00000000..347071c1 --- /dev/null +++ b/.github/actions/build-binaries/action.yml @@ -0,0 +1,41 @@ +# The CLI and the runtime it installs, built on this runner and uploaded +# as two artifacts, for release.yml's linux and macos jobs. A local +# install runs `weft-runtime` as a process next to `weft`. +name: build-binaries +description: build weft + weft-runtime in release mode and upload both as artifacts +inputs: + asset: + description: >- + The CLI's asset name (`weft--`); the runtime's is the + same with `weft-` turned into `weft-runtime-`. + required: true +runs: + using: composite + steps: + # rustup reads rust-toolchain.toml (the single source of the + # version and components; see ci.yml's rust job for why no + # third-party toolchain action). + - shell: bash + run: rustup toolchain install + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + # SYNC: the runtime asset name (weft- -> weft-runtime-) <-> setup.sh + # (prebuilt artifacts block: runtime_asset) + - shell: bash + run: cargo build --release --locked -p weft-cli -p weft-runtime + - shell: bash + env: + ASSET: ${{ inputs.asset }} + run: | + cp target/release/weft "$ASSET" + cp target/release/weft-runtime "weft-runtime-${ASSET#weft-}" + echo "RUNTIME_ASSET=weft-runtime-${ASSET#weft-}" >> "$GITHUB_ENV" + - uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.asset }} + path: ${{ inputs.asset }} + if-no-files-found: error + - uses: actions/upload-artifact@v4 + with: + name: ${{ env.RUNTIME_ASSET }} + path: ${{ env.RUNTIME_ASSET }} + if-no-files-found: error diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6002566..c60308d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,12 @@ jobs: # pinned to a commit it demands an explicit toolchain input, # which would fork the version out of the toml. - run: rustup toolchain install + # Restores everywhere, saves only on main (release.yml's call of + # this file): a PR reads main's copy and leaves nothing behind, and + # release.yml's prune-caches keeps one copy per job. - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + with: + save-if: ${{ github.ref == 'refs/heads/main' }} # No compile-time query macros anywhere, so the workspace suite needs no # database. The tests that do want one are behind the `db-tests` feature. @@ -99,7 +104,10 @@ jobs: - uses: actions/checkout@v4 # rustup reads rust-toolchain.toml (see the rust job's comment). - run: rustup toolchain install + # Saves only on main, like the rust job's. - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + with: + save-if: ${{ github.ref == 'refs/heads/main' }} # The script runs the suites under cargo-nextest (every test side by # side); its prebuilt binary, from the project's own install script. - run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin" diff --git a/.github/workflows/install-gcp.yml b/.github/workflows/install-gcp.yml index d9514324..16991e2c 100644 --- a/.github/workflows/install-gcp.yml +++ b/.github/workflows/install-gcp.yml @@ -30,7 +30,9 @@ concurrency: jobs: install: - runs-on: ubuntu-latest + # 22.04: a runtime built here goes into the runtime image, whose + # Debian bookworm has an older glibc than newer Ubuntus link against. + runs-on: ubuntu-22.04 # No deadline: a first install builds weft's images, and a run cut # mid-apply leaves the Terraform state locked. GitHub's own 6-hour job # limit is the only cap. @@ -89,41 +91,44 @@ jobs: gcloud pubsub topics describe cloud-builds --project "${{ vars.GCP_PROJECT_ID }}" >/dev/null 2>&1 \ || gcloud pubsub topics create cloud-builds --project "${{ vars.GCP_PROJECT_ID }}" - # The CLI the release published for this exact commit, checked - # against its sha256, when the fork sits on that commit (the same - # match setup.sh makes); otherwise built from this checkout. - # SYNC: release tag + asset name + manifest.json keys <-> .github/workflows/release.yml (the release job), setup.sh (prebuilt artifacts block), crates/weft-cli/templates/ci/gcp.yml (download the weft CLI) + # The CLI weft's release built from this exact source, waited for + # when that release is still being built; otherwise built here, + # with the runtime, from a cache that keeps the dependencies + # between runs, so only what changed in weft compiles again. - name: the weft CLI for this checkout + id: release + env: + GH_TOKEN: ${{ github.token }} run: | - base=https://github.com/WeaveMindAI/weft/releases/download/latest - manifest=$(curl -fsSL --max-time 10 "$base/manifest.json" || true) - commit=$(sed -n 's/.*"commit": *"\([^"]*\)".*/\1/p' <<<"$manifest") - sha=$(sed -n 's/.*"sha256_weft-x86_64-linux": *"\([^"]*\)".*/\1/p' <<<"$manifest") - mkdir -p target/release - # A release being replaced can serve a binary its manifest does - # not vouch for yet: the checksum then fails, and the CLI is built. - if [ "$commit" = "$GITHUB_SHA" ] && [ -n "$sha" ] \ - && curl -fsSL "$base/weft-x86_64-linux" -o target/release/weft \ - && echo "$sha target/release/weft" | sha256sum -c -; then - chmod +x target/release/weft - echo "using the CLI the release built from $commit" + if scripts/release-cli.sh . target/release; then + echo "found=yes" >> "$GITHUB_OUTPUT" else - echo "this commit has no published CLI that checks out; building it" - rm -f target/release/weft - rustup toolchain install - cargo build --release --locked -p weft-cli + rc=$? + [ "$rc" -eq 2 ] || exit "$rc" fi + - if: steps.release.outputs.found != 'yes' + run: rustup toolchain install + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + if: steps.release.outputs.found != 'yes' + - name: build the weft CLI and runtime + if: steps.release.outputs.found != 'yes' + run: cargo build --release --locked -p weft-cli -p weft-runtime # The runtime image and the worker builder base this checkout names, # pushed to the install's registry. Each is taken from the install's # registry if there, else copied from the release's (the same tag is # the same contents), else built. A fork carrying its own changes - # hashes to new tags and gets exactly its own runtime. + # hashes to new tags and gets exactly its own runtime, made from the + # `weft-runtime` built above instead of compiled again in Docker. - name: build the runtime's images + env: + BUILT_HERE: ${{ steps.release.outputs.found != 'yes' }} run: | registry=$(terraform -chdir=deploy/terraform/gcp output -raw registry) gcloud auth configure-docker "${{ vars.GCP_REGION }}-docker.pkg.dev" --quiet - WEFT_IMAGE_REGISTRY="$registry" ./target/release/weft build-images --push > "$RUNNER_TEMP/refs.txt" + runtime=() + [ "$BUILT_HERE" = true ] && runtime=(--runtime-binary target/release/weft-runtime) + WEFT_IMAGE_REGISTRY="$registry" ./target/release/weft build-images --push "${runtime[@]}" > "$RUNNER_TEMP/refs.txt" # SYNC: `weft build-images` stdout and its order <-> crates/weft-cli/src/images.rs (SharedImages::bare_refs), .github/workflows/release.yml (build + push shared images) echo "TF_VAR_runtime_image=$(sed -n 1p "$RUNNER_TEMP/refs.txt")" >> "$GITHUB_ENV" echo "TF_VAR_builder_base_image=$(sed -n 2p "$RUNNER_TEMP/refs.txt")" >> "$GITHUB_ENV" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 02529a96..e860845d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -61,16 +61,23 @@ jobs: if: github.repository == 'WeaveMindAI/weft' uses: ./.github/workflows/ci.yml - # ---- shared images -> ghcr ------------------------------------------ + # ---- Linux binaries, and the shared images -> ghcr ------------------ # - # One leg per architecture, each on its NATIVE runner (no qemu), each - # pushing `-`; the stitch job below then publishes the bare - # `` as a multi-arch manifest list, so an Apple-silicon / ARM - # machine pulls arm64 and an x86 machine pulls amd64 from the same - # content-addressed tag. The tag itself hashes source content only, so - # both legs compute the identical ref (the stitch job verifies it). - images: - name: runtime image + builder base (${{ matrix.arch }}) + # Each leg builds the CLI and the runtime for its architecture, then the + # shared images, on its NATIVE runner (no qemu), pushing `-`; + # images-manifest then publishes the bare `` as a multi-arch + # manifest list, so an Apple-silicon / ARM machine pulls arm64 and an + # x86 machine pulls amd64 from the same content-addressed tag. The tag + # hashes source content only, so both legs compute the identical ref + # (the stitch job verifies it). + # + # The runtime image takes the `weft-runtime` its leg just built, so the + # image carries the very binary the release ships and nothing compiles + # it a second time inside Docker. That works because these legs run on + # Ubuntu 22.04, whose glibc is older than the image's Debian bookworm + # (`weft build-images` starts the binary in the image to make sure). + linux: + name: ${{ matrix.asset }} + images needs: [ci] strategy: # One architecture failing must not kill the other's build @@ -79,8 +86,12 @@ jobs: fail-fast: false matrix: include: - - { runner: ubuntu-latest, arch: amd64 } - - { runner: ubuntu-24.04-arm, arch: arm64 } + # ubuntu-22.04: oldest supported runner glibc, so the binary + # runs on every distro at least that new, and in the image. + # SYNC: CLI asset names <-> setup.sh (prebuilt artifacts + # block: the uname case), the macos job's matrix + - { runner: ubuntu-22.04, asset: weft-x86_64-linux, arch: amd64 } + - { runner: ubuntu-22.04-arm, asset: weft-aarch64-linux, arch: arm64 } runs-on: ${{ matrix.runner }} permissions: contents: read @@ -92,11 +103,9 @@ jobs: # Android/.NET/GHC toolchains are dead weight here. - name: free disk space run: sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/.ghcup - # rustup reads rust-toolchain.toml (the single source of the - # version and components; see ci.yml's rust job for why no - # third-party toolchain action). - - run: rustup toolchain install - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + - uses: ./.github/actions/build-binaries + with: + asset: ${{ matrix.asset }} - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 with: registry: ghcr.io @@ -109,17 +118,48 @@ jobs: # the stitch job. # SYNC: `weft build-images` stdout <-> crates/weft-cli/src/images.rs (SharedImages::bare_refs), .github/workflows/install-gcp.yml (build the runtime's images) - name: build + push shared images - run: cargo run --locked -p weft-cli -- build-images --push-suffix "-${{ matrix.arch }}" > image-refs.txt + run: | + ./target/release/weft build-images \ + --push-suffix "-${{ matrix.arch }}" \ + --runtime-binary target/release/weft-runtime > image-refs.txt - uses: actions/upload-artifact@v4 with: name: image-refs-${{ matrix.arch }} path: image-refs.txt if-no-files-found: error + # ---- macOS binaries -------------------------------------------------- + # + # Their own job, so nothing waits on them: the latest release goes out + # with the Linux binaries and the images, and release-macos adds these + # once they are built. The Intel runner is the slowest machine in the + # whole release by far. + macos: + name: ${{ matrix.asset }} + needs: [ci] + strategy: + fail-fast: false + matrix: + include: + # macos-latest is arm64 (Apple silicon); macos-15-intel is + # the x86_64 runner GitHub keeps until August 2027. + # SYNC: CLI asset names <-> setup.sh (prebuilt artifacts + # block: the uname case), the linux job's matrix + - { runner: macos-latest, asset: weft-aarch64-macos } + - { runner: macos-15-intel, asset: weft-x86_64-macos } + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/build-binaries + with: + asset: ${{ matrix.asset }} + + # The bare `` becomes a multi-arch manifest list over the two + # halves the linux job pushed. images-manifest: name: publish multi-arch image tags runs-on: ubuntu-latest - needs: [images] + needs: [linux] permissions: contents: read packages: write @@ -171,56 +211,39 @@ jobs: || { echo "::error::$ref is not anonymously pullable; open github.com/orgs/WeaveMindAI/packages and flip this package to Public"; exit 1; } done < refs-amd64/image-refs.txt - # ---- CLI binaries ---------------------------------------------------- - cli: - name: cli (${{ matrix.asset }}) - needs: [ci] - strategy: - # One platform failing must not take the others' binaries down - # with it; the release job below ships whichever legs succeeded - # (a missing asset just means "build from source" there, and the - # manifest lists only the assets that exist). - fail-fast: false - matrix: - include: - # ubuntu-22.04: oldest supported runner glibc, so the binary - # runs on every distro at least that new. - # SYNC: CLI asset names <-> setup.sh (prebuilt artifacts - # block: the uname case) - - { runner: ubuntu-22.04, asset: weft-x86_64-linux } - - { runner: ubuntu-22.04-arm, asset: weft-aarch64-linux } - # macos-latest is arm64 (Apple silicon); macos-15-intel is - # the x86_64 runner GitHub keeps until August 2027. - - { runner: macos-latest, asset: weft-aarch64-macos } - - { runner: macos-15-intel, asset: weft-x86_64-macos } - runs-on: ${{ matrix.runner }} + # ---- build caches ---------------------------------------------------- + # + # Only main saves Rust build caches (ci.yml's jobs skip saving on a + # PR), and each save is keyed by the Cargo.lock and toolchain it was + # built with, so a dependency bump leaves the previous copy behind next + # to the new one. Once every job that saves one has finished, each + # cache keeps its newest copy and the rest are deleted, so the repo's + # cache space holds one warm copy per job and platform. A cache no job + # saves any more goes on its own after seven days unused. + prune-caches: + name: keep one copy of each build cache + needs: [ci, linux, macos] + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + permissions: + actions: write + env: + GH_TOKEN: ${{ github.token }} steps: - - uses: actions/checkout@v4 - # rustup reads rust-toolchain.toml (the single source of the - # version and components; see ci.yml's rust job for why no - # third-party toolchain action). - - run: rustup toolchain install - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 - # The CLI and the runtime it installs, side by side: a local - # install runs `weft-runtime` as a process next to `weft`. - # SYNC: the runtime asset name (weft- -> weft-runtime-) <-> setup.sh - # (prebuilt artifacts block: runtime_asset) - - run: cargo build --release --locked -p weft-cli -p weft-runtime - - run: | - cp target/release/weft "${{ matrix.asset }}" - runtime_asset="${{ matrix.asset }}" - cp target/release/weft-runtime "weft-runtime-${runtime_asset#weft-}" - echo "RUNTIME_ASSET=weft-runtime-${runtime_asset#weft-}" >> "$GITHUB_ENV" - - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.asset }} - path: ${{ matrix.asset }} - if-no-files-found: error - - uses: actions/upload-artifact@v4 - with: - name: ${{ env.RUNTIME_ASSET }} - path: ${{ env.RUNTIME_ASSET }} - if-no-files-found: error + # A rust-cache key ends in two 8-hex segments (the toolchain and + # environment, then the lockfile); what is left before them names + # the job and platform, the thing that should exist only once. + - name: drop superseded copies + run: | + set -euo pipefail + gh cache list --repo "$GITHUB_REPOSITORY" --ref refs/heads/main --key v0-rust- \ + --limit 1000 --json id,key,createdAt \ + | jq -r 'group_by(.key | sub("-[0-9a-f]{8}-[0-9a-f]{8}$"; "")) + | map(sort_by(.createdAt) | reverse | .[1:][]) | .[] | "\(.id) \(.key)"' \ + | while read -r id key; do + echo "dropping $key (a newer copy replaced it)" + gh cache delete "$id" --repo "$GITHUB_REPOSITORY" + done # ---- VS Code extension ---------------------------------------------- vsix: @@ -302,11 +325,12 @@ jobs: release: name: update latest release runs-on: ubuntu-latest - needs: [cli, vsix, browser, images-manifest] - # Runs even when a CLI leg or the browser packaging failed (one - # unbuildable artifact must not strand every other's assets), but - # never without the vsix job (it supplies the version the manifest - # needs) and never without the stitched images: manifest.json is + needs: [linux, vsix, browser, images-manifest] + # Runs even when the browser packaging failed (one unbuildable + # artifact must not strand every other's assets), but never without + # the vsix job (it supplies the version the manifest needs) and never + # without the stitched images (which also means both Linux binaries + # built): manifest.json is # the "prebuilts for this commit are ready" signal setup.sh keys # on, and publishing it while no image is pullable would hand a # toolchain-less user a prebuilt CLI whose first `daemon start` @@ -320,10 +344,12 @@ jobs: - uses: actions/checkout@v4 # Exactly the release payload, by name: the image-refs artifacts # (internal plumbing for the stitch job) must never leak into the - # public asset list. + # public asset list. The macOS binaries are release-macos's, even + # when one happens to be built already, so what this job publishes + # never depends on which runner was faster. - uses: actions/download-artifact@v4 with: - pattern: weft-* + pattern: weft-*-linux path: dist merge-multiple: true # A pattern, so a failed browser job just means fewer files (the @@ -360,10 +386,14 @@ jobs: # missing manifest reads as "nothing published". The # manifest goes LAST so its commit stamp never vouches for # assets that are not up yet. - # SYNC: manifest.json keys <-> setup.sh (prebuilt artifacts block: manifest_value), .github/workflows/install-gcp.yml (the weft CLI for this checkout), crates/weft-cli/templates/ci/gcp.yml (download the weft CLI) + # SYNC: manifest.json keys <-> setup.sh (prebuilt artifacts block: manifest_value), scripts/release-cli.sh, the release-macos job gh release upload latest dist/* --clobber + # `tree` names the source content: a fork that merged this + # commit into an unchanged copy has another commit but the same + # tree, and the same binaries are right for it. { printf '{\n "commit": "%s",\n' "${GITHUB_SHA}" + printf ' "tree": "%s",\n' "$(git rev-parse "${GITHUB_SHA}^{tree}")" printf ' "vscode_version": "%s",\n' "${VSCODE_VERSION}" for f in dist/*; do printf ' "sha256_%s": "%s",\n' "$(basename "$f")" "$(sha256sum "$f" | cut -d' ' -f1)" @@ -386,6 +416,50 @@ jobs: done <<< "$assets" gh release edit latest --notes "Built from ${GITHUB_SHA}. setup.sh downloads these when its checkout matches manifest.json's commit." + # The macOS binaries join the release once they are built, each with + # its sha added to the manifest the release job wrote. + release-macos: + name: add the macOS binaries to the latest release + runs-on: ubuntu-latest + needs: [release, macos] + # Whichever macOS legs succeeded, as long as the release they join + # went out. + if: ${{ !cancelled() && needs.release.result == 'success' }} + permissions: + contents: write + env: + GH_TOKEN: ${{ github.token }} + steps: + # A pattern, so a failed leg just means one file fewer. + - uses: actions/download-artifact@v4 + with: + pattern: weft-*-macos + path: dist + merge-multiple: true + # SYNC: manifest.json keys <-> the release job (assemble + upload) + - name: upload + add to the manifest + run: | + set -euo pipefail + shopt -s nullglob + files=(dist/*) + if [ ${#files[@]} -eq 0 ]; then + echo "::warning::no macOS binary was built; the release has none for this commit" + exit 0 + fi + gh release download latest --repo "$GITHUB_REPOSITORY" -p manifest.json -D . + # One release at a time (the workflow's concurrency group), so + # the manifest is this run's; anything else is a bug to see. + [ "$(jq -r .commit manifest.json)" = "$GITHUB_SHA" ] || { + echo "::error::the latest release names $(jq -r .commit manifest.json), not $GITHUB_SHA" + exit 1 + } + gh release upload latest "${files[@]}" --repo "$GITHUB_REPOSITORY" --clobber + for f in "${files[@]}"; do + jq --arg k "sha256_$(basename "$f")" --arg v "$(sha256sum "$f" | cut -d' ' -f1)" \ + '. + {($k): $v}' manifest.json > manifest.next && mv manifest.next manifest.json + done + gh release upload latest manifest.json --repo "$GITHUB_REPOSITORY" --clobber + # ---- store publishes (each store gated on its own version tag, its # own tag pushed immediately after its publish, so a failure or an # interrupted run in one store never blocks another's retry and a diff --git a/Cargo.lock b/Cargo.lock index ca1d5ab3..cad0d6c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -202,28 +202,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "aws-lc-rs" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" -dependencies = [ - "aws-lc-sys", - "zeroize", -] - -[[package]] -name = "aws-lc-sys" -version = "0.41.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" -dependencies = [ - "cc", - "cmake", - "dunce", - "fs_extra", -] - [[package]] name = "aws-runtime" version = "1.7.5" @@ -415,18 +393,12 @@ dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", - "h2 0.3.27", "h2 0.4.15", - "http 0.2.12", "http 1.4.0", - "http-body 0.4.6", - "hyper 0.14.32", "hyper 1.9.0", - "hyper-rustls 0.24.2", "hyper-rustls 0.27.9", "hyper-util", "pin-project-lite", - "rustls 0.21.12", "rustls 0.23.40", "rustls-native-certs 0.8.3", "rustls-pki-types", @@ -474,7 +446,6 @@ checksum = "b8e6f5caf6fea86f8c2206541ab5857cfcda9013426cdbe8fa0098b9e2d32182" dependencies = [ "aws-smithy-async", "aws-smithy-http", - "aws-smithy-http-client", "aws-smithy-observability", "aws-smithy-runtime-api", "aws-smithy-schema", @@ -745,8 +716,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" dependencies = [ "find-msvc-tools", - "jobserver", - "libc", "shlex 1.3.0", ] @@ -859,15 +828,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" -[[package]] -name = "cmake" -version = "0.1.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" -dependencies = [ - "cc", -] - [[package]] name = "cmov" version = "0.5.4" @@ -1166,12 +1126,6 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" -[[package]] -name = "dunce" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" - [[package]] name = "ecdsa" version = "0.16.9" @@ -1418,12 +1372,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fs_extra" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" - [[package]] name = "futures" version = "0.3.32" @@ -2134,16 +2082,6 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" -[[package]] -name = "jobserver" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" -dependencies = [ - "getrandom 0.3.4", - "libc", -] - [[package]] name = "js-sys" version = "0.3.97" @@ -3152,7 +3090,6 @@ version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ - "aws-lc-rs", "once_cell", "ring", "rustls-pki-types", @@ -3220,7 +3157,6 @@ version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ - "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -5104,6 +5040,7 @@ dependencies = [ "aws-config", "aws-credential-types", "aws-sdk-s3", + "aws-smithy-http-client", "aws-smithy-types", "bytes", "futures", diff --git a/Cargo.toml b/Cargo.toml index dfd0af13..7274e864 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,8 +92,13 @@ tokio-tungstenite = { version = "0.24", features = ["rustls-tls-native-roots"] } # S3-compatible object storage (the ObjectStore slot: GCS / SeaweedFS / AWS / # any S3 endpoint via an endpoint-URL override). rustls so we don't drag in # a native TLS toolchain, matching reqwest above. -aws-config = { version = "1", default-features = false, features = ["rustls", "rt-tokio", "behavior-version-latest"] } -aws-sdk-s3 = { version = "1", default-features = false, features = ["rustls", "rt-tokio", "sigv4a"] } +aws-config = { version = "1", default-features = false, features = ["rt-tokio", "behavior-version-latest"] } +aws-sdk-s3 = { version = "1", default-features = false, features = ["rt-tokio", "sigv4a"] } +# The AWS SDK's HTTPS client, on `ring` like every other TLS client here. +# The SDK's own `rustls` feature picks `aws-lc-rs`, a second crypto library +# that compiles a whole C library with cmake on every cold build; the client +# is handed to the SDK instead (`weft_platform_traits::object_store`). +aws-smithy-http-client = { version = "1", default-features = false, features = ["rustls-ring"] } aws-credential-types = { version = "1", features = ["hardcoded-credentials"] } aws-smithy-types = "1" diff --git a/crates/weft-broker/src/runtime_storage.rs b/crates/weft-broker/src/runtime_storage.rs index ca2751fc..ff008c28 100644 --- a/crates/weft-broker/src/runtime_storage.rs +++ b/crates/weft-broker/src/runtime_storage.rs @@ -144,7 +144,7 @@ pub fn router() -> Router> { // `Other`/anyhow chain carries internal detail (SQL text, driver messages, table // names) that must not be disclosed to attacker-controlled code. The typed 4xx // variants below are safe and actionable, so they keep their specific messages. -const INTERNAL_STORAGE_ERROR_BODY: &str = "internal storage error"; +const INTERNAL_STORAGE_ERROR_BODY: &str = "internal storage error (the broker's log names the cause)"; fn map_err(e: RuntimeStoreError) -> ApiError { let completing = matches!(e, RuntimeStoreError::Completing(_)); diff --git a/crates/weft-cli/src/commands/build.rs b/crates/weft-cli/src/commands/build.rs index 0d371e59..9953442f 100644 --- a/crates/weft-cli/src/commands/build.rs +++ b/crates/weft-cli/src/commands/build.rs @@ -54,8 +54,15 @@ pub async fn run(ctx: Ctx, node_set: weft_core::builds::NodeSet) -> Result<()> { /// one per line, for the workflow to capture; progress rides stderr. /// `--print` stops after resolving: the refs this tree's content hashes /// to, touching no image (setup.sh keys its engine-change sweep on the -/// builder-base line moving). -pub async fn run_build_images(push: bool, push_suffix: Option, print: bool) -> Result<()> { +/// builder-base line moving). `runtime_binary` is a `weft-runtime` built +/// outside Docker, copied into the runtime image instead of compiled +/// (`images::ensure_runtime_image`). +pub async fn run_build_images( + push: bool, + push_suffix: Option, + print: bool, + runtime_binary: Option<&std::path::Path>, +) -> Result<()> { if print { // The SAME list the ensure path prints (`bare_refs` carries // the stdout contract), never a second construction of it. @@ -72,6 +79,7 @@ pub async fn run_build_images(push: bool, push_suffix: Option, print: bo let shared = crate::images::ensure_all_shared_images( false, push_suffix.as_deref(), + runtime_binary, ) .await?; if push || push_suffix.is_some() { diff --git a/crates/weft-cli/src/commands/daemon.rs b/crates/weft-cli/src/commands/daemon.rs index 1cc0ce4f..e08b9a01 100644 --- a/crates/weft-cli/src/commands/daemon.rs +++ b/crates/weft-cli/src/commands/daemon.rs @@ -1267,7 +1267,7 @@ async fn start(install: &Install, rebuild: bool) -> Result<()> { refuse_an_older_install(install).await?; let ports = ports(install)?; - let shared = images::ensure_all_shared_images(rebuild, None).await?; + let shared = images::ensure_all_shared_images(rebuild, None, None).await?; images::hold_standard_worker(&install.id, &shared.worker).await?; ensure_network().await?; ensure_postgres(install, ports.postgres).await?; diff --git a/crates/weft-cli/src/images.rs b/crates/weft-cli/src/images.rs index f5895ee7..d4284797 100644 --- a/crates/weft-cli/src/images.rs +++ b/crates/weft-cli/src/images.rs @@ -260,7 +260,7 @@ pub async fn ensure_builder_base_at(image_ref: &str, rebuild: bool) -> Result<() // `build::stage_builder_base_context`. let ctx = weft_compiler::build::stage_builder_base_context(&root) .map_err(|e| anyhow::anyhow!("stage builder-base context: {e}"))?; - docker_build(image_ref, &ctx.join("Dockerfile"), &ctx, &[], &[]).await?; + docker_build(image_ref, &ctx.join("Dockerfile"), &ctx, &[], &[], &[]).await?; } } // Builder-base images are large (~1GB+: debian + rustup + @@ -329,11 +329,24 @@ pub fn runtime_image_ref() -> Result { Ok(qualified_ref(RUNTIME_REPO, &short_hex(&hasher.finalize()))) } +/// The build context a prebuilt runtime binary rides in, and the file +/// name it has there. +// SYNC: PREBUILT_CONTEXT + PREBUILT_BINARY + the RUNTIME_BINARY arg <-> deploy/docker/runtime.Dockerfile (the prebuilt stage) +const PREBUILT_CONTEXT: &str = "weft-prebuilt"; +const PREBUILT_BINARY: &str = "weft-runtime"; + /// Make the runtime image exist locally under `image_ref`: present -> /// done; pull -> done; build `deploy/docker/runtime.Dockerfile`. /// `rebuild` skips straight to the build, for when a present image is /// corrupt or hand-modified. -pub async fn ensure_runtime_image(image_ref: &str, rebuild: bool) -> Result<()> { +/// +/// With `prebuilt`, the build copies that `weft-runtime` binary in +/// instead of compiling one. It must be built from this same checkout: +/// the ref hashes the source, and nothing can check a binary against +/// it. Whether it RUNS in the image (this machine's architecture, a +/// glibc no newer than Debian bookworm's) is checked by starting it +/// there once, and an image it cannot run in is removed, never pushed. +pub async fn ensure_runtime_image(image_ref: &str, rebuild: bool, prebuilt: Option<&Path>) -> Result<()> { if !rebuild { if image_present(image_ref).await? { eprintln!("image {image_ref} present; skipping"); @@ -345,7 +358,40 @@ pub async fn ensure_runtime_image(image_ref: &str, rebuild: bool) -> Result<()> } let root = weft_compiler::build::resolve_weft_root().map_err(|e| anyhow::anyhow!("resolve weft repo root: {e}"))?; let dockerfile = root.join("deploy/docker").join(RUNTIME_DOCKERFILE); - docker_build(image_ref, &dockerfile, &root, &[], &[]).await + let Some(binary) = prebuilt else { + return docker_build(image_ref, &dockerfile, &root, &[], &[], &[]).await; + }; + // Its own context holding only the binary, under the name the + // Dockerfile copies, whatever the file was called on disk. + let staged = tempfile::tempdir()?; + std::fs::copy(binary, staged.path().join(PREBUILT_BINARY)) + .map_err(|e| anyhow::anyhow!("stage the prebuilt runtime {}: {e}", binary.display()))?; + docker_build( + image_ref, + &dockerfile, + &root, + &[], + &["RUNTIME_BINARY=prebuilt".to_string()], + &[(PREBUILT_CONTEXT, staged.path())], + ) + .await?; + let probe = docker().args(["run", "--rm", image_ref, "--version"]).output().await?; + if !probe.status.success() { + let removed = docker().args(["rmi", image_ref]).output().await?; + anyhow::ensure!( + removed.status.success(), + "remove {image_ref} after its runtime failed to start: {}", + String::from_utf8_lossy(&removed.stderr).trim() + ); + anyhow::bail!( + "the prebuilt runtime {} does not run in the image (removed {image_ref}): {}\n\ + build it for this machine's architecture against a glibc no newer than \ + Debian bookworm's (Ubuntu 22.04 is), or leave out --runtime-binary to compile it in the image", + binary.display(), + String::from_utf8_lossy(&probe.stderr).trim() + ); + } + Ok(()) } /// Every shared image one ensure materialized: the runtime, the builder @@ -387,16 +433,21 @@ pub fn suffixed_ref(bare: &str, ref_suffix: Option<&str>) -> String { /// content-addressed refs. With `ref_suffix`, each image is materialized /// under `` instead of its bare name: the release workflow's /// per-architecture halves, checked and pulled by their OWN names. +/// `prebuilt_runtime` is [`ensure_runtime_image`]'s `prebuilt`. /// /// `tokio::join!`, NOT `try_join!`: an early bail would drop the sibling /// future while its `docker build` child keeps running detached; join! /// lets both finish, then every error is reported together. -pub async fn ensure_all_shared_images(rebuild: bool, ref_suffix: Option<&str>) -> Result { +pub async fn ensure_all_shared_images( + rebuild: bool, + ref_suffix: Option<&str>, + prebuilt_runtime: Option<&Path>, +) -> Result { let runtime = runtime_image_ref()?; let base = builder_base_ref()?; let (runtime_target, base_target) = (suffixed_ref(&runtime, ref_suffix), suffixed_ref(&base, ref_suffix)); let (r, b) = tokio::join!( - ensure_runtime_image(&runtime_target, rebuild), + ensure_runtime_image(&runtime_target, rebuild, prebuilt_runtime), ensure_builder_base_at(&base_target, rebuild), ); let failures: Vec = [("runtime", r), ("builder-base", b)] @@ -558,12 +609,15 @@ fn idle_days(last_used: &str) -> u64 { /// builds (builder base, runtime, worker, infra, node-test) /// funnels here. `labels` stamp the `weft.dev/*` filters later GC /// selects on (empty for the shared images, which GC by repo+tag). +/// `named_contexts` are extra build contexts the Dockerfile reads with +/// `COPY --from=`. pub(crate) async fn docker_build( image_ref: &str, dockerfile: &Path, context: &Path, labels: &[String], build_args: &[String], + named_contexts: &[(&str, &Path)], ) -> Result<()> { eprintln!( "building image {image_ref} (this may take several minutes on first run; \ @@ -583,6 +637,9 @@ pub(crate) async fn docker_build( for arg in build_args { cmd.args(["--build-arg", arg]); } + for (name, dir) in named_contexts { + cmd.arg("--build-context").arg(format!("{name}={}", dir.display())); + } let status = cmd.arg(context).status().await?; if !status.success() { anyhow::bail!("docker build {image_ref} failed with {status}"); @@ -604,7 +661,7 @@ pub(crate) async fn docker_build_worker( ) -> Result<()> { let lane = CompileLane::hold().await?; let arg = format!("{}={}", weft_compiler::worker_image::COMPILE_LANE_ARG, lane.number); - docker_build(image_ref, dockerfile, context, labels, &[arg]).await + docker_build(image_ref, dockerfile, context, labels, &[arg], &[]).await } /// How many worker builds compile side by side when nobody says diff --git a/crates/weft-cli/src/main.rs b/crates/weft-cli/src/main.rs index 419445b2..e15cb2ba 100644 --- a/crates/weft-cli/src/main.rs +++ b/crates/weft-cli/src/main.rs @@ -111,8 +111,15 @@ enum Cmd { /// Only print the refs this tree resolves to, one per line, /// touching nothing (no ensure, no docker). What setup.sh /// keys its engine-change sweep on. - #[arg(long, conflicts_with_all = ["push", "push_suffix"])] + #[arg(long, conflicts_with_all = ["push", "push_suffix", "runtime_binary"])] print: bool, + /// Put this `weft-runtime` binary in the runtime image instead of + /// compiling one inside Docker. It must be built from this same + /// checkout, for this machine, against a glibc no newer than + /// Debian bookworm's. The release workflow passes the binary its + /// CLI job built, so the runtime compiles once per release. + #[arg(long, value_name = "PATH")] + runtime_binary: Option, }, /// Manage a node's service connection from the terminal: list the /// stored connections and pick one, connect a new account (paste a @@ -1687,8 +1694,8 @@ async fn run(cli: Cli) -> anyhow::Result<()> { Cmd::New { name, assistants, remember, ci } => commands::new::run(ctx, name, assistants, remember, ci).await, Cmd::Ci { action: CiCmd::Add { cloud } } => commands::ci::add(ctx, cloud).await, Cmd::Build { referenced } => commands::build::run(ctx, node_set(referenced)).await, - Cmd::BuildImages { push, push_suffix, print } => { - commands::build::run_build_images(push, push_suffix, print).await + Cmd::BuildImages { push, push_suffix, print, runtime_binary } => { + commands::build::run_build_images(push, push_suffix, print, runtime_binary.as_deref()).await } Cmd::Connect { opts } => commands::connect::run(ctx, opts).await, Cmd::Options { step, field, search } => { diff --git a/crates/weft-cli/templates/ci/gcp.yml b/crates/weft-cli/templates/ci/gcp.yml index 416d308e..3e8dbb7d 100644 --- a/crates/weft-cli/templates/ci/gcp.yml +++ b/crates/weft-cli/templates/ci/gcp.yml @@ -44,10 +44,10 @@ jobs: # The source of the exact weft the install runs (the CLI compiles the # project against its nodes), and its CLI: a project compiled by - # another weft is refused there. The release's own CLI when the - # release was built from that commit (checked against its sha256), - # otherwise built from the source and cached for the next run. - # SYNC: release tag + asset name + manifest.json keys <-> .github/workflows/release.yml (the release job), .github/workflows/install-gcp.yml (the weft CLI for this checkout) + # another weft is refused there. The CLI weft's release built from + # that source (waited for when the release is still being built), + # otherwise built here from a cache that keeps the dependencies + # between runs, so a newer weft compiles only what changed in it. - uses: actions/checkout@v7 with: repository: ${{ vars.WEFT_SOURCE_REPOSITORY }} @@ -56,55 +56,31 @@ jobs: - name: download the weft CLI id: download + env: + GH_TOKEN: ${{ github.token }} run: | - base=https://github.com/WeaveMindAI/weft/releases/download/latest - manifest=$(curl -fsSL --max-time 10 "$base/manifest.json" || true) - commit=$(sed -n 's/.*"commit": *"\([^"]*\)".*/\1/p' <<<"$manifest") - sha=$(sed -n 's/.*"sha256_weft-x86_64-linux": *"\([^"]*\)".*/\1/p' <<<"$manifest") - mkdir -p "$RUNNER_TEMP/weft" - # A release being replaced can serve a binary its manifest does - # not vouch for yet: the checksum then fails, and the CLI is built. - if [ "$commit" = "${{ vars.WEFT_SOURCE_COMMIT }}" ] && [ -n "$sha" ] \ - && curl -fsSL "$base/weft-x86_64-linux" -o "$RUNNER_TEMP/weft/weft" \ - && echo "$sha $RUNNER_TEMP/weft/weft" | sha256sum -c -; then - chmod +x "$RUNNER_TEMP/weft/weft" + if .weft/scripts/release-cli.sh .weft "$RUNNER_TEMP/weft"; then echo "$RUNNER_TEMP/weft" >> "$GITHUB_PATH" - echo "using the CLI the release built from $commit" echo "found=yes" >> "$GITHUB_OUTPUT" else - echo "the release has no CLI built from ${{ vars.WEFT_SOURCE_COMMIT }} that checks out; building it" + rc=$? + [ "$rc" -eq 2 ] || exit "$rc" fi - # Restored and saved as two steps, so a run that fails after the - # build still keeps it for the next one. - - uses: actions/cache/restore@v6 + - if: steps.download.outputs.found != 'yes' + working-directory: .weft + run: rustup toolchain install + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 if: steps.download.outputs.found != 'yes' - id: cache with: - path: | - ~/.cargo/registry - ~/.cargo/git - .weft/target - key: weft-cli-${{ runner.os }}-${{ vars.WEFT_SOURCE_COMMIT }} - + workspaces: .weft -> target - name: build the weft CLI - id: build if: steps.download.outputs.found != 'yes' working-directory: .weft run: | - rustup toolchain install cargo build --release --locked -p weft-cli echo "$WEFT_REPO_ROOT/target/release" >> "$GITHUB_PATH" - - uses: actions/cache/save@v6 - if: always() && steps.build.outcome == 'success' && steps.cache.outputs.cache-hit != 'true' - with: - path: | - ~/.cargo/registry - ~/.cargo/git - .weft/target - key: weft-cli-${{ runner.os }}-${{ vars.WEFT_SOURCE_COMMIT }} - # A program that is off is switched on; one whose triggers are on is # brought up to date. `--mode park` holds the calls that arrive while # its triggers come down and back up, and runs them once they are diff --git a/crates/weft-core/src/net.rs b/crates/weft-core/src/net.rs index 969c9947..29e26be4 100644 --- a/crates/weft-core/src/net.rs +++ b/crates/weft-core/src/net.rs @@ -42,15 +42,16 @@ pub fn is_loopback_url(url: &str) -> bool { } /// Install ring as the process-level rustls crypto provider, once. -/// Called at every binary's startup (and the worker's process entry): the -/// dependency graph carries TWO providers (ring everywhere, aws-lc-rs -/// via the S3 stack), and rustls refuses to guess between them, so any -/// library that builds TLS from the process default (a WebSocket -/// client, a mail library) would panic at its first dial. Installing -/// the default up front means every present AND future dependency -/// builds TLS from a provider that is actually installed; weft's own -/// dialers additionally pin ring explicitly in -/// [`tls_config`]. Idempotent (a second install is a no-op). +/// Called at every binary's startup (and the worker's process entry). +/// ring is the only provider weft builds (the S3 stack is handed a ring +/// client too), but rustls refuses to guess the moment a dependency (a +/// node's library, a future crate) brings a second one, and any library +/// that builds TLS from the process default (a WebSocket client, a mail +/// library) would then panic at its first dial. Installing the default up +/// front means every present AND future dependency builds TLS from a +/// provider that is actually installed; weft's own dialers additionally +/// pin ring explicitly in [`tls_config`]. Idempotent (a second install is +/// a no-op). pub fn install_crypto_provider() { let _ = tokio_rustls::rustls::crypto::ring::default_provider().install_default(); } diff --git a/crates/weft-platform-gcp/src/accounts.rs b/crates/weft-platform-gcp/src/accounts.rs index d8b3540d..96b815ff 100644 --- a/crates/weft-platform-gcp/src/accounts.rs +++ b/crates/weft-platform-gcp/src/accounts.rs @@ -2,6 +2,8 @@ //! machines run as, so each can prove which project it is and nothing //! more. +use std::time::Duration; + use serde_json::{json, Value}; use weft_platform_traits::config::GcpPlatform; @@ -94,10 +96,23 @@ pub async fn revoke_project_account(google: &Google, gcp: &GcpPlatform, project: Ok(()) } -/// How many times a call is tried while a new account spreads through -/// IAM, and how long between tries (doubling). -const SPREAD_TRIES: u32 = 6; -const SPREAD_FIRST_PAUSE: std::time::Duration = std::time::Duration::from_secs(2); +/// How long a call waits for Google to apply an IAM change before its +/// refusal is taken as real: the pause between tries starts at `first` +/// and doubles up to `longest`, and the pauses add up to `budget` at most. +pub struct Patience { + first: Duration, + longest: Duration, + budget: Duration, +} + +/// A new account reaching every API: seconds, usually. +const ACCOUNT_SPREAD: Patience = + Patience { first: Duration::from_secs(2), longest: Duration::from_secs(32), budget: Duration::from_secs(62) }; + +/// A new grant taking effect. Google applies most within two minutes and +/// some take seven or more. +pub const GRANT_APPLIES: Patience = + Patience { first: Duration::from_secs(5), longest: Duration::from_secs(30), budget: Duration::from_secs(600) }; /// Whether `e` is Google refusing an account it has not heard of yet. A /// new account, or a new grant to it, takes a while to reach every API, @@ -133,17 +148,40 @@ where /// Run `call` until it stops failing on an account IAM has not spread yet /// or on what `also` names as settling on its own. One budget for both, /// so the waits never multiply. -pub async fn until_settled(also: impl Fn(&anyhow::Error) -> bool, mut call: F) -> anyhow::Result +pub async fn until_settled(also: impl Fn(&anyhow::Error) -> bool, call: F) -> anyhow::Result where F: FnMut() -> Fut, Fut: std::future::Future>, { - let mut pause = SPREAD_FIRST_PAUSE; - for _ in 1..SPREAD_TRIES { + until_settled_within(&ACCOUNT_SPREAD, also, call).await +} + +/// [`until_settled`] with the patience a change needs, for a change that +/// takes longer to apply than a new account does to spread (a new grant: +/// [`GRANT_APPLIES`]). Each wait is logged with the refusal that caused +/// it, so a call held up here says why. +pub async fn until_settled_within( + patience: &Patience, + also: impl Fn(&anyhow::Error) -> bool, + mut call: F, +) -> anyhow::Result +where + F: FnMut() -> Fut, + Fut: std::future::Future>, +{ + let (mut pause, mut waited) = (patience.first, Duration::ZERO); + while waited + pause <= patience.budget { match call().await { Err(e) if account_not_spread(&e) || also(&e) => { + tracing::info!( + target: "weft_platform_gcp::accounts", + error = format!("{e:#}"), + "Google has not applied a recent IAM change yet; asking again in {}s", + pause.as_secs() + ); tokio::time::sleep(pause).await; - pause *= 2; + waited += pause; + pause = (pause * 2).min(patience.longest); } done => return done, } @@ -251,6 +289,38 @@ mod tests { anyhow::Error::new(ApiError { status, body: body.into() }).context("POST x") } + /// A refusal `also` names is asked again until it stops, or until the + /// pauses would pass the budget, and then the last answer stands; any + /// other refusal is returned at once. + #[tokio::test] + async fn a_settling_refusal_is_asked_again_within_its_budget() { + let ms = Duration::from_millis; + let patience = Patience { first: ms(1), longest: ms(2), budget: ms(5) }; + let calls = std::sync::atomic::AtomicU32::new(0); + let refusing = |until: u32| { + let calls = &calls; + move || { + let n = calls.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1; + async move { if n < until { Err(api(403, "denied")) } else { Ok(n) } } + } + }; + let settles = |e: &anyhow::Error| is_status(e, 403); + + // Pauses of 1, 2 and 2 ms fit the 5 ms budget: three asks, then a + // fourth that answers. + assert_eq!(until_settled_within(&patience, settles, refusing(4)).await.unwrap(), 4); + + // Past the budget the fifth answer, still a refusal, stands. + calls.store(0, std::sync::atomic::Ordering::Relaxed); + assert!(is_status(&until_settled_within(&patience, settles, refusing(9)).await.unwrap_err(), 403)); + assert_eq!(calls.load(std::sync::atomic::Ordering::Relaxed), 4); + + // A refusal nobody said settles is final on the first ask. + calls.store(0, std::sync::atomic::Ordering::Relaxed); + assert!(until_settled_within(&patience, |_| false, refusing(9)).await.is_err()); + assert_eq!(calls.load(std::sync::atomic::Ordering::Relaxed), 1); + } + #[test] fn only_an_account_not_known_yet_is_waited_out() { assert!(account_not_spread(&api(400, "Service account wp-x@acme.iam.gserviceaccount.com does not exist."))); diff --git a/crates/weft-platform-gcp/src/storage.rs b/crates/weft-platform-gcp/src/storage.rs index 65623dd8..09374840 100644 --- a/crates/weft-platform-gcp/src/storage.rs +++ b/crates/weft-platform-gcp/src/storage.rs @@ -11,6 +11,7 @@ //! API, which has S3's shape; listing goes through its JSON API, which //! answers in JSON. Links are V4 signed URLs (`GOOG4-RSA-SHA256`). +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::SystemTime; use anyhow::{bail, Context as _, Result}; @@ -32,6 +33,12 @@ pub struct GcsObjectStore { /// The service account the process runs as: the one links are signed /// for. account: String, + /// Whether Google has signed for this process yet. Until it has, a + /// refusal may be the install's own grant to sign still taking effect + /// (a fresh install signs its first link, the standard library + /// preload, about a minute after granting it), so it is waited out; + /// after that, a refusal means the grant is gone. + signed: AtomicBool, } impl GcsObjectStore { @@ -39,7 +46,7 @@ impl GcsObjectStore { /// Fails at boot, naming the cause, when the bucket cannot be reached. pub async fn new(google: Google, bucket: String) -> Result { let account = google.tokens().account_email().await?; - let store = Self { google, bucket, account }; + let store = Self { google, bucket, account, signed: AtomicBool::new(false) }; let probe = store.request(Method::GET, &format!("https://{HOST}/storage/v1/b/{}", store.bucket), &[]).await; if let Err(e) = Self::ok("reach the object-store bucket", probe?).await { let advice = if is_status(&e, 403) { @@ -106,17 +113,18 @@ impl GcsObjectStore { ttl_secs, }; let (to_sign, query) = link.string_to_sign(); - let signed = self - .google - .post( - &format!("https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/{}:signBlob", self.account), - &serde_json::json!({ "payload": base64::engine::general_purpose::STANDARD.encode(to_sign.as_bytes()) }), - ) - .await - .map_err(|e| { - let advice = if is_status(&e, 403) { ": the account needs roles/iam.serviceAccountTokenCreator on itself" } else { "" }; - e.context(format!("sign a link as {}{advice}", self.account)) - })?; + let url = format!("https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/{}:signBlob", self.account); + let body = serde_json::json!({ "payload": base64::engine::general_purpose::STANDARD.encode(to_sign.as_bytes()) }); + let not_yet_granted = |e: &anyhow::Error| is_status(e, 403) && !self.signed.load(Ordering::Relaxed); + let signed = crate::accounts::until_settled_within(&crate::accounts::GRANT_APPLIES, not_yet_granted, || { + self.google.post(&url, &body) + }) + .await + .map_err(|e| { + let advice = if is_status(&e, 403) { ": the account needs roles/iam.serviceAccountTokenCreator on itself" } else { "" }; + e.context(format!("sign a link as {}{advice}", self.account)) + })?; + self.signed.store(true, Ordering::Relaxed); let blob = signed.get("signedBlob").and_then(|v| v.as_str()).context("signBlob answered no signedBlob")?; let signature = base64::engine::general_purpose::STANDARD.decode(blob).context("signBlob's signature is not base64")?; Ok(format!("https://{HOST}{}?{query}&X-Goog-Signature={}", link.path, hex(&signature))) diff --git a/crates/weft-platform-traits/Cargo.toml b/crates/weft-platform-traits/Cargo.toml index ba512881..e8a99e65 100644 --- a/crates/weft-platform-traits/Cargo.toml +++ b/crates/weft-platform-traits/Cargo.toml @@ -34,6 +34,7 @@ sha2 = { workspace = true } # via an endpoint-URL override, which is exactly the deploy-time slot. aws-config = { workspace = true } aws-sdk-s3 = { workspace = true } +aws-smithy-http-client = { workspace = true } aws-credential-types = { workspace = true } aws-smithy-types = { workspace = true } # Fake-only lock. parking_lot's Mutex never poisons and lints cleanly diff --git a/crates/weft-platform-traits/src/object_store.rs b/crates/weft-platform-traits/src/object_store.rs index 761c0922..38fbc5ed 100644 --- a/crates/weft-platform-traits/src/object_store.rs +++ b/crates/weft-platform-traits/src/object_store.rs @@ -324,7 +324,15 @@ impl S3ObjectStore { &cfg.secret_access_key, None, ); + // HTTPS on `ring`, like every other TLS client in weft (see the + // workspace Cargo.toml's `aws-smithy-http-client`). + let https = aws_smithy_http_client::Builder::new() + .tls_provider(aws_smithy_http_client::tls::Provider::Rustls( + aws_smithy_http_client::tls::rustls_provider::CryptoMode::Ring, + )) + .build_https(); let shared = aws_config::defaults(aws_config::BehaviorVersion::latest()) + .http_client(https) .region(aws_sdk_s3::config::Region::new(cfg.region.clone())) .credentials_provider(creds) // Set the retry posture EXPLICITLY rather than inheriting the SDK's diff --git a/deploy/docker/runtime.Dockerfile b/deploy/docker/runtime.Dockerfile index 5ba854a3..5c10cd95 100644 --- a/deploy/docker/runtime.Dockerfile +++ b/deploy/docker/runtime.Dockerfile @@ -6,10 +6,18 @@ # workflow for a cloud one (where it runs the machine, the serverless # roles and the agents). # -# The builder uses a plain base + rustup so the toolchain is read from -# `rust-toolchain.toml` (the single source of truth for the whole +# Where the `weft-runtime` binary comes from: `compiled` builds it here +# from the source in the context; `prebuilt` takes a binary built from +# the same source outside Docker, passed as the `weft-prebuilt` build +# context (`weft build-images --runtime-binary`), which is how the +# release workflow reuses the binary its CLI job already compiled. +# SYNC: RUNTIME_BINARY + the weft-prebuilt context <-> crates/weft-cli/src/images.rs (ensure_runtime_image) +ARG RUNTIME_BINARY=compiled + +# The compiled path uses a plain base + rustup so the toolchain is read +# from `rust-toolchain.toml` (the single source of truth for the whole # system), NOT baked into a `rust:X` image. -FROM debian:bookworm-slim AS builder +FROM debian:bookworm-slim AS compiled RUN apt-get update \ && apt-get install -y --no-install-recommends \ @@ -31,6 +39,13 @@ RUN --mount=type=cache,id=weft-cargo-registry,target=/root/.cargo/registry,shari cargo build --release -p weft-runtime --bin weft-runtime \ && cp /build/target/release/weft-runtime /usr/local/bin/ +# A downloaded artifact loses its executable bit, so the copy sets it. +FROM scratch AS prebuilt +COPY --from=weft-prebuilt --chmod=0755 weft-runtime /usr/local/bin/weft-runtime + +# Only the stage named here is built; the other is never touched. +FROM ${RUNTIME_BINARY} AS binary + # The Docker command line, for the host agent on a cloud machine: it runs # the machine's unit on the machine's own Docker through its socket. FROM debian:bookworm-slim AS docker-cli @@ -52,7 +67,7 @@ FROM debian:bookworm-slim RUN apt-get update \ && apt-get install -y --no-install-recommends ca-certificates \ && rm -rf /var/lib/apt/lists/* -COPY --from=builder /usr/local/bin/weft-runtime /usr/local/bin/weft-runtime +COPY --from=binary /usr/local/bin/weft-runtime /usr/local/bin/weft-runtime COPY --from=docker-cli /usr/local/bin/docker /usr/local/bin/docker # The weft source a project version is compiled against: the dispatcher # compiles every version itself and stages its worker build, so this diff --git a/docs/src/running/cloud.md b/docs/src/running/cloud.md index eb2d077a..938d3dee 100644 --- a/docs/src/running/cloud.md +++ b/docs/src/running/cloud.md @@ -136,9 +136,14 @@ in a repository secret named `WEFT_ACCESS_APPS`. Otherwise skip it. ### Run the workflow Open the Actions tab of your fork. GitHub turns workflows off in a new fork, so if the tab asks, enable them first. Then pick **install on GCP** and run it. It -takes weft's images from the release when your fork matches it (and -builds the ones it changed), pushes them to your project, and creates -everything listed under [what you get](#what-you-get). +takes weft's CLI and images from the release when your fork holds the same +source (and builds the ones it changed), pushes the images to your project, +and creates everything listed under [what you get](#what-you-get). If you +run it right after weft's main moved and your fork caught up, the release +for that source may still be building: the workflow waits for it rather +than compiling the same thing, and says so in its log. When it does have to +build, it keeps the compiled dependencies for the next run, so after an +upgrade only what changed in weft compiles again. When it finishes, its last step (in the run's summary, and in its log for `gh run view --log`) gives you the install's address, a Cloud Run @@ -412,7 +417,8 @@ Put those in the frontend's environment. `weft ci add` writes `.github/workflows/deploy.yml`, a workflow you run by hand from the Actions tab. It takes the weft CLI the release built from the -commit your install runs (or builds it, when the release has none), deploys +source your install runs (waiting for it while that release is still +building, or building it when the release has none), deploys the program (`weft activate --on prod` when it is off, `weft resync --on prod --mode park` when it is on: edit the mode in the file if `hibernate` or `wipe` suits the program better), then diff --git a/scripts/release-cli.sh b/scripts/release-cli.sh new file mode 100755 index 00000000..7beb4a23 --- /dev/null +++ b/scripts/release-cli.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# Put the weft CLI the release built from a checkout's exact source into +# a directory, for a CI job that would otherwise compile it (an install +# on GCP from a fork, a project's deploy workflow). +# +# The release names the source tree it was built from (manifest.json's +# `tree`), so a fork that merged weft into an unchanged copy matches too, +# under a commit of its own. When the release for that tree is still +# being built upstream (the usual case right after a merge into main), +# this waits for it, printing where it is, rather than compiling the +# same thing again. +# +# Usage: scripts/release-cli.sh +# Exit 0: /weft is the release's linux x86_64 CLI, checked +# against its sha256. +# Exit 2: no release was built from this source; build it instead. +# Anything else is an error. +# +# GH_TOKEN, when set, authenticates the GitHub API reads (a job's own +# token reads a public repository's runs); unauthenticated reads share a +# small hourly limit per address. +set -euo pipefail + +checkout="${1:-}" +dest="${2:-}" +if [ -z "$checkout" ] || [ -z "$dest" ]; then + echo "usage: $0 " >&2 + exit 1 +fi + +# SYNC: the release repository, tag, asset name and manifest.json keys <-> .github/workflows/release.yml (the release job), setup.sh (prebuilt artifacts block) +repo=WeaveMindAI/weft +base="https://github.com/$repo/releases/download/latest" +asset=weft-x86_64-linux +tree="$(git -C "$checkout" rev-parse 'HEAD^{tree}')" + +api() { + local auth=() + [ -n "${GH_TOKEN:-}" ] && auth=(-H "Authorization: Bearer $GH_TOKEN") + curl -fsSL --max-time 20 "${auth[@]}" -H "Accept: application/vnd.github+json" "https://api.github.com/$1" +} + +# Prints the manifest when it was built from this tree. +published_manifest() { + local manifest + manifest="$(curl -fsSL --max-time 10 "$base/manifest.json" 2>/dev/null)" || return 1 + [ "$(jq -r '.tree // empty' <<<"$manifest")" = "$tree" ] || return 1 + printf '%s' "$manifest" +} + +# Prints the url of a release run building this tree that has not +# finished yet, if any. GitHub not answering counts as none: the caller +# then builds, which is slower but just as right. +release_in_flight() { + local status runs + for status in in_progress queued; do + if ! runs="$(api "repos/$repo/actions/workflows/release.yml/runs?status=$status&per_page=20")"; then + echo "could not ask GitHub whether weft's release for this source is still running; not waiting for it" >&2 + return 0 + fi + jq -r --arg tree "$tree" \ + '[.workflow_runs[] | select(.head_commit.tree_id == $tree) | .html_url][0] // empty' <<<"$runs" \ + | grep . && return 0 + done + return 0 +} + +manifest="$(published_manifest)" || manifest="" +if [ -z "$manifest" ]; then + run="$(release_in_flight)" + if [ -z "$run" ]; then + echo "the release has no CLI built from this source ($tree)" + exit 2 + fi + echo "weft's release for this source is still being built ($run); waiting for it instead of compiling" + while [ -z "$manifest" ]; do + sleep 30 + manifest="$(published_manifest)" || manifest="" + [ -n "$manifest" ] && break + if [ -z "$(release_in_flight)" ]; then + # It may have published between the two reads. + manifest="$(published_manifest)" || manifest="" + if [ -z "$manifest" ]; then + echo "that release finished without publishing a CLI for this source" + exit 2 + fi + else + echo "still building ($run)" + fi + done +fi + +sha="$(jq -r --arg k "sha256_$asset" '.[$k] // empty' <<<"$manifest")" +if [ -z "$sha" ]; then + echo "the release for this source carries no $asset (its build failed)" + exit 2 +fi +mkdir -p "$dest" +curl -fsSL --retry 3 "$base/$asset" -o "$dest/weft" +# A release being replaced serves a binary its manifest no longer vouches +# for: that is no CLI for this source. +if ! echo "$sha $dest/weft" | sha256sum -c - >/dev/null; then + rm -f "$dest/weft" + echo "the published $asset no longer matches the manifest (a newer release is replacing it)" + exit 2 +fi +chmod +x "$dest/weft" +echo "using the CLI weft's release built from commit $(jq -r .commit <<<"$manifest")" diff --git a/setup.sh b/setup.sh index 4094646c..fdc83273 100755 --- a/setup.sh +++ b/setup.sh @@ -770,16 +770,18 @@ hint "installing from ${C_BOLD}${here}${C_RESET}" # # CI publishes a rolling release on every push to the release branch: # CLI binaries, the .vsix, and a manifest.json naming the exact commit -# they were built from plus a sha256 per asset. When this checkout IS -# that commit with a clean tree, downloading is equivalent to building, +# and source tree they were built from plus a sha256 per asset. When +# this checkout holds that source with no local change, downloading is +# equivalent to building, # so the install skips the compilers entirely (no Rust or Node -# toolchain needed). Any local change, a different commit, or no +# toolchain needed). Any local change, a different source tree, or no # network takes the build path instead, and every branch says so. # -# SYNC: release tag + asset names + manifest.json keys (commit, +# SYNC: release tag + asset names + manifest.json keys (commit, tree, # vscode_version, sha256_) <-> -# .github/workflows/release.yml (cli matrix `asset` values, the -# vsix `mv` target, the release job's manifest generation) +# .github/workflows/release.yml (the linux and macos matrix `asset` +# values, the vsix `mv` target, the release job's manifest +# generation, release-macos), scripts/release-cli.sh release_assets_url="https://github.com/WeaveMindAI/weft/releases/download/latest" prebuilt_dir="${HOME}/.local/share/weft/prebuilt" # THE home of the `weft` binary on PATH, whichever path produced it: @@ -840,12 +842,15 @@ decide_prebuilt_use() { fi prebuilt_commit="$(manifest_value "commit")" prebuilt_vscode_version="$(manifest_value "vscode_version")" - if [[ -z "${prebuilt_commit}" ]]; then - hint "the published manifest names no commit; building locally" + # The source tree, not the commit: a merge that brings in exactly the + # published source is another commit with the same tree. + prebuilt_tree="$(manifest_value "tree")" + if [[ -z "${prebuilt_tree}" ]]; then + hint "the published manifest names no source tree; building locally" return 0 fi - if [[ "${prebuilt_commit}" != "$(git -C "${here}" rev-parse HEAD)" ]]; then - hint "no prebuilt artifacts for this commit yet (latest published: ${prebuilt_commit:0:12}); building locally. If you just pulled, CI is likely still building; re-run in a while to download instead." + if [[ "${prebuilt_tree}" != "$(git -C "${here}" rev-parse 'HEAD^{tree}')" ]]; then + hint "no prebuilt artifacts for this source yet (latest published: commit ${prebuilt_commit:0:12}); building locally. If you just pulled, CI is likely still building; re-run in a while to download instead." return 0 fi if [[ $build_cli -eq 1 ]]; then @@ -861,7 +866,7 @@ decide_prebuilt_use() { if [[ -n "${cli_sha256}" && -n "${runtime_sha256}" ]]; then use_prebuilt_cli=1 else - hint "this commit's ${cli_asset} or ${runtime_asset} was not published (its build failed in CI); building the CLI and the runtime locally" + hint "this commit's ${cli_asset} or ${runtime_asset} is not published (still building in CI, or its build failed); building the CLI and the runtime locally. Re-run in a while to download instead." fi fi fi