diff --git a/.github/actions/build-binaries/action.yml b/.github/actions/build-binaries/action.yml new file mode 100644 index 00000000..347071c1 --- /dev/null +++ b/.github/actions/build-binaries/action.yml @@ -0,0 +1,41 @@ +# The CLI and the runtime it installs, built on this runner and uploaded +# as two artifacts, for release.yml's linux and macos jobs. A local +# install runs `weft-runtime` as a process next to `weft`. +name: build-binaries +description: build weft + weft-runtime in release mode and upload both as artifacts +inputs: + asset: + description: >- + The CLI's asset name (`weft--`); the runtime's is the + same with `weft-` turned into `weft-runtime-`. + required: true +runs: + using: composite + steps: + # rustup reads rust-toolchain.toml (the single source of the + # version and components; see ci.yml's rust job for why no + # third-party toolchain action). + - shell: bash + run: rustup toolchain install + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + # SYNC: the runtime asset name (weft- -> weft-runtime-) <-> setup.sh + # (prebuilt artifacts block: runtime_asset) + - shell: bash + run: cargo build --release --locked -p weft-cli -p weft-runtime + - shell: bash + env: + ASSET: ${{ inputs.asset }} + run: | + cp target/release/weft "$ASSET" + cp target/release/weft-runtime "weft-runtime-${ASSET#weft-}" + echo "RUNTIME_ASSET=weft-runtime-${ASSET#weft-}" >> "$GITHUB_ENV" + - uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.asset }} + path: ${{ inputs.asset }} + if-no-files-found: error + - uses: actions/upload-artifact@v4 + with: + name: ${{ env.RUNTIME_ASSET }} + path: ${{ env.RUNTIME_ASSET }} + if-no-files-found: error diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6002566..c60308d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,12 @@ jobs: # pinned to a commit it demands an explicit toolchain input, # which would fork the version out of the toml. - run: rustup toolchain install + # Restores everywhere, saves only on main (release.yml's call of + # this file): a PR reads main's copy and leaves nothing behind, and + # release.yml's prune-caches keeps one copy per job. - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + with: + save-if: ${{ github.ref == 'refs/heads/main' }} # No compile-time query macros anywhere, so the workspace suite needs no # database. The tests that do want one are behind the `db-tests` feature. @@ -99,7 +104,10 @@ jobs: - uses: actions/checkout@v4 # rustup reads rust-toolchain.toml (see the rust job's comment). - run: rustup toolchain install + # Saves only on main, like the rust job's. - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + with: + save-if: ${{ github.ref == 'refs/heads/main' }} # The script runs the suites under cargo-nextest (every test side by # side); its prebuilt binary, from the project's own install script. - run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin" diff --git a/.github/workflows/install-gcp.yml b/.github/workflows/install-gcp.yml index d9514324..16991e2c 100644 --- a/.github/workflows/install-gcp.yml +++ b/.github/workflows/install-gcp.yml @@ -30,7 +30,9 @@ concurrency: jobs: install: - runs-on: ubuntu-latest + # 22.04: a runtime built here goes into the runtime image, whose + # Debian bookworm has an older glibc than newer Ubuntus link against. + runs-on: ubuntu-22.04 # No deadline: a first install builds weft's images, and a run cut # mid-apply leaves the Terraform state locked. GitHub's own 6-hour job # limit is the only cap. @@ -89,41 +91,44 @@ jobs: gcloud pubsub topics describe cloud-builds --project "${{ vars.GCP_PROJECT_ID }}" >/dev/null 2>&1 \ || gcloud pubsub topics create cloud-builds --project "${{ vars.GCP_PROJECT_ID }}" - # The CLI the release published for this exact commit, checked - # against its sha256, when the fork sits on that commit (the same - # match setup.sh makes); otherwise built from this checkout. - # SYNC: release tag + asset name + manifest.json keys <-> .github/workflows/release.yml (the release job), setup.sh (prebuilt artifacts block), crates/weft-cli/templates/ci/gcp.yml (download the weft CLI) + # The CLI weft's release built from this exact source, waited for + # when that release is still being built; otherwise built here, + # with the runtime, from a cache that keeps the dependencies + # between runs, so only what changed in weft compiles again. - name: the weft CLI for this checkout + id: release + env: + GH_TOKEN: ${{ github.token }} run: | - base=https://github.com/WeaveMindAI/weft/releases/download/latest - manifest=$(curl -fsSL --max-time 10 "$base/manifest.json" || true) - commit=$(sed -n 's/.*"commit": *"\([^"]*\)".*/\1/p' <<<"$manifest") - sha=$(sed -n 's/.*"sha256_weft-x86_64-linux": *"\([^"]*\)".*/\1/p' <<<"$manifest") - mkdir -p target/release - # A release being replaced can serve a binary its manifest does - # not vouch for yet: the checksum then fails, and the CLI is built. - if [ "$commit" = "$GITHUB_SHA" ] && [ -n "$sha" ] \ - && curl -fsSL "$base/weft-x86_64-linux" -o target/release/weft \ - && echo "$sha target/release/weft" | sha256sum -c -; then - chmod +x target/release/weft - echo "using the CLI the release built from $commit" + if scripts/release-cli.sh . target/release; then + echo "found=yes" >> "$GITHUB_OUTPUT" else - echo "this commit has no published CLI that checks out; building it" - rm -f target/release/weft - rustup toolchain install - cargo build --release --locked -p weft-cli + rc=$? + [ "$rc" -eq 2 ] || exit "$rc" fi + - if: steps.release.outputs.found != 'yes' + run: rustup toolchain install + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + if: steps.release.outputs.found != 'yes' + - name: build the weft CLI and runtime + if: steps.release.outputs.found != 'yes' + run: cargo build --release --locked -p weft-cli -p weft-runtime # The runtime image and the worker builder base this checkout names, # pushed to the install's registry. Each is taken from the install's # registry if there, else copied from the release's (the same tag is # the same contents), else built. A fork carrying its own changes - # hashes to new tags and gets exactly its own runtime. + # hashes to new tags and gets exactly its own runtime, made from the + # `weft-runtime` built above instead of compiled again in Docker. - name: build the runtime's images + env: + BUILT_HERE: ${{ steps.release.outputs.found != 'yes' }} run: | registry=$(terraform -chdir=deploy/terraform/gcp output -raw registry) gcloud auth configure-docker "${{ vars.GCP_REGION }}-docker.pkg.dev" --quiet - WEFT_IMAGE_REGISTRY="$registry" ./target/release/weft build-images --push > "$RUNNER_TEMP/refs.txt" + runtime=() + [ "$BUILT_HERE" = true ] && runtime=(--runtime-binary target/release/weft-runtime) + WEFT_IMAGE_REGISTRY="$registry" ./target/release/weft build-images --push "${runtime[@]}" > "$RUNNER_TEMP/refs.txt" # SYNC: `weft build-images` stdout and its order <-> crates/weft-cli/src/images.rs (SharedImages::bare_refs), .github/workflows/release.yml (build + push shared images) echo "TF_VAR_runtime_image=$(sed -n 1p "$RUNNER_TEMP/refs.txt")" >> "$GITHUB_ENV" echo "TF_VAR_builder_base_image=$(sed -n 2p "$RUNNER_TEMP/refs.txt")" >> "$GITHUB_ENV" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 02529a96..e860845d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -61,16 +61,23 @@ jobs: if: github.repository == 'WeaveMindAI/weft' uses: ./.github/workflows/ci.yml - # ---- shared images -> ghcr ------------------------------------------ + # ---- Linux binaries, and the shared images -> ghcr ------------------ # - # One leg per architecture, each on its NATIVE runner (no qemu), each - # pushing `-`; the stitch job below then publishes the bare - # `` as a multi-arch manifest list, so an Apple-silicon / ARM - # machine pulls arm64 and an x86 machine pulls amd64 from the same - # content-addressed tag. The tag itself hashes source content only, so - # both legs compute the identical ref (the stitch job verifies it). - images: - name: runtime image + builder base (${{ matrix.arch }}) + # Each leg builds the CLI and the runtime for its architecture, then the + # shared images, on its NATIVE runner (no qemu), pushing `-`; + # images-manifest then publishes the bare `` as a multi-arch + # manifest list, so an Apple-silicon / ARM machine pulls arm64 and an + # x86 machine pulls amd64 from the same content-addressed tag. The tag + # hashes source content only, so both legs compute the identical ref + # (the stitch job verifies it). + # + # The runtime image takes the `weft-runtime` its leg just built, so the + # image carries the very binary the release ships and nothing compiles + # it a second time inside Docker. That works because these legs run on + # Ubuntu 22.04, whose glibc is older than the image's Debian bookworm + # (`weft build-images` starts the binary in the image to make sure). + linux: + name: ${{ matrix.asset }} + images needs: [ci] strategy: # One architecture failing must not kill the other's build @@ -79,8 +86,12 @@ jobs: fail-fast: false matrix: include: - - { runner: ubuntu-latest, arch: amd64 } - - { runner: ubuntu-24.04-arm, arch: arm64 } + # ubuntu-22.04: oldest supported runner glibc, so the binary + # runs on every distro at least that new, and in the image. + # SYNC: CLI asset names <-> setup.sh (prebuilt artifacts + # block: the uname case), the macos job's matrix + - { runner: ubuntu-22.04, asset: weft-x86_64-linux, arch: amd64 } + - { runner: ubuntu-22.04-arm, asset: weft-aarch64-linux, arch: arm64 } runs-on: ${{ matrix.runner }} permissions: contents: read @@ -92,11 +103,9 @@ jobs: # Android/.NET/GHC toolchains are dead weight here. - name: free disk space run: sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/.ghcup - # rustup reads rust-toolchain.toml (the single source of the - # version and components; see ci.yml's rust job for why no - # third-party toolchain action). - - run: rustup toolchain install - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + - uses: ./.github/actions/build-binaries + with: + asset: ${{ matrix.asset }} - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 with: registry: ghcr.io @@ -109,17 +118,48 @@ jobs: # the stitch job. # SYNC: `weft build-images` stdout <-> crates/weft-cli/src/images.rs (SharedImages::bare_refs), .github/workflows/install-gcp.yml (build the runtime's images) - name: build + push shared images - run: cargo run --locked -p weft-cli -- build-images --push-suffix "-${{ matrix.arch }}" > image-refs.txt + run: | + ./target/release/weft build-images \ + --push-suffix "-${{ matrix.arch }}" \ + --runtime-binary target/release/weft-runtime > image-refs.txt - uses: actions/upload-artifact@v4 with: name: image-refs-${{ matrix.arch }} path: image-refs.txt if-no-files-found: error + # ---- macOS binaries -------------------------------------------------- + # + # Their own job, so nothing waits on them: the latest release goes out + # with the Linux binaries and the images, and release-macos adds these + # once they are built. The Intel runner is the slowest machine in the + # whole release by far. + macos: + name: ${{ matrix.asset }} + needs: [ci] + strategy: + fail-fast: false + matrix: + include: + # macos-latest is arm64 (Apple silicon); macos-15-intel is + # the x86_64 runner GitHub keeps until August 2027. + # SYNC: CLI asset names <-> setup.sh (prebuilt artifacts + # block: the uname case), the linux job's matrix + - { runner: macos-latest, asset: weft-aarch64-macos } + - { runner: macos-15-intel, asset: weft-x86_64-macos } + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/build-binaries + with: + asset: ${{ matrix.asset }} + + # The bare `` becomes a multi-arch manifest list over the two + # halves the linux job pushed. images-manifest: name: publish multi-arch image tags runs-on: ubuntu-latest - needs: [images] + needs: [linux] permissions: contents: read packages: write @@ -171,56 +211,39 @@ jobs: || { echo "::error::$ref is not anonymously pullable; open github.com/orgs/WeaveMindAI/packages and flip this package to Public"; exit 1; } done < refs-amd64/image-refs.txt - # ---- CLI binaries ---------------------------------------------------- - cli: - name: cli (${{ matrix.asset }}) - needs: [ci] - strategy: - # One platform failing must not take the others' binaries down - # with it; the release job below ships whichever legs succeeded - # (a missing asset just means "build from source" there, and the - # manifest lists only the assets that exist). - fail-fast: false - matrix: - include: - # ubuntu-22.04: oldest supported runner glibc, so the binary - # runs on every distro at least that new. - # SYNC: CLI asset names <-> setup.sh (prebuilt artifacts - # block: the uname case) - - { runner: ubuntu-22.04, asset: weft-x86_64-linux } - - { runner: ubuntu-22.04-arm, asset: weft-aarch64-linux } - # macos-latest is arm64 (Apple silicon); macos-15-intel is - # the x86_64 runner GitHub keeps until August 2027. - - { runner: macos-latest, asset: weft-aarch64-macos } - - { runner: macos-15-intel, asset: weft-x86_64-macos } - runs-on: ${{ matrix.runner }} + # ---- build caches ---------------------------------------------------- + # + # Only main saves Rust build caches (ci.yml's jobs skip saving on a + # PR), and each save is keyed by the Cargo.lock and toolchain it was + # built with, so a dependency bump leaves the previous copy behind next + # to the new one. Once every job that saves one has finished, each + # cache keeps its newest copy and the rest are deleted, so the repo's + # cache space holds one warm copy per job and platform. A cache no job + # saves any more goes on its own after seven days unused. + prune-caches: + name: keep one copy of each build cache + needs: [ci, linux, macos] + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + permissions: + actions: write + env: + GH_TOKEN: ${{ github.token }} steps: - - uses: actions/checkout@v4 - # rustup reads rust-toolchain.toml (the single source of the - # version and components; see ci.yml's rust job for why no - # third-party toolchain action). - - run: rustup toolchain install - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 - # The CLI and the runtime it installs, side by side: a local - # install runs `weft-runtime` as a process next to `weft`. - # SYNC: the runtime asset name (weft- -> weft-runtime-) <-> setup.sh - # (prebuilt artifacts block: runtime_asset) - - run: cargo build --release --locked -p weft-cli -p weft-runtime - - run: | - cp target/release/weft "${{ matrix.asset }}" - runtime_asset="${{ matrix.asset }}" - cp target/release/weft-runtime "weft-runtime-${runtime_asset#weft-}" - echo "RUNTIME_ASSET=weft-runtime-${runtime_asset#weft-}" >> "$GITHUB_ENV" - - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.asset }} - path: ${{ matrix.asset }} - if-no-files-found: error - - uses: actions/upload-artifact@v4 - with: - name: ${{ env.RUNTIME_ASSET }} - path: ${{ env.RUNTIME_ASSET }} - if-no-files-found: error + # A rust-cache key ends in two 8-hex segments (the toolchain and + # environment, then the lockfile); what is left before them names + # the job and platform, the thing that should exist only once. + - name: drop superseded copies + run: | + set -euo pipefail + gh cache list --repo "$GITHUB_REPOSITORY" --ref refs/heads/main --key v0-rust- \ + --limit 1000 --json id,key,createdAt \ + | jq -r 'group_by(.key | sub("-[0-9a-f]{8}-[0-9a-f]{8}$"; "")) + | map(sort_by(.createdAt) | reverse | .[1:][]) | .[] | "\(.id) \(.key)"' \ + | while read -r id key; do + echo "dropping $key (a newer copy replaced it)" + gh cache delete "$id" --repo "$GITHUB_REPOSITORY" + done # ---- VS Code extension ---------------------------------------------- vsix: @@ -302,11 +325,12 @@ jobs: release: name: update latest release runs-on: ubuntu-latest - needs: [cli, vsix, browser, images-manifest] - # Runs even when a CLI leg or the browser packaging failed (one - # unbuildable artifact must not strand every other's assets), but - # never without the vsix job (it supplies the version the manifest - # needs) and never without the stitched images: manifest.json is + needs: [linux, vsix, browser, images-manifest] + # Runs even when the browser packaging failed (one unbuildable + # artifact must not strand every other's assets), but never without + # the vsix job (it supplies the version the manifest needs) and never + # without the stitched images (which also means both Linux binaries + # built): manifest.json is # the "prebuilts for this commit are ready" signal setup.sh keys # on, and publishing it while no image is pullable would hand a # toolchain-less user a prebuilt CLI whose first `daemon start` @@ -320,10 +344,12 @@ jobs: - uses: actions/checkout@v4 # Exactly the release payload, by name: the image-refs artifacts # (internal plumbing for the stitch job) must never leak into the - # public asset list. + # public asset list. The macOS binaries are release-macos's, even + # when one happens to be built already, so what this job publishes + # never depends on which runner was faster. - uses: actions/download-artifact@v4 with: - pattern: weft-* + pattern: weft-*-linux path: dist merge-multiple: true # A pattern, so a failed browser job just means fewer files (the @@ -360,10 +386,14 @@ jobs: # missing manifest reads as "nothing published". The # manifest goes LAST so its commit stamp never vouches for # assets that are not up yet. - # SYNC: manifest.json keys <-> setup.sh (prebuilt artifacts block: manifest_value), .github/workflows/install-gcp.yml (the weft CLI for this checkout), crates/weft-cli/templates/ci/gcp.yml (download the weft CLI) + # SYNC: manifest.json keys <-> setup.sh (prebuilt artifacts block: manifest_value), scripts/release-cli.sh, the release-macos job gh release upload latest dist/* --clobber + # `tree` names the source content: a fork that merged this + # commit into an unchanged copy has another commit but the same + # tree, and the same binaries are right for it. { printf '{\n "commit": "%s",\n' "${GITHUB_SHA}" + printf ' "tree": "%s",\n' "$(git rev-parse "${GITHUB_SHA}^{tree}")" printf ' "vscode_version": "%s",\n' "${VSCODE_VERSION}" for f in dist/*; do printf ' "sha256_%s": "%s",\n' "$(basename "$f")" "$(sha256sum "$f" | cut -d' ' -f1)" @@ -386,6 +416,50 @@ jobs: done <<< "$assets" gh release edit latest --notes "Built from ${GITHUB_SHA}. setup.sh downloads these when its checkout matches manifest.json's commit." + # The macOS binaries join the release once they are built, each with + # its sha added to the manifest the release job wrote. + release-macos: + name: add the macOS binaries to the latest release + runs-on: ubuntu-latest + needs: [release, macos] + # Whichever macOS legs succeeded, as long as the release they join + # went out. + if: ${{ !cancelled() && needs.release.result == 'success' }} + permissions: + contents: write + env: + GH_TOKEN: ${{ github.token }} + steps: + # A pattern, so a failed leg just means one file fewer. + - uses: actions/download-artifact@v4 + with: + pattern: weft-*-macos + path: dist + merge-multiple: true + # SYNC: manifest.json keys <-> the release job (assemble + upload) + - name: upload + add to the manifest + run: | + set -euo pipefail + shopt -s nullglob + files=(dist/*) + if [ ${#files[@]} -eq 0 ]; then + echo "::warning::no macOS binary was built; the release has none for this commit" + exit 0 + fi + gh release download latest --repo "$GITHUB_REPOSITORY" -p manifest.json -D . + # One release at a time (the workflow's concurrency group), so + # the manifest is this run's; anything else is a bug to see. + [ "$(jq -r .commit manifest.json)" = "$GITHUB_SHA" ] || { + echo "::error::the latest release names $(jq -r .commit manifest.json), not $GITHUB_SHA" + exit 1 + } + gh release upload latest "${files[@]}" --repo "$GITHUB_REPOSITORY" --clobber + for f in "${files[@]}"; do + jq --arg k "sha256_$(basename "$f")" --arg v "$(sha256sum "$f" | cut -d' ' -f1)" \ + '. + {($k): $v}' manifest.json > manifest.next && mv manifest.next manifest.json + done + gh release upload latest manifest.json --repo "$GITHUB_REPOSITORY" --clobber + # ---- store publishes (each store gated on its own version tag, its # own tag pushed immediately after its publish, so a failure or an # interrupted run in one store never blocks another's retry and a diff --git a/Cargo.lock b/Cargo.lock index ca1d5ab3..cad0d6c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -202,28 +202,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "aws-lc-rs" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" -dependencies = [ - "aws-lc-sys", - "zeroize", -] - -[[package]] -name = "aws-lc-sys" -version = "0.41.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" -dependencies = [ - "cc", - "cmake", - "dunce", - "fs_extra", -] - [[package]] name = "aws-runtime" version = "1.7.5" @@ -415,18 +393,12 @@ dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", - "h2 0.3.27", "h2 0.4.15", - "http 0.2.12", "http 1.4.0", - "http-body 0.4.6", - "hyper 0.14.32", "hyper 1.9.0", - "hyper-rustls 0.24.2", "hyper-rustls 0.27.9", "hyper-util", "pin-project-lite", - "rustls 0.21.12", "rustls 0.23.40", "rustls-native-certs 0.8.3", "rustls-pki-types", @@ -474,7 +446,6 @@ checksum = "b8e6f5caf6fea86f8c2206541ab5857cfcda9013426cdbe8fa0098b9e2d32182" dependencies = [ "aws-smithy-async", "aws-smithy-http", - "aws-smithy-http-client", "aws-smithy-observability", "aws-smithy-runtime-api", "aws-smithy-schema", @@ -745,8 +716,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" dependencies = [ "find-msvc-tools", - "jobserver", - "libc", "shlex 1.3.0", ] @@ -859,15 +828,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" -[[package]] -name = "cmake" -version = "0.1.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" -dependencies = [ - "cc", -] - [[package]] name = "cmov" version = "0.5.4" @@ -1166,12 +1126,6 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" -[[package]] -name = "dunce" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" - [[package]] name = "ecdsa" version = "0.16.9" @@ -1418,12 +1372,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fs_extra" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" - [[package]] name = "futures" version = "0.3.32" @@ -2134,16 +2082,6 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" -[[package]] -name = "jobserver" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" -dependencies = [ - "getrandom 0.3.4", - "libc", -] - [[package]] name = "js-sys" version = "0.3.97" @@ -3152,7 +3090,6 @@ version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ - "aws-lc-rs", "once_cell", "ring", "rustls-pki-types", @@ -3220,7 +3157,6 @@ version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ - "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -5104,6 +5040,7 @@ dependencies = [ "aws-config", "aws-credential-types", "aws-sdk-s3", + "aws-smithy-http-client", "aws-smithy-types", "bytes", "futures", diff --git a/Cargo.toml b/Cargo.toml index dfd0af13..7274e864 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,8 +92,13 @@ tokio-tungstenite = { version = "0.24", features = ["rustls-tls-native-roots"] } # S3-compatible object storage (the ObjectStore slot: GCS / SeaweedFS / AWS / # any S3 endpoint via an endpoint-URL override). rustls so we don't drag in # a native TLS toolchain, matching reqwest above. -aws-config = { version = "1", default-features = false, features = ["rustls", "rt-tokio", "behavior-version-latest"] } -aws-sdk-s3 = { version = "1", default-features = false, features = ["rustls", "rt-tokio", "sigv4a"] } +aws-config = { version = "1", default-features = false, features = ["rt-tokio", "behavior-version-latest"] } +aws-sdk-s3 = { version = "1", default-features = false, features = ["rt-tokio", "sigv4a"] } +# The AWS SDK's HTTPS client, on `ring` like every other TLS client here. +# The SDK's own `rustls` feature picks `aws-lc-rs`, a second crypto library +# that compiles a whole C library with cmake on every cold build; the client +# is handed to the SDK instead (`weft_platform_traits::object_store`). +aws-smithy-http-client = { version = "1", default-features = false, features = ["rustls-ring"] } aws-credential-types = { version = "1", features = ["hardcoded-credentials"] } aws-smithy-types = "1" diff --git a/crates/weft-broker/src/runtime_storage.rs b/crates/weft-broker/src/runtime_storage.rs index ca2751fc..ff008c28 100644 --- a/crates/weft-broker/src/runtime_storage.rs +++ b/crates/weft-broker/src/runtime_storage.rs @@ -144,7 +144,7 @@ pub fn router() -> Router> { // `Other`/anyhow chain carries internal detail (SQL text, driver messages, table // names) that must not be disclosed to attacker-controlled code. The typed 4xx // variants below are safe and actionable, so they keep their specific messages. -const INTERNAL_STORAGE_ERROR_BODY: &str = "internal storage error"; +const INTERNAL_STORAGE_ERROR_BODY: &str = "internal storage error (the broker's log names the cause)"; fn map_err(e: RuntimeStoreError) -> ApiError { let completing = matches!(e, RuntimeStoreError::Completing(_)); diff --git a/crates/weft-cli/src/commands/build.rs b/crates/weft-cli/src/commands/build.rs index 0d371e59..9953442f 100644 --- a/crates/weft-cli/src/commands/build.rs +++ b/crates/weft-cli/src/commands/build.rs @@ -54,8 +54,15 @@ pub async fn run(ctx: Ctx, node_set: weft_core::builds::NodeSet) -> Result<()> { /// one per line, for the workflow to capture; progress rides stderr. /// `--print` stops after resolving: the refs this tree's content hashes /// to, touching no image (setup.sh keys its engine-change sweep on the -/// builder-base line moving). -pub async fn run_build_images(push: bool, push_suffix: Option, print: bool) -> Result<()> { +/// builder-base line moving). `runtime_binary` is a `weft-runtime` built +/// outside Docker, copied into the runtime image instead of compiled +/// (`images::ensure_runtime_image`). +pub async fn run_build_images( + push: bool, + push_suffix: Option, + print: bool, + runtime_binary: Option<&std::path::Path>, +) -> Result<()> { if print { // The SAME list the ensure path prints (`bare_refs` carries // the stdout contract), never a second construction of it. @@ -72,6 +79,7 @@ pub async fn run_build_images(push: bool, push_suffix: Option, print: bo let shared = crate::images::ensure_all_shared_images( false, push_suffix.as_deref(), + runtime_binary, ) .await?; if push || push_suffix.is_some() { diff --git a/crates/weft-cli/src/commands/daemon.rs b/crates/weft-cli/src/commands/daemon.rs index 1cc0ce4f..e08b9a01 100644 --- a/crates/weft-cli/src/commands/daemon.rs +++ b/crates/weft-cli/src/commands/daemon.rs @@ -1267,7 +1267,7 @@ async fn start(install: &Install, rebuild: bool) -> Result<()> { refuse_an_older_install(install).await?; let ports = ports(install)?; - let shared = images::ensure_all_shared_images(rebuild, None).await?; + let shared = images::ensure_all_shared_images(rebuild, None, None).await?; images::hold_standard_worker(&install.id, &shared.worker).await?; ensure_network().await?; ensure_postgres(install, ports.postgres).await?; diff --git a/crates/weft-cli/src/images.rs b/crates/weft-cli/src/images.rs index f5895ee7..d4284797 100644 --- a/crates/weft-cli/src/images.rs +++ b/crates/weft-cli/src/images.rs @@ -260,7 +260,7 @@ pub async fn ensure_builder_base_at(image_ref: &str, rebuild: bool) -> Result<() // `build::stage_builder_base_context`. let ctx = weft_compiler::build::stage_builder_base_context(&root) .map_err(|e| anyhow::anyhow!("stage builder-base context: {e}"))?; - docker_build(image_ref, &ctx.join("Dockerfile"), &ctx, &[], &[]).await?; + docker_build(image_ref, &ctx.join("Dockerfile"), &ctx, &[], &[], &[]).await?; } } // Builder-base images are large (~1GB+: debian + rustup + @@ -329,11 +329,24 @@ pub fn runtime_image_ref() -> Result { Ok(qualified_ref(RUNTIME_REPO, &short_hex(&hasher.finalize()))) } +/// The build context a prebuilt runtime binary rides in, and the file +/// name it has there. +// SYNC: PREBUILT_CONTEXT + PREBUILT_BINARY + the RUNTIME_BINARY arg <-> deploy/docker/runtime.Dockerfile (the prebuilt stage) +const PREBUILT_CONTEXT: &str = "weft-prebuilt"; +const PREBUILT_BINARY: &str = "weft-runtime"; + /// Make the runtime image exist locally under `image_ref`: present -> /// done; pull -> done; build `deploy/docker/runtime.Dockerfile`. /// `rebuild` skips straight to the build, for when a present image is /// corrupt or hand-modified. -pub async fn ensure_runtime_image(image_ref: &str, rebuild: bool) -> Result<()> { +/// +/// With `prebuilt`, the build copies that `weft-runtime` binary in +/// instead of compiling one. It must be built from this same checkout: +/// the ref hashes the source, and nothing can check a binary against +/// it. Whether it RUNS in the image (this machine's architecture, a +/// glibc no newer than Debian bookworm's) is checked by starting it +/// there once, and an image it cannot run in is removed, never pushed. +pub async fn ensure_runtime_image(image_ref: &str, rebuild: bool, prebuilt: Option<&Path>) -> Result<()> { if !rebuild { if image_present(image_ref).await? { eprintln!("image {image_ref} present; skipping"); @@ -345,7 +358,40 @@ pub async fn ensure_runtime_image(image_ref: &str, rebuild: bool) -> Result<()> } let root = weft_compiler::build::resolve_weft_root().map_err(|e| anyhow::anyhow!("resolve weft repo root: {e}"))?; let dockerfile = root.join("deploy/docker").join(RUNTIME_DOCKERFILE); - docker_build(image_ref, &dockerfile, &root, &[], &[]).await + let Some(binary) = prebuilt else { + return docker_build(image_ref, &dockerfile, &root, &[], &[], &[]).await; + }; + // Its own context holding only the binary, under the name the + // Dockerfile copies, whatever the file was called on disk. + let staged = tempfile::tempdir()?; + std::fs::copy(binary, staged.path().join(PREBUILT_BINARY)) + .map_err(|e| anyhow::anyhow!("stage the prebuilt runtime {}: {e}", binary.display()))?; + docker_build( + image_ref, + &dockerfile, + &root, + &[], + &["RUNTIME_BINARY=prebuilt".to_string()], + &[(PREBUILT_CONTEXT, staged.path())], + ) + .await?; + let probe = docker().args(["run", "--rm", image_ref, "--version"]).output().await?; + if !probe.status.success() { + let removed = docker().args(["rmi", image_ref]).output().await?; + anyhow::ensure!( + removed.status.success(), + "remove {image_ref} after its runtime failed to start: {}", + String::from_utf8_lossy(&removed.stderr).trim() + ); + anyhow::bail!( + "the prebuilt runtime {} does not run in the image (removed {image_ref}): {}\n\ + build it for this machine's architecture against a glibc no newer than \ + Debian bookworm's (Ubuntu 22.04 is), or leave out --runtime-binary to compile it in the image", + binary.display(), + String::from_utf8_lossy(&probe.stderr).trim() + ); + } + Ok(()) } /// Every shared image one ensure materialized: the runtime, the builder @@ -387,16 +433,21 @@ pub fn suffixed_ref(bare: &str, ref_suffix: Option<&str>) -> String { /// content-addressed refs. With `ref_suffix`, each image is materialized /// under `` instead of its bare name: the release workflow's /// per-architecture halves, checked and pulled by their OWN names. +/// `prebuilt_runtime` is [`ensure_runtime_image`]'s `prebuilt`. /// /// `tokio::join!`, NOT `try_join!`: an early bail would drop the sibling /// future while its `docker build` child keeps running detached; join! /// lets both finish, then every error is reported together. -pub async fn ensure_all_shared_images(rebuild: bool, ref_suffix: Option<&str>) -> Result { +pub async fn ensure_all_shared_images( + rebuild: bool, + ref_suffix: Option<&str>, + prebuilt_runtime: Option<&Path>, +) -> Result { let runtime = runtime_image_ref()?; let base = builder_base_ref()?; let (runtime_target, base_target) = (suffixed_ref(&runtime, ref_suffix), suffixed_ref(&base, ref_suffix)); let (r, b) = tokio::join!( - ensure_runtime_image(&runtime_target, rebuild), + ensure_runtime_image(&runtime_target, rebuild, prebuilt_runtime), ensure_builder_base_at(&base_target, rebuild), ); let failures: Vec = [("runtime", r), ("builder-base", b)] @@ -558,12 +609,15 @@ fn idle_days(last_used: &str) -> u64 { /// builds (builder base, runtime, worker, infra, node-test) /// funnels here. `labels` stamp the `weft.dev/*` filters later GC /// selects on (empty for the shared images, which GC by repo+tag). +/// `named_contexts` are extra build contexts the Dockerfile reads with +/// `COPY --from=`. pub(crate) async fn docker_build( image_ref: &str, dockerfile: &Path, context: &Path, labels: &[String], build_args: &[String], + named_contexts: &[(&str, &Path)], ) -> Result<()> { eprintln!( "building image {image_ref} (this may take several minutes on first run; \ @@ -583,6 +637,9 @@ pub(crate) async fn docker_build( for arg in build_args { cmd.args(["--build-arg", arg]); } + for (name, dir) in named_contexts { + cmd.arg("--build-context").arg(format!("{name}={}", dir.display())); + } let status = cmd.arg(context).status().await?; if !status.success() { anyhow::bail!("docker build {image_ref} failed with {status}"); @@ -604,7 +661,7 @@ pub(crate) async fn docker_build_worker( ) -> Result<()> { let lane = CompileLane::hold().await?; let arg = format!("{}={}", weft_compiler::worker_image::COMPILE_LANE_ARG, lane.number); - docker_build(image_ref, dockerfile, context, labels, &[arg]).await + docker_build(image_ref, dockerfile, context, labels, &[arg], &[]).await } /// How many worker builds compile side by side when nobody says diff --git a/crates/weft-cli/src/main.rs b/crates/weft-cli/src/main.rs index 419445b2..e15cb2ba 100644 --- a/crates/weft-cli/src/main.rs +++ b/crates/weft-cli/src/main.rs @@ -111,8 +111,15 @@ enum Cmd { /// Only print the refs this tree resolves to, one per line, /// touching nothing (no ensure, no docker). What setup.sh /// keys its engine-change sweep on. - #[arg(long, conflicts_with_all = ["push", "push_suffix"])] + #[arg(long, conflicts_with_all = ["push", "push_suffix", "runtime_binary"])] print: bool, + /// Put this `weft-runtime` binary in the runtime image instead of + /// compiling one inside Docker. It must be built from this same + /// checkout, for this machine, against a glibc no newer than + /// Debian bookworm's. The release workflow passes the binary its + /// CLI job built, so the runtime compiles once per release. + #[arg(long, value_name = "PATH")] + runtime_binary: Option, }, /// Manage a node's service connection from the terminal: list the /// stored connections and pick one, connect a new account (paste a @@ -1687,8 +1694,8 @@ async fn run(cli: Cli) -> anyhow::Result<()> { Cmd::New { name, assistants, remember, ci } => commands::new::run(ctx, name, assistants, remember, ci).await, Cmd::Ci { action: CiCmd::Add { cloud } } => commands::ci::add(ctx, cloud).await, Cmd::Build { referenced } => commands::build::run(ctx, node_set(referenced)).await, - Cmd::BuildImages { push, push_suffix, print } => { - commands::build::run_build_images(push, push_suffix, print).await + Cmd::BuildImages { push, push_suffix, print, runtime_binary } => { + commands::build::run_build_images(push, push_suffix, print, runtime_binary.as_deref()).await } Cmd::Connect { opts } => commands::connect::run(ctx, opts).await, Cmd::Options { step, field, search } => { diff --git a/crates/weft-cli/templates/ci/gcp.yml b/crates/weft-cli/templates/ci/gcp.yml index 416d308e..3e8dbb7d 100644 --- a/crates/weft-cli/templates/ci/gcp.yml +++ b/crates/weft-cli/templates/ci/gcp.yml @@ -44,10 +44,10 @@ jobs: # The source of the exact weft the install runs (the CLI compiles the # project against its nodes), and its CLI: a project compiled by - # another weft is refused there. The release's own CLI when the - # release was built from that commit (checked against its sha256), - # otherwise built from the source and cached for the next run. - # SYNC: release tag + asset name + manifest.json keys <-> .github/workflows/release.yml (the release job), .github/workflows/install-gcp.yml (the weft CLI for this checkout) + # another weft is refused there. The CLI weft's release built from + # that source (waited for when the release is still being built), + # otherwise built here from a cache that keeps the dependencies + # between runs, so a newer weft compiles only what changed in it. - uses: actions/checkout@v7 with: repository: ${{ vars.WEFT_SOURCE_REPOSITORY }} @@ -56,55 +56,31 @@ jobs: - name: download the weft CLI id: download + env: + GH_TOKEN: ${{ github.token }} run: | - base=https://github.com/WeaveMindAI/weft/releases/download/latest - manifest=$(curl -fsSL --max-time 10 "$base/manifest.json" || true) - commit=$(sed -n 's/.*"commit": *"\([^"]*\)".*/\1/p' <<<"$manifest") - sha=$(sed -n 's/.*"sha256_weft-x86_64-linux": *"\([^"]*\)".*/\1/p' <<<"$manifest") - mkdir -p "$RUNNER_TEMP/weft" - # A release being replaced can serve a binary its manifest does - # not vouch for yet: the checksum then fails, and the CLI is built. - if [ "$commit" = "${{ vars.WEFT_SOURCE_COMMIT }}" ] && [ -n "$sha" ] \ - && curl -fsSL "$base/weft-x86_64-linux" -o "$RUNNER_TEMP/weft/weft" \ - && echo "$sha $RUNNER_TEMP/weft/weft" | sha256sum -c -; then - chmod +x "$RUNNER_TEMP/weft/weft" + if .weft/scripts/release-cli.sh .weft "$RUNNER_TEMP/weft"; then echo "$RUNNER_TEMP/weft" >> "$GITHUB_PATH" - echo "using the CLI the release built from $commit" echo "found=yes" >> "$GITHUB_OUTPUT" else - echo "the release has no CLI built from ${{ vars.WEFT_SOURCE_COMMIT }} that checks out; building it" + rc=$? + [ "$rc" -eq 2 ] || exit "$rc" fi - # Restored and saved as two steps, so a run that fails after the - # build still keeps it for the next one. - - uses: actions/cache/restore@v6 + - if: steps.download.outputs.found != 'yes' + working-directory: .weft + run: rustup toolchain install + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 if: steps.download.outputs.found != 'yes' - id: cache with: - path: | - ~/.cargo/registry - ~/.cargo/git - .weft/target - key: weft-cli-${{ runner.os }}-${{ vars.WEFT_SOURCE_COMMIT }} - + workspaces: .weft -> target - name: build the weft CLI - id: build if: steps.download.outputs.found != 'yes' working-directory: .weft run: | - rustup toolchain install cargo build --release --locked -p weft-cli echo "$WEFT_REPO_ROOT/target/release" >> "$GITHUB_PATH" - - uses: actions/cache/save@v6 - if: always() && steps.build.outcome == 'success' && steps.cache.outputs.cache-hit != 'true' - with: - path: | - ~/.cargo/registry - ~/.cargo/git - .weft/target - key: weft-cli-${{ runner.os }}-${{ vars.WEFT_SOURCE_COMMIT }} - # A program that is off is switched on; one whose triggers are on is # brought up to date. `--mode park` holds the calls that arrive while # its triggers come down and back up, and runs them once they are diff --git a/crates/weft-core/src/net.rs b/crates/weft-core/src/net.rs index 969c9947..29e26be4 100644 --- a/crates/weft-core/src/net.rs +++ b/crates/weft-core/src/net.rs @@ -42,15 +42,16 @@ pub fn is_loopback_url(url: &str) -> bool { } /// Install ring as the process-level rustls crypto provider, once. -/// Called at every binary's startup (and the worker's process entry): the -/// dependency graph carries TWO providers (ring everywhere, aws-lc-rs -/// via the S3 stack), and rustls refuses to guess between them, so any -/// library that builds TLS from the process default (a WebSocket -/// client, a mail library) would panic at its first dial. Installing -/// the default up front means every present AND future dependency -/// builds TLS from a provider that is actually installed; weft's own -/// dialers additionally pin ring explicitly in -/// [`tls_config`]. Idempotent (a second install is a no-op). +/// Called at every binary's startup (and the worker's process entry). +/// ring is the only provider weft builds (the S3 stack is handed a ring +/// client too), but rustls refuses to guess the moment a dependency (a +/// node's library, a future crate) brings a second one, and any library +/// that builds TLS from the process default (a WebSocket client, a mail +/// library) would then panic at its first dial. Installing the default up +/// front means every present AND future dependency builds TLS from a +/// provider that is actually installed; weft's own dialers additionally +/// pin ring explicitly in [`tls_config`]. Idempotent (a second install is +/// a no-op). pub fn install_crypto_provider() { let _ = tokio_rustls::rustls::crypto::ring::default_provider().install_default(); } diff --git a/crates/weft-platform-gcp/src/accounts.rs b/crates/weft-platform-gcp/src/accounts.rs index d8b3540d..96b815ff 100644 --- a/crates/weft-platform-gcp/src/accounts.rs +++ b/crates/weft-platform-gcp/src/accounts.rs @@ -2,6 +2,8 @@ //! machines run as, so each can prove which project it is and nothing //! more. +use std::time::Duration; + use serde_json::{json, Value}; use weft_platform_traits::config::GcpPlatform; @@ -94,10 +96,23 @@ pub async fn revoke_project_account(google: &Google, gcp: &GcpPlatform, project: Ok(()) } -/// How many times a call is tried while a new account spreads through -/// IAM, and how long between tries (doubling). -const SPREAD_TRIES: u32 = 6; -const SPREAD_FIRST_PAUSE: std::time::Duration = std::time::Duration::from_secs(2); +/// How long a call waits for Google to apply an IAM change before its +/// refusal is taken as real: the pause between tries starts at `first` +/// and doubles up to `longest`, and the pauses add up to `budget` at most. +pub struct Patience { + first: Duration, + longest: Duration, + budget: Duration, +} + +/// A new account reaching every API: seconds, usually. +const ACCOUNT_SPREAD: Patience = + Patience { first: Duration::from_secs(2), longest: Duration::from_secs(32), budget: Duration::from_secs(62) }; + +/// A new grant taking effect. Google applies most within two minutes and +/// some take seven or more. +pub const GRANT_APPLIES: Patience = + Patience { first: Duration::from_secs(5), longest: Duration::from_secs(30), budget: Duration::from_secs(600) }; /// Whether `e` is Google refusing an account it has not heard of yet. A /// new account, or a new grant to it, takes a while to reach every API, @@ -133,17 +148,40 @@ where /// Run `call` until it stops failing on an account IAM has not spread yet /// or on what `also` names as settling on its own. One budget for both, /// so the waits never multiply. -pub async fn until_settled(also: impl Fn(&anyhow::Error) -> bool, mut call: F) -> anyhow::Result +pub async fn until_settled(also: impl Fn(&anyhow::Error) -> bool, call: F) -> anyhow::Result where F: FnMut() -> Fut, Fut: std::future::Future>, { - let mut pause = SPREAD_FIRST_PAUSE; - for _ in 1..SPREAD_TRIES { + until_settled_within(&ACCOUNT_SPREAD, also, call).await +} + +/// [`until_settled`] with the patience a change needs, for a change that +/// takes longer to apply than a new account does to spread (a new grant: +/// [`GRANT_APPLIES`]). Each wait is logged with the refusal that caused +/// it, so a call held up here says why. +pub async fn until_settled_within( + patience: &Patience, + also: impl Fn(&anyhow::Error) -> bool, + mut call: F, +) -> anyhow::Result +where + F: FnMut() -> Fut, + Fut: std::future::Future>, +{ + let (mut pause, mut waited) = (patience.first, Duration::ZERO); + while waited + pause <= patience.budget { match call().await { Err(e) if account_not_spread(&e) || also(&e) => { + tracing::info!( + target: "weft_platform_gcp::accounts", + error = format!("{e:#}"), + "Google has not applied a recent IAM change yet; asking again in {}s", + pause.as_secs() + ); tokio::time::sleep(pause).await; - pause *= 2; + waited += pause; + pause = (pause * 2).min(patience.longest); } done => return done, } @@ -251,6 +289,38 @@ mod tests { anyhow::Error::new(ApiError { status, body: body.into() }).context("POST x") } + /// A refusal `also` names is asked again until it stops, or until the + /// pauses would pass the budget, and then the last answer stands; any + /// other refusal is returned at once. + #[tokio::test] + async fn a_settling_refusal_is_asked_again_within_its_budget() { + let ms = Duration::from_millis; + let patience = Patience { first: ms(1), longest: ms(2), budget: ms(5) }; + let calls = std::sync::atomic::AtomicU32::new(0); + let refusing = |until: u32| { + let calls = &calls; + move || { + let n = calls.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1; + async move { if n < until { Err(api(403, "denied")) } else { Ok(n) } } + } + }; + let settles = |e: &anyhow::Error| is_status(e, 403); + + // Pauses of 1, 2 and 2 ms fit the 5 ms budget: three asks, then a + // fourth that answers. + assert_eq!(until_settled_within(&patience, settles, refusing(4)).await.unwrap(), 4); + + // Past the budget the fifth answer, still a refusal, stands. + calls.store(0, std::sync::atomic::Ordering::Relaxed); + assert!(is_status(&until_settled_within(&patience, settles, refusing(9)).await.unwrap_err(), 403)); + assert_eq!(calls.load(std::sync::atomic::Ordering::Relaxed), 4); + + // A refusal nobody said settles is final on the first ask. + calls.store(0, std::sync::atomic::Ordering::Relaxed); + assert!(until_settled_within(&patience, |_| false, refusing(9)).await.is_err()); + assert_eq!(calls.load(std::sync::atomic::Ordering::Relaxed), 1); + } + #[test] fn only_an_account_not_known_yet_is_waited_out() { assert!(account_not_spread(&api(400, "Service account wp-x@acme.iam.gserviceaccount.com does not exist."))); diff --git a/crates/weft-platform-gcp/src/storage.rs b/crates/weft-platform-gcp/src/storage.rs index 65623dd8..09374840 100644 --- a/crates/weft-platform-gcp/src/storage.rs +++ b/crates/weft-platform-gcp/src/storage.rs @@ -11,6 +11,7 @@ //! API, which has S3's shape; listing goes through its JSON API, which //! answers in JSON. Links are V4 signed URLs (`GOOG4-RSA-SHA256`). +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::SystemTime; use anyhow::{bail, Context as _, Result}; @@ -32,6 +33,12 @@ pub struct GcsObjectStore { /// The service account the process runs as: the one links are signed /// for. account: String, + /// Whether Google has signed for this process yet. Until it has, a + /// refusal may be the install's own grant to sign still taking effect + /// (a fresh install signs its first link, the standard library + /// preload, about a minute after granting it), so it is waited out; + /// after that, a refusal means the grant is gone. + signed: AtomicBool, } impl GcsObjectStore { @@ -39,7 +46,7 @@ impl GcsObjectStore { /// Fails at boot, naming the cause, when the bucket cannot be reached. pub async fn new(google: Google, bucket: String) -> Result { let account = google.tokens().account_email().await?; - let store = Self { google, bucket, account }; + let store = Self { google, bucket, account, signed: AtomicBool::new(false) }; let probe = store.request(Method::GET, &format!("https://{HOST}/storage/v1/b/{}", store.bucket), &[]).await; if let Err(e) = Self::ok("reach the object-store bucket", probe?).await { let advice = if is_status(&e, 403) { @@ -106,17 +113,18 @@ impl GcsObjectStore { ttl_secs, }; let (to_sign, query) = link.string_to_sign(); - let signed = self - .google - .post( - &format!("https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/{}:signBlob", self.account), - &serde_json::json!({ "payload": base64::engine::general_purpose::STANDARD.encode(to_sign.as_bytes()) }), - ) - .await - .map_err(|e| { - let advice = if is_status(&e, 403) { ": the account needs roles/iam.serviceAccountTokenCreator on itself" } else { "" }; - e.context(format!("sign a link as {}{advice}", self.account)) - })?; + let url = format!("https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/{}:signBlob", self.account); + let body = serde_json::json!({ "payload": base64::engine::general_purpose::STANDARD.encode(to_sign.as_bytes()) }); + let not_yet_granted = |e: &anyhow::Error| is_status(e, 403) && !self.signed.load(Ordering::Relaxed); + let signed = crate::accounts::until_settled_within(&crate::accounts::GRANT_APPLIES, not_yet_granted, || { + self.google.post(&url, &body) + }) + .await + .map_err(|e| { + let advice = if is_status(&e, 403) { ": the account needs roles/iam.serviceAccountTokenCreator on itself" } else { "" }; + e.context(format!("sign a link as {}{advice}", self.account)) + })?; + self.signed.store(true, Ordering::Relaxed); let blob = signed.get("signedBlob").and_then(|v| v.as_str()).context("signBlob answered no signedBlob")?; let signature = base64::engine::general_purpose::STANDARD.decode(blob).context("signBlob's signature is not base64")?; Ok(format!("https://{HOST}{}?{query}&X-Goog-Signature={}", link.path, hex(&signature))) diff --git a/crates/weft-platform-traits/Cargo.toml b/crates/weft-platform-traits/Cargo.toml index ba512881..e8a99e65 100644 --- a/crates/weft-platform-traits/Cargo.toml +++ b/crates/weft-platform-traits/Cargo.toml @@ -34,6 +34,7 @@ sha2 = { workspace = true } # via an endpoint-URL override, which is exactly the deploy-time slot. aws-config = { workspace = true } aws-sdk-s3 = { workspace = true } +aws-smithy-http-client = { workspace = true } aws-credential-types = { workspace = true } aws-smithy-types = { workspace = true } # Fake-only lock. parking_lot's Mutex never poisons and lints cleanly diff --git a/crates/weft-platform-traits/src/object_store.rs b/crates/weft-platform-traits/src/object_store.rs index 761c0922..38fbc5ed 100644 --- a/crates/weft-platform-traits/src/object_store.rs +++ b/crates/weft-platform-traits/src/object_store.rs @@ -324,7 +324,15 @@ impl S3ObjectStore { &cfg.secret_access_key, None, ); + // HTTPS on `ring`, like every other TLS client in weft (see the + // workspace Cargo.toml's `aws-smithy-http-client`). + let https = aws_smithy_http_client::Builder::new() + .tls_provider(aws_smithy_http_client::tls::Provider::Rustls( + aws_smithy_http_client::tls::rustls_provider::CryptoMode::Ring, + )) + .build_https(); let shared = aws_config::defaults(aws_config::BehaviorVersion::latest()) + .http_client(https) .region(aws_sdk_s3::config::Region::new(cfg.region.clone())) .credentials_provider(creds) // Set the retry posture EXPLICITLY rather than inheriting the SDK's diff --git a/deploy/docker/runtime.Dockerfile b/deploy/docker/runtime.Dockerfile index 5ba854a3..5c10cd95 100644 --- a/deploy/docker/runtime.Dockerfile +++ b/deploy/docker/runtime.Dockerfile @@ -6,10 +6,18 @@ # workflow for a cloud one (where it runs the machine, the serverless # roles and the agents). # -# The builder uses a plain base + rustup so the toolchain is read from -# `rust-toolchain.toml` (the single source of truth for the whole +# Where the `weft-runtime` binary comes from: `compiled` builds it here +# from the source in the context; `prebuilt` takes a binary built from +# the same source outside Docker, passed as the `weft-prebuilt` build +# context (`weft build-images --runtime-binary`), which is how the +# release workflow reuses the binary its CLI job already compiled. +# SYNC: RUNTIME_BINARY + the weft-prebuilt context <-> crates/weft-cli/src/images.rs (ensure_runtime_image) +ARG RUNTIME_BINARY=compiled + +# The compiled path uses a plain base + rustup so the toolchain is read +# from `rust-toolchain.toml` (the single source of truth for the whole # system), NOT baked into a `rust:X` image. -FROM debian:bookworm-slim AS builder +FROM debian:bookworm-slim AS compiled RUN apt-get update \ && apt-get install -y --no-install-recommends \ @@ -31,6 +39,13 @@ RUN --mount=type=cache,id=weft-cargo-registry,target=/root/.cargo/registry,shari cargo build --release -p weft-runtime --bin weft-runtime \ && cp /build/target/release/weft-runtime /usr/local/bin/ +# A downloaded artifact loses its executable bit, so the copy sets it. +FROM scratch AS prebuilt +COPY --from=weft-prebuilt --chmod=0755 weft-runtime /usr/local/bin/weft-runtime + +# Only the stage named here is built; the other is never touched. +FROM ${RUNTIME_BINARY} AS binary + # The Docker command line, for the host agent on a cloud machine: it runs # the machine's unit on the machine's own Docker through its socket. FROM debian:bookworm-slim AS docker-cli @@ -52,7 +67,7 @@ FROM debian:bookworm-slim RUN apt-get update \ && apt-get install -y --no-install-recommends ca-certificates \ && rm -rf /var/lib/apt/lists/* -COPY --from=builder /usr/local/bin/weft-runtime /usr/local/bin/weft-runtime +COPY --from=binary /usr/local/bin/weft-runtime /usr/local/bin/weft-runtime COPY --from=docker-cli /usr/local/bin/docker /usr/local/bin/docker # The weft source a project version is compiled against: the dispatcher # compiles every version itself and stages its worker build, so this diff --git a/docs/src/running/cloud.md b/docs/src/running/cloud.md index eb2d077a..938d3dee 100644 --- a/docs/src/running/cloud.md +++ b/docs/src/running/cloud.md @@ -136,9 +136,14 @@ in a repository secret named `WEFT_ACCESS_APPS`. Otherwise skip it. ### Run the workflow Open the Actions tab of your fork. GitHub turns workflows off in a new fork, so if the tab asks, enable them first. Then pick **install on GCP** and run it. It -takes weft's images from the release when your fork matches it (and -builds the ones it changed), pushes them to your project, and creates -everything listed under [what you get](#what-you-get). +takes weft's CLI and images from the release when your fork holds the same +source (and builds the ones it changed), pushes the images to your project, +and creates everything listed under [what you get](#what-you-get). If you +run it right after weft's main moved and your fork caught up, the release +for that source may still be building: the workflow waits for it rather +than compiling the same thing, and says so in its log. When it does have to +build, it keeps the compiled dependencies for the next run, so after an +upgrade only what changed in weft compiles again. When it finishes, its last step (in the run's summary, and in its log for `gh run view --log`) gives you the install's address, a Cloud Run @@ -412,7 +417,8 @@ Put those in the frontend's environment. `weft ci add` writes `.github/workflows/deploy.yml`, a workflow you run by hand from the Actions tab. It takes the weft CLI the release built from the -commit your install runs (or builds it, when the release has none), deploys +source your install runs (waiting for it while that release is still +building, or building it when the release has none), deploys the program (`weft activate --on prod` when it is off, `weft resync --on prod --mode park` when it is on: edit the mode in the file if `hibernate` or `wipe` suits the program better), then diff --git a/scripts/release-cli.sh b/scripts/release-cli.sh new file mode 100755 index 00000000..7beb4a23 --- /dev/null +++ b/scripts/release-cli.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# Put the weft CLI the release built from a checkout's exact source into +# a directory, for a CI job that would otherwise compile it (an install +# on GCP from a fork, a project's deploy workflow). +# +# The release names the source tree it was built from (manifest.json's +# `tree`), so a fork that merged weft into an unchanged copy matches too, +# under a commit of its own. When the release for that tree is still +# being built upstream (the usual case right after a merge into main), +# this waits for it, printing where it is, rather than compiling the +# same thing again. +# +# Usage: scripts/release-cli.sh +# Exit 0: /weft is the release's linux x86_64 CLI, checked +# against its sha256. +# Exit 2: no release was built from this source; build it instead. +# Anything else is an error. +# +# GH_TOKEN, when set, authenticates the GitHub API reads (a job's own +# token reads a public repository's runs); unauthenticated reads share a +# small hourly limit per address. +set -euo pipefail + +checkout="${1:-}" +dest="${2:-}" +if [ -z "$checkout" ] || [ -z "$dest" ]; then + echo "usage: $0 " >&2 + exit 1 +fi + +# SYNC: the release repository, tag, asset name and manifest.json keys <-> .github/workflows/release.yml (the release job), setup.sh (prebuilt artifacts block) +repo=WeaveMindAI/weft +base="https://github.com/$repo/releases/download/latest" +asset=weft-x86_64-linux +tree="$(git -C "$checkout" rev-parse 'HEAD^{tree}')" + +api() { + local auth=() + [ -n "${GH_TOKEN:-}" ] && auth=(-H "Authorization: Bearer $GH_TOKEN") + curl -fsSL --max-time 20 "${auth[@]}" -H "Accept: application/vnd.github+json" "https://api.github.com/$1" +} + +# Prints the manifest when it was built from this tree. +published_manifest() { + local manifest + manifest="$(curl -fsSL --max-time 10 "$base/manifest.json" 2>/dev/null)" || return 1 + [ "$(jq -r '.tree // empty' <<<"$manifest")" = "$tree" ] || return 1 + printf '%s' "$manifest" +} + +# Prints the url of a release run building this tree that has not +# finished yet, if any. GitHub not answering counts as none: the caller +# then builds, which is slower but just as right. +release_in_flight() { + local status runs + for status in in_progress queued; do + if ! runs="$(api "repos/$repo/actions/workflows/release.yml/runs?status=$status&per_page=20")"; then + echo "could not ask GitHub whether weft's release for this source is still running; not waiting for it" >&2 + return 0 + fi + jq -r --arg tree "$tree" \ + '[.workflow_runs[] | select(.head_commit.tree_id == $tree) | .html_url][0] // empty' <<<"$runs" \ + | grep . && return 0 + done + return 0 +} + +manifest="$(published_manifest)" || manifest="" +if [ -z "$manifest" ]; then + run="$(release_in_flight)" + if [ -z "$run" ]; then + echo "the release has no CLI built from this source ($tree)" + exit 2 + fi + echo "weft's release for this source is still being built ($run); waiting for it instead of compiling" + while [ -z "$manifest" ]; do + sleep 30 + manifest="$(published_manifest)" || manifest="" + [ -n "$manifest" ] && break + if [ -z "$(release_in_flight)" ]; then + # It may have published between the two reads. + manifest="$(published_manifest)" || manifest="" + if [ -z "$manifest" ]; then + echo "that release finished without publishing a CLI for this source" + exit 2 + fi + else + echo "still building ($run)" + fi + done +fi + +sha="$(jq -r --arg k "sha256_$asset" '.[$k] // empty' <<<"$manifest")" +if [ -z "$sha" ]; then + echo "the release for this source carries no $asset (its build failed)" + exit 2 +fi +mkdir -p "$dest" +curl -fsSL --retry 3 "$base/$asset" -o "$dest/weft" +# A release being replaced serves a binary its manifest no longer vouches +# for: that is no CLI for this source. +if ! echo "$sha $dest/weft" | sha256sum -c - >/dev/null; then + rm -f "$dest/weft" + echo "the published $asset no longer matches the manifest (a newer release is replacing it)" + exit 2 +fi +chmod +x "$dest/weft" +echo "using the CLI weft's release built from commit $(jq -r .commit <<<"$manifest")" diff --git a/setup.sh b/setup.sh index 4094646c..fdc83273 100755 --- a/setup.sh +++ b/setup.sh @@ -770,16 +770,18 @@ hint "installing from ${C_BOLD}${here}${C_RESET}" # # CI publishes a rolling release on every push to the release branch: # CLI binaries, the .vsix, and a manifest.json naming the exact commit -# they were built from plus a sha256 per asset. When this checkout IS -# that commit with a clean tree, downloading is equivalent to building, +# and source tree they were built from plus a sha256 per asset. When +# this checkout holds that source with no local change, downloading is +# equivalent to building, # so the install skips the compilers entirely (no Rust or Node -# toolchain needed). Any local change, a different commit, or no +# toolchain needed). Any local change, a different source tree, or no # network takes the build path instead, and every branch says so. # -# SYNC: release tag + asset names + manifest.json keys (commit, +# SYNC: release tag + asset names + manifest.json keys (commit, tree, # vscode_version, sha256_) <-> -# .github/workflows/release.yml (cli matrix `asset` values, the -# vsix `mv` target, the release job's manifest generation) +# .github/workflows/release.yml (the linux and macos matrix `asset` +# values, the vsix `mv` target, the release job's manifest +# generation, release-macos), scripts/release-cli.sh release_assets_url="https://github.com/WeaveMindAI/weft/releases/download/latest" prebuilt_dir="${HOME}/.local/share/weft/prebuilt" # THE home of the `weft` binary on PATH, whichever path produced it: @@ -840,12 +842,15 @@ decide_prebuilt_use() { fi prebuilt_commit="$(manifest_value "commit")" prebuilt_vscode_version="$(manifest_value "vscode_version")" - if [[ -z "${prebuilt_commit}" ]]; then - hint "the published manifest names no commit; building locally" + # The source tree, not the commit: a merge that brings in exactly the + # published source is another commit with the same tree. + prebuilt_tree="$(manifest_value "tree")" + if [[ -z "${prebuilt_tree}" ]]; then + hint "the published manifest names no source tree; building locally" return 0 fi - if [[ "${prebuilt_commit}" != "$(git -C "${here}" rev-parse HEAD)" ]]; then - hint "no prebuilt artifacts for this commit yet (latest published: ${prebuilt_commit:0:12}); building locally. If you just pulled, CI is likely still building; re-run in a while to download instead." + if [[ "${prebuilt_tree}" != "$(git -C "${here}" rev-parse 'HEAD^{tree}')" ]]; then + hint "no prebuilt artifacts for this source yet (latest published: commit ${prebuilt_commit:0:12}); building locally. If you just pulled, CI is likely still building; re-run in a while to download instead." return 0 fi if [[ $build_cli -eq 1 ]]; then @@ -861,7 +866,7 @@ decide_prebuilt_use() { if [[ -n "${cli_sha256}" && -n "${runtime_sha256}" ]]; then use_prebuilt_cli=1 else - hint "this commit's ${cli_asset} or ${runtime_asset} was not published (its build failed in CI); building the CLI and the runtime locally" + hint "this commit's ${cli_asset} or ${runtime_asset} is not published (still building in CI, or its build failed); building the CLI and the runtime locally. Re-run in a while to download instead." fi fi fi