From 4b5bfc467150aa2ee38a04b690962408c17ff501 Mon Sep 17 00:00:00 2001 From: Jeremy Howard Date: Sun, 23 Aug 2026 07:03:53 +1000 Subject: [PATCH] Add cross-platform source build CI --- .github/workflows/build.yml | 38 +++++++++++++++++++++++++ DEV.md | 33 +++++++++++----------- tools/setup_source_deps.sh | 55 +++++++++++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100755 tools/setup_source_deps.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..5e4f783 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,38 @@ +name: Build + +on: + workflow_dispatch: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +concurrency: + group: build-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + strategy: + fail-fast: false + matrix: + include: + - name: linux-x86_64 + runner: ubicloud-standard-8 + - name: linux-arm64 + runner: ubicloud-standard-8-arm + - name: macos-arm64 + runner: macos-26 + name: ${{ matrix.name }} + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v7 + - name: Install Linux build tools + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y cmake ninja-build pkg-config + - name: Set up source dependencies + run: tools/setup_source_deps.sh + - name: Build xmojo + run: ./bazelw build --config=public-cache @xmojo//:xmojo diff --git a/DEV.md b/DEV.md index b07e9e0..f478279 100644 --- a/DEV.md +++ b/DEV.md @@ -75,23 +75,24 @@ Published wheels remain macOS-only. The experimental IREE runtime story is currently Metal-only; the interactive engine and compiler-only SPIR-V story build and run on both operating systems. -Clone xmojo and its editable dependencies into one directory: +Clone xmojo, then let its setup helper create the pinned sibling worktrees and +enable LLVM's SPIR-V target in Modular: ```bash git clone --depth 1 https://github.com/AnswerDotAI/xmojo.git -git clone --depth 1 https://github.com/modular/modular.git -git clone --depth 1 https://github.com/iree-org/iree.git -git clone --depth 1 https://github.com/nlohmann/json.git nlohmann-json -git clone --depth 1 https://github.com/jupyter-xeus/xeus.git -git clone --depth 1 https://github.com/jupyter-xeus/xeus-zmq.git -git clone --depth 1 https://github.com/zeromq/libzmq.git -git clone --depth 1 https://github.com/zeromq/cppzmq.git -git -C iree submodule update --init --depth 1 third_party/flatcc +cd xmojo +tools/setup_source_deps.sh ``` -For a reproducible release build, check out the revisions in -`bazel/versions.bzl` and the IREE revision in `docs/modular-gpu-pipeline.md`. -Normal development intentionally uses the current local worktrees. +The helper clones only missing dependencies and leaves existing worktrees +untouched, so normal development can continue to use current local revisions. +The experimental IREE runtime additionally needs the revision recorded in +`docs/modular-gpu-pipeline.md` and its FlatCC submodule: + +```bash +git clone --depth 1 https://github.com/iree-org/iree.git ../iree +git -C ../iree submodule update --init --depth 1 third_party/flatcc +``` The open Modular build enables only its default LLVM targets. xmojo's SPIR-V backend needs one local root-module configuration line. Add it between the @@ -103,10 +104,10 @@ llvm_configure.configure(extra_targets = ["SPIRV"]) use_repo(llvm_configure, "llvm-project") ``` -This is the only local Modular edit. It selects code already present in -Modular's pinned LLVM checkout; xmojo contains the target implementation. -Keeping it in the root module means every xmojo build uses the same LLVM -configuration and Bazel cache key. +This is the only local Modular edit, and `tools/setup_source_deps.sh` applies it +idempotently. It selects code already present in Modular's pinned LLVM checkout; +xmojo contains the target implementation. Keeping it in the root module means +every xmojo build uses the same LLVM configuration and Bazel cache key. Set up the editable Python project and its native command with: diff --git a/tools/setup_source_deps.sh b/tools/setup_source_deps.sh new file mode 100755 index 0000000..f9b068e --- /dev/null +++ b/tools/setup_source_deps.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +deps_root="${XMOJO_DEPS_ROOT:-$(dirname "$script_root")}" +source "$script_root/bazel/versions.bzl" + +clone_revision() { + local name="$1" url="$2" revision="$3" destination="$deps_root/$1" + if [[ -e "$destination" ]]; then + if [[ ! -d "$destination/.git" ]]; then + echo "$destination exists but is not a git worktree" >&2 + exit 2 + fi + echo "Using existing $destination" + return + fi + + echo "Cloning $name at $revision" + local temporary + temporary="$(mktemp -d "$deps_root/.xmojo-$name.XXXXXX")" + if git init -q "$temporary" && + git -C "$temporary" remote add origin "$url" && + git -C "$temporary" fetch -q --depth 1 origin "$revision" && + git -C "$temporary" checkout -q --detach FETCH_HEAD; then + mv "$temporary" "$destination" + else + rm -rf "$temporary" + return 1 + fi +} + +mkdir -p "$deps_root" +clone_revision modular https://github.com/modular/modular.git "$MODULAR_REVISION" +clone_revision nlohmann-json https://github.com/nlohmann/json.git "$NLOHMANN_JSON_REVISION" +clone_revision xeus https://github.com/jupyter-xeus/xeus.git "$XEUS_REVISION" +clone_revision xeus-zmq https://github.com/jupyter-xeus/xeus-zmq.git "$XEUS_ZMQ_REVISION" +clone_revision libzmq https://github.com/zeromq/libzmq.git "$LIBZMQ_REVISION" +clone_revision cppzmq https://github.com/zeromq/cppzmq.git "$CPPZMQ_REVISION" + +module_file="$deps_root/modular/MODULE.bazel" +spirv_config='llvm_configure.configure(extra_targets = ["SPIRV"])' +if ! grep -Fqx "$spirv_config" "$module_file"; then + marker='use_repo(llvm_configure, "llvm-project")' + if [[ "$(grep -Fxc "$marker" "$module_file")" -ne 1 ]]; then + echo "Could not locate the LLVM repository declaration in $module_file" >&2 + exit 2 + fi + temporary="$module_file.xmojo" + awk -v marker="$marker" -v config="$spirv_config" \ + '$0 == marker { print config } { print }' "$module_file" > "$temporary" + mv "$temporary" "$module_file" + echo "Enabled LLVM SPIR-V in $module_file" +fi