rerun-sdk: add build-rerun-sdk.yml for riscv64 wheels

.github/workflows/build-rerun-sdk.yml

@@ -0,0 +1,250 @@
+# SPDX-FileCopyrightText: 2026 The RISE Project
+# SPDX-License-Identifier: MIT
+---
+# Based on the `linux-x64` leg of upstream's own wheel pipeline:
+# https://github.com/rerun-io/rerun/blob/0.37.1/.github/workflows/reusable_build_and_upload_wheels.yml
+# (MODE=pr: `--no-default-features --features perf_telemetry,extension-module`,
+# i.e. the crate's own default features -- no --features flags needed).
+#
+# Scope: this builds only the `rerun_bindings` Python SDK extension (rerun_py/),
+# not the `rerun-cli` native viewer. Upstream's published wheel also bundles a
+# prebuilt `rerun-cli` binary (a full wgpu/egui desktop+web viewer, built by a
+# separate ~16-core job that itself needs a wasm/JS toolchain for the bundled
+# web viewer) at rerun_sdk/rerun_cli/rerun; building that from source is out of
+# scope here. rerun_py's own build.rs fails without it unless
+# RERUN_ALLOW_MISSING_BIN is set (see below), and upstream's own
+# `[tool.maturin] include` comment already documents that a missing binary is
+# "not a packaging failure" -- `import rerun` and all logging/recording/gRPC
+# APIs work; only `rerun.spawn()`/`serve()` and the `rerun` console script
+# (which shell out to the bundled binary) raise a clear error instead of
+# launching a viewer.
+name: Build rerun-sdk wheels (riscv64)
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'rerun-sdk version to build (git tag, e.g. 0.37.1)'
+        required: true
+        default: '0.37.1'
+  pull_request:
+    paths:
+      - '.github/workflows/build-rerun-sdk.yml'
+      - 'patches/rerun-sdk/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ inputs.version || '0.37.1' }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read  # to fetch code (actions/checkout)
+
+env:
+  # `inputs.version` is empty on pull_request events; default to 0.37.1 there.
+  RERUN_SDK_VERSION: ${{ inputs.version || '0.37.1' }}
+  MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64
+
+jobs:
+  setup:
+    uses: $/.github/workflows/_setup.yml
+
+  build_wheels:
+    needs: [setup]
+    # pyo3 carries `abi3-py310` unconditionally in rerun_py/Cargo.toml, so
+    # upstream ships one cp310-abi3 wheel. Our registry has no riscv64 pyarrow
+    # (a hard runtime dependency) for cp310/cp311, so the build floor here is
+    # cp312 instead -- the wheel is still tagged cp310-abi3 by the pyo3
+    # feature regardless of which interpreter compiles it (CLAUDE.md gotcha
+    # 96), it is simply never tested below cp312.
+    name: Build rerun-sdk ${{ inputs.version || '0.37.1' }} cp310-abi3-manylinux_riscv64
+    runs-on: ubuntu-24.04-riscv
+    timeout-minutes: 1440
+
+    steps:
+      # rerun_py is a member of the rerun Cargo workspace (~760 crates on
+      # riscv64: datafusion, lance, tonic/tokio, hdf5-pure, ...) and builds
+      # against its sibling crates, so the checkout root has to be the
+      # workspace root, not rerun_py itself.
+      # lfs: true is required: the test suite's binary fixtures (HDF5, MP4,
+      # ...) are Git LFS pointers without it, and get fed straight into their
+      # native parsers -- "HDF5 signature not found" / "MP4 ... box with a
+      # larger size than it" is what that looks like from the Python side.
+      - name: Checkout rerun ${{ env.RERUN_SDK_VERSION }}
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          repository: rerun-io/rerun
+          ref: ${{ env.RERUN_SDK_VERSION }}
+          lfs: true
+          persist-credentials: false
+
+      - name: Checkout python-wheels
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          path: python-wheels
+          persist-credentials: false
+
+      # lance-core and lance-linalg 9.0.0 (build-dependencies of re_datafusion
+      # via lance) have per-arch SIMD structs/closures with cfg arms for
+      # aarch64/x86_64/loongarch64 and no catch-all, so they don't compile on
+      # riscv64: lance-core's SIMD-tier-detection closure evaluates to `()`
+      # instead of `SimdSupport` (E0308), and lance-linalg's f32x8/f32x16/
+      # f64x4/f64x8/i32x8 types don't exist at all there (E0425) (CLAUDE.md
+      # gotchas 287/300). There is no compatible bugfix release to `cargo
+      # update` to, so riscv64-fixed copies are vendored here (the crates.io
+      # tarballs, already free of the workspace-inherited fields the git
+      # checkouts carry) and wired in via `[patch.crates-io]`.
+      - name: Vendor riscv64-fixed lance-core and lance-linalg
+        run: |
+          curl -fsSL -A "python-wheels (https://github.com/riseproject-dev/python-wheels)" -o lance-core.tar.gz https://crates.io/api/v1/crates/lance-core/9.0.0/download
+          mkdir lance-core-9.0.0-riscv64
+          tar xzf lance-core.tar.gz -C lance-core-9.0.0-riscv64 --strip-components=1
+          rm lance-core.tar.gz
+          patch -p1 -d lance-core-9.0.0-riscv64 < python-wheels/patches/rerun-sdk/${{ env.RERUN_SDK_VERSION }}/0001-lance-core-riscv64-simd-fallback.patch
+
+          curl -fsSL -A "python-wheels (https://github.com/riseproject-dev/python-wheels)" -o lance-linalg.tar.gz https://crates.io/api/v1/crates/lance-linalg/9.0.0/download
+          mkdir lance-linalg-9.0.0-riscv64
+          tar xzf lance-linalg.tar.gz -C lance-linalg-9.0.0-riscv64 --strip-components=1
+          rm lance-linalg.tar.gz
+          patch -p1 -d lance-linalg-9.0.0-riscv64 < python-wheels/patches/rerun-sdk/${{ env.RERUN_SDK_VERSION }}/0003-lance-linalg-riscv64-simd-fallback.patch
+
+          git apply python-wheels/patches/rerun-sdk/${{ env.RERUN_SDK_VERSION }}/0002-cargo-patch-lance-crates-for-riscv64.patch
+
+      # This workspace links ~760 crates (datafusion, lance, tonic, hdf5-pure,
+      # ...) concurrently; same OOM guard build-deltalake.yml/
+      # build-polars-runtime.yml need for a comparably sized Rust build.
+      - name: Set swap space
+        uses: pierotofy/set-swap-space@fc79b3f67fa8a838184ce84a674ca12238d2c761  # master
+        with:
+          swap-size-gb: 10
+
+      - name: Build wheel
+        uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074  # v4.2.0
+        with:
+          package-dir: rerun_py
+          output-dir: wheelhouse/
+          # musllinux is dropped: rustup.rs ships no riscv64 musl toolchain.
+          # cp310/cp311 are dropped: our registry has no riscv64 pyarrow
+          # there (see the floor note above); cibuildwheel builds once on
+          # cp312 and re-tests the same abi3 wheel on cp313/cp314.
+          # Upstream ships no free-threaded wheel, so cp314t is not added.
+        env:
+          CIBW_BUILD: cp312-manylinux_riscv64 cp313-manylinux_riscv64 cp314-manylinux_riscv64
+          CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }}
+          # rerun_py ships no [tool.cibuildwheel]; the Rust toolchain its
+          # maturin backend needs is installed in-container here. lance (a
+          # build-dependency of re_datafusion) needs protoc at build time to
+          # compile its manifest .proto files -- upstream's own before-build
+          # pulls a wheel-packaged protoc, which has no riscv64 build; Rocky
+          # 10's CRB repo (enabled in the image) has one (CLAUDE.md gotcha
+          # 100).
+          CIBW_BEFORE_ALL_LINUX: >-
+            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y &&
+            yum install -y protobuf-compiler protobuf-devel
+          # The repo's own .cargo/config.toml unconditionally points
+          # PYO3_CONFIG_FILE at rerun_py/pyo3-build.cfg for every cargo
+          # invocation, expecting a pixi activation hook to have generated it
+          # first; outside pixi it must be written by hand (upstream's own
+          # fallback: `python scripts/generate_pyo3_config.py`). abi3 pins the
+          # `version` field to 3.10 regardless of the interpreter that writes
+          # it, so which one runs this does not matter.
+          CIBW_BEFORE_BUILD_LINUX: |
+            python -c "
+            import sys; sys.path.insert(0, 'rerun_pixi_env/src')
+            from pathlib import Path
+            from rerun_pixi_env.pyo3_config import generate_config_file
+            generate_config_file(Path('rerun_py/pyo3-build.cfg'))"
+          CIBW_ENVIRONMENT_LINUX: >-
+            PATH="$PATH:$HOME/.cargo/bin"
+            PROTOC_INCLUDE=/usr/include
+            RERUN_ALLOW_MISSING_BIN=1
+            CARGO_BUILD_JOBS=2
+            PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/
+          # Without this pip prefers PyPI's newer numpy/pillow/pandas/torch,
+          # none of which have riscv64 wheels, and falls back to slow source
+          # builds in the container (CLAUDE.md gotcha 12).
+          CIBW_TEST_ENVIRONMENT_LINUX: PIP_ONLY_BINARY=numpy,pillow,pandas,torch,pyarrow
+          # Subset of upstream's `tests` extra (pyproject.toml
+          # [project.optional-dependencies]) that has riscv64 wheels
+          # somewhere: av, torchvision, datafusion (the PyPI package) and
+          # polars publish none, so the handful of test files that need them
+          # are ignored below instead. opencv-python is on our registry as a
+          # cp37-abi3 wheel, forward-compatible with cp312+.
+          CIBW_TEST_REQUIRES: >-
+            pytest opencv-python>4.6 torch>=2.5 pandas>=2
+            syrupy==5.0.0 inline-snapshot==0.31.1 semver>=3.0,<3.1 tomli==2.0.1
+          # test-sources resolves against the checkout root, not package-dir.
+          # tests/assets is the workspace-level fixture dir test_asset3d.py
+          # reads via `Path(__file__).parents[3] / "tests" / "assets"`; the
+          # importer_mcap assets and the animated_urdf example data are the
+          # same shape for test_mcap_reader.py / test_urdf_tree.py. Cargo.toml
+          # is test_version.py's own fixture (it parses the workspace version
+          # straight out of it).
+          CIBW_TEST_SOURCES: >-
+            rerun_py/tests rerun_py/pyproject.toml tests/assets
+            crates/store/re_importer/src/importer_mcap/tests/assets
+            examples/rust/animated_urdf/data Cargo.toml
+          # e2e_redap_tests needs the "server" Cargo feature (not built, see
+          # the scope note above) to start its local catalog server.
+          # api_sandbox mixes in drafts of a future, unreleased API. Both are
+          # upstream-labelled sandboxes/integration suites, not the SDK's
+          # regression tests. test_headless_viewer.py spawns the (unbuilt)
+          # rerun-cli binary, same as test_rrd_reader_multi_store.py's
+          # session fixture (`rerun rrd merge`). test_server.py needs the
+          # `datafusion` Python package (no riscv64 wheel anywhere, same as
+          # test_datafusion_utils.py); the remaining ignored files all reach
+          # rerun.experimental.dataloader, whose decoders import torchvision
+          # or datafusion transitively. `-o markers=local_only` registers the
+          # mark e2e_redap_tests/conftest.py normally defines -- pytest only
+          # loads conftest.py files under the paths it actually collects, so
+          # excluding that directory drops the registration too and
+          # `filterwarnings = error` turns the resulting "unknown mark"
+          # warning into a collection failure. The deselected tests are
+          # single failures in otherwise-passing files, for the same three
+          # reasons (rerun-cli, av, datafusion) plus
+          # test_multiprocessing_gc, whose own comment says
+          # torch.multiprocessing (torch is on CIBW_TEST_REQUIRES) "causes
+          # more issues" than stdlib multiprocessing for this fork+gc race;
+          # it hits the test's 5s join() timeout consistently on this
+          # runner but not the reason any other test here was dropped --
+          # deselect, don't chase (CLAUDE.md gotcha 14).
+          CIBW_TEST_COMMAND: >-
+            cd rerun_py &&
+            python -c "import rerun_bindings.rerun_bindings as m; assert m.__file__.endswith('.so'), m.__file__" &&
+            python -m pytest tests -v -o markers=local_only
+            --ignore=tests/e2e_redap_tests
+            --ignore=tests/api_sandbox
+            --ignore=tests/integration/test_headless_viewer.py
+            --ignore=tests/integration/test_dataloader_video.py
+            --ignore=tests/integration/test_dataloader_video_codecs.py
+            --ignore=tests/integration/test_chunk_store_reader.py
+            --ignore=tests/integration/test_rrd_reader_multi_store.py
+            --ignore=tests/unit/test_dataloader_decoder_helpers.py
+            --ignore=tests/unit/test_dataloader_manifest.py
+            --ignore=tests/unit/test_dataloader_query_indices.py
+            --ignore=tests/unit/test_dataloader_skip_incomplete.py
+            --ignore=tests/unit/test_sample_index.py
+            --ignore=tests/unit/test_datafusion_utils.py
+            --ignore=tests/unit/test_viewer_client.py
+            --ignore=tests/unit/test_server.py
+            --deselect=tests/integration/test_chunk_store.py::test_collect_optimize_video_stream_summary
+            --deselect=tests/integration/test_lazy_chunk_stream.py::test_identity_roundtrip
+            --deselect=tests/unit/test_binary_stream.py::test_binary_stream
+            --deselect=tests/unit/test_multi_stream.py::test_isolated_streams
+            --deselect=tests/unit/test_send_dataframe.py::test_send_dataframe_roundtrip
+            --deselect=tests/unit/test_multiprocessing_gc.py::test_multiprocessing_gc
+
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: rerun-sdk-${{ env.RERUN_SDK_VERSION }}-cp310-abi3-manylinux_riscv64
+          path: wheelhouse/*.whl
+          if-no-files-found: error
+
+  publish:
+    name: Publish rerun-sdk ${{ inputs.version || '0.37.1' }}
+    needs: [setup, build_wheels]
+    permissions:
+      contents: write
+      pull-requests: write
+    uses: $/.github/workflows/_publish-wheel.yml
+    with:
+      artifact-pattern: rerun-sdk-${{ inputs.version || '0.37.1' }}-*-manylinux_riscv64

patches/rerun-sdk/0.37.1/0001-lance-core-riscv64-simd-fallback.patch

@@ -0,0 +1,22 @@
+Upstream-Status: To upstream [lance's own SIMD-tier cfg arms have no catch-all for architectures without a dedicated tier (riscv64 included); not filed upstream per this session's policy against opening external issues/PRs]
+
+--- a/src/utils/cpu.rs
++++ b/src/utils/cpu.rs
+@@ -210,6 +210,17 @@
+             SimdSupport::None
+         }
+     }
++    // riscv64 (and any other architecture without a dedicated tier above) has
++    // no SIMD kernels in lance yet; upstream's cfg arms have no catch-all, so
++    // this closure otherwise evaluates to `()` here instead of `SimdSupport`.
++    #[cfg(not(any(
++        target_arch = "aarch64",
++        target_arch = "x86_64",
++        target_arch = "loongarch64"
++    )))]
++    {
++        SimdSupport::None
++    }
+ });
+
+ #[cfg(target_arch = "x86_64")]

patches/rerun-sdk/0.37.1/0002-cargo-patch-lance-crates-for-riscv64.patch

@@ -0,0 +1,37 @@
+Upstream-Status: Inappropriate [points at a directory this port's workflow materializes at runtime; not a fix rerun itself would carry]
+
+diff --git a/Cargo.lock b/Cargo.lock
+index 52b6e5b..4e5ec02 100644
+--- a/Cargo.lock
++++ b/Cargo.lock
+@@ -5622,8 +5622,6 @@ dependencies = [
+ [[package]]
+ name = "lance-core"
+ version = "9.0.0"
+-source = "registry+https://github.com/rust-lang/crates.io-index"
+-checksum = "238c8a58308e7718d6bd96b53494eb7953fa299778bc4911cc571c3576e9446d"
+ dependencies = [
+  "arrow-array",
+  "arrow-buffer",
+@@ -5923,8 +5921,6 @@ dependencies = [
+ [[package]]
+ name = "lance-linalg"
+ version = "9.0.0"
+-source = "registry+https://github.com/rust-lang/crates.io-index"
+-checksum = "0e5a9b99bd1f49bc2fe5afb81323141abc506c818f589de95dbab4f6143d9a88"
+ dependencies = [
+  "arrow-array",
+  "arrow-buffer",
+diff --git a/Cargo.toml b/Cargo.toml
+index ac0093e..b2093a8 100644
+--- a/Cargo.toml
++++ b/Cargo.toml
+@@ -845,6 +845,8 @@ self_named_module_files = "allow" # Disabled waiting on https://github.com/rust-
+ significant_drop_tightening = "allow" # An update of parking_lot made this trigger in a lot of places. TODO(emilk): fix those places
+
+ [patch.crates-io]
++lance-core = { path = "lance-core-9.0.0-riscv64" }
++lance-linalg = { path = "lance-linalg-9.0.0-riscv64" }
+ # Try to avoid patching crates! It prevents us from publishing the crates on crates.io.
+ # If you do patch always prefer to patch to the trunk branch of the upstream repo (i.e. `main`, `master`, …).
+ # If that is not possible, patch to a branch that has a PR open on the upstream repo.