Skip to content

Commit 95f423b

Browse files
authored
mpi4py: add build-mpi4py.yml for riscv64 wheels (#1686)
* **Package**: `mpi4py` * **Version**: `4.1.2` * **Source**: https://github.com/mpi4py/mpi4py * **Docs**: https://mpi4py.github.io/mpi4py/ Compiles mpi4py's Cython bindings for MPI into a `mpi4py.MPI` extension linked against a real MPI implementation. Upstream publishes no riscv64 wheel. Mirrors [upstream's own `cd-wheel.yml`](https://github.com/mpi4py/mpi4py/blob/4.1.2/.github/workflows/cd-wheel.yml), narrowed to Linux. **Differs from upstream** - Links against Rocky 10's `openmpi-devel` via `mpicc` on `PATH` - upstream's own wheels build against a portable "MPI ABI" stub with MPI supplied by conda-forge, which carries no packages for linux-riscv64. - `auditwheel repair` excludes the whole Open MPI/PMIx/libfabric/hwloc/libevent chain instead of vendoring it, since Open MPI's MCA components are compiled in rather than a relocatable plugin tree; the target system provides its own Open MPI at runtime, matching mpi4py's own documented `dnf install openmpi-devel` instructions. **Testing** - same as upstream's `test/main.py` suite (`mpiexec -n 1` and `-n 2`, matching `ci-test.yml`), instead of the release job's own thinner smoke checks **License**: OK
1 parent 277b76a commit 95f423b

1 file changed

Lines changed: 144 additions & 0 deletions

File tree

.github/workflows/build-mpi4py.yml

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
# SPDX-FileCopyrightText: 2026 The RISE Project
2+
# SPDX-License-Identifier: MIT
3+
---
4+
# Upstream's own release wheels (cd-wheel.yml) build against a portable "MPI ABI" stub
5+
# and merge mpich/openmpi variants into one wheel via conda-forge-provided MPI - conda-forge
6+
# ships no MPI packages for linux-riscv64. This instead uses mpi4py's other documented build
7+
# path (the error message in conf/mpidistutils.py's configure_mpi names it directly): link
8+
# against a real MPI found via `mpicc` on PATH, here Rocky 10's own openmpi-devel, and leave
9+
# it to the target system the same way a source build would (auditwheel excludes the whole
10+
# Open MPI/PMIx/libfabric/hwloc/libevent chain rather than vendoring a partial runtime that
11+
# would be missing Open MPI's own MCA component search path).
12+
name: Build mpi4py wheels (riscv64)
13+
14+
on:
15+
workflow_dispatch:
16+
inputs:
17+
version:
18+
description: 'mpi4py version to build (git tag, e.g. 4.1.2)'
19+
required: true
20+
default: '4.1.2'
21+
pull_request:
22+
paths:
23+
- '.github/workflows/build-mpi4py.yml'
24+
25+
concurrency:
26+
group: ${{ github.workflow }}-${{ inputs.version || '4.1.2' }}-${{ github.head_ref || github.run_id }}
27+
cancel-in-progress: true
28+
29+
permissions:
30+
contents: read # to fetch code (actions/checkout)
31+
32+
env:
33+
MPI4PY_VERSION: ${{ inputs.version || '4.1.2' }}
34+
MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64
35+
36+
jobs:
37+
setup:
38+
uses: $/.github/workflows/_setup.yml
39+
40+
build_wheels:
41+
needs: [setup]
42+
name: Build mpi4py ${{ inputs.version || '4.1.2' }} ${{ matrix.python }}-manylinux_riscv64
43+
runs-on: ubuntu-24.04-riscv
44+
timeout-minutes: 90
45+
strategy:
46+
fail-fast: false
47+
matrix:
48+
python:
49+
- "cp312"
50+
- "cp313"
51+
- "cp314"
52+
- "cp314t"
53+
54+
steps:
55+
- name: Checkout mpi4py ${{ env.MPI4PY_VERSION }}
56+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
57+
with:
58+
repository: mpi4py/mpi4py
59+
ref: ${{ env.MPI4PY_VERSION }}
60+
persist-credentials: false
61+
62+
- name: Write repair script
63+
run: |
64+
cat > riscv64-repair.sh <<'EOF'
65+
#!/bin/bash
66+
set -euo pipefail
67+
wheel="$1"
68+
dest_dir="$2"
69+
tmpdir=$(mktemp -d)
70+
trap 'rm -rf "$tmpdir"' EXIT
71+
unzip -q "$wheel" -d "$tmpdir"
72+
so=$(find "$tmpdir/mpi4py" -name 'MPI.*.so')
73+
args=()
74+
for lib in $(ldd "$so" | awk '{print $1}' | grep -E '^lib(mpi|open-pal|open-rte|event|hwloc|pmix|fabric)' | sort -u); do
75+
args+=(--exclude "$lib")
76+
done
77+
auditwheel repair "${args[@]}" -w "$dest_dir" "$wheel"
78+
EOF
79+
80+
- name: Build wheels
81+
uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0
82+
with:
83+
package-dir: .
84+
output-dir: wheelhouse/
85+
only: ${{ matrix.python }}-manylinux_riscv64
86+
env:
87+
CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }}
88+
# Matches upstream's own ci-test.yml "Configure hostname" step: without a resolvable
89+
# hostname, Open MPI's out-of-band TCP setup for a multi-process run can hang.
90+
CIBW_BEFORE_ALL_LINUX: >-
91+
dnf -y install openmpi-devel &&
92+
echo "127.0.0.1 $(hostname)" >> /etc/hosts
93+
CIBW_ENVIRONMENT_LINUX: >-
94+
PATH="/usr/lib64/openmpi/bin:$PATH"
95+
LD_LIBRARY_PATH="/usr/lib64/openmpi/lib:${LD_LIBRARY_PATH:-}"
96+
# Open MPI compiles its MCA components statically into libmpi/libopen-pal on this
97+
# image (no lib/openmpi/*.so plugin tree), so excluding the whole chain here and
98+
# requiring the target system's own Open MPI is safe - a partial vendor would
99+
# otherwise ship a libmpi.so with no matching runtime.
100+
CIBW_REPAIR_WHEEL_COMMAND_LINUX: bash {project}/riscv64-repair.sh {wheel} {dest_dir}
101+
# The manylinux container runs as root, which Open MPI's mpiexec refuses by default.
102+
CIBW_TEST_ENVIRONMENT_LINUX: >-
103+
OMPI_ALLOW_RUN_AS_ROOT=1
104+
OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
105+
CIBW_TEST_SOURCES: test
106+
# Dynamic process management (MPI_Comm_spawn, used by test_spawn/test_dynproc and
107+
# mpi4py.futures' pool in test_util_pool) hangs PRTE's resource mapper in this
108+
# container ("prte-rmaps-base:all-available-resources-used", then no progress) -
109+
# deselect those modules; `timeout` turns any other hang into a fast failure
110+
# instead of riding out the job timeout. `--oversubscribe` is needed because the
111+
# riscv64 CI runner exposes fewer slots than the `-n 2` run requests, and PRTE
112+
# refuses to oversubscribe by default.
113+
CIBW_TEST_COMMAND: >-
114+
python -m mpi4py --prefix &&
115+
python -m mpi4py --mpi-library &&
116+
timeout 600 mpiexec --oversubscribe -n 1 python test/main.py -v -e spawn -e dynproc -e util_pool &&
117+
timeout 600 mpiexec --oversubscribe -n 2 python test/main.py -v -f -e spawn -e dynproc -e util_pool
118+
119+
- name: Check the wheel links Open MPI instead of vendoring it
120+
run: |
121+
python3 - wheelhouse/*.whl <<'EOF'
122+
import sys, zipfile
123+
for whl in sys.argv[1:]:
124+
names = zipfile.ZipFile(whl).namelist()
125+
assert any(n.startswith("mpi4py/MPI") and n.endswith(".so") for n in names), f"no C extension in {whl}"
126+
assert not any(".libs/" in n or ".libs" == n.split("/")[0] for n in names), f"unexpected vendored libs in {whl}: {names}"
127+
print(whl, "ok")
128+
EOF
129+
130+
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
131+
with:
132+
name: mpi4py-${{ env.MPI4PY_VERSION }}-${{ matrix.python }}-manylinux_riscv64
133+
path: wheelhouse/*.whl
134+
if-no-files-found: error
135+
136+
publish:
137+
name: Publish mpi4py ${{ inputs.version || '4.1.2' }}
138+
needs: [setup, build_wheels]
139+
permissions:
140+
contents: write
141+
pull-requests: write
142+
uses: $/.github/workflows/_publish-wheel.yml
143+
with:
144+
artifact-pattern: mpi4py-${{ inputs.version || '4.1.2' }}-*-manylinux_riscv64

0 commit comments

Comments
 (0)