Benchmarks

.github/workflows/docs.yml

@@ -3,24 +3,19 @@ name: Documentation
 on:
   push:
     branches: [main]
-    paths:
-      - ".github/workflows/docs.yml"
-      - "README.md"
-      - "docs/**"
-      - "docs_theme/**"
-      - "mkdocs.yml"
-      - "pyproject.toml"
-      - "tests/shared/docs/**"
-      - "tools/mkdocs_publication.py"
   pull_request:
     paths:
       - ".github/workflows/docs.yml"
       - "README.md"
+      - "benchmarks/**"
       - "docs/**"
       - "docs_theme/**"
       - "mkdocs.yml"
       - "pyproject.toml"
+      - "tests/shared/architecture/test_test_suite_layout.py"
       - "tests/shared/docs/**"
+      - "tests/shared/tools/test_generate_performance_docs.py"
+      - "tools/generate_performance_docs.py"
       - "tools/mkdocs_publication.py"
   workflow_dispatch:
 
@@ -31,10 +26,86 @@ concurrency:
   group: pages
   cancel-in-progress: true
 
+env:
+  X2PY_GFORTRAN_BINARY: gfortran-13
+  X2PY_GFORTRAN_PACKAGE: gfortran-13
+
 jobs:
+  benchmark:
+    name: Benchmark
+    if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
+    runs-on: ${{ vars.X2PY_BENCHMARK_RUNNER || 'ubuntu-24.04' }}
+    timeout-minutes: 90
+    permissions:
+      contents: read
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Install benchmark dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[docs]" \
+            "numpy==2.5.1" \
+            "meson==1.11.2" \
+            "ninja==1.13.0"
+
+      - name: Install pinned GFortran
+        shell: bash
+        run: |
+          if ! command -v "$X2PY_GFORTRAN_BINARY" >/dev/null 2>&1; then
+            sudo apt-get update
+            sudo apt-get install --yes "$X2PY_GFORTRAN_PACKAGE"
+          fi
+          compiler_dir="$RUNNER_TEMP/x2py-gfortran"
+          mkdir -p "$compiler_dir"
+          ln -sf "$(command -v "$X2PY_GFORTRAN_BINARY")" "$compiler_dir/gfortran"
+          echo "$compiler_dir" >> "$GITHUB_PATH"
+          "$compiler_dir/gfortran" --version
+
+      - name: Run correctness and rigorous performance suite
+        shell: bash
+        run: |
+          if (( GITHUB_RUN_NUMBER % 2 == 0 )); then
+            export X2PY_BENCHMARK_FIRST=f2py
+          else
+            export X2PY_BENCHMARK_FIRST=x2py
+          fi
+          bash benchmarks/run.sh
+
+      - name: Generate public Performance snapshot
+        run: |
+          python tools/generate_performance_docs.py \
+            --commit "$GITHUB_SHA"
+
+      - name: Report measurement stability
+        run: python -m pyperf check benchmarks/results/f2py.json benchmarks/results/x2py.json
+
+      - name: Upload Performance snapshot and raw results
+        uses: actions/upload-artifact@v4
+        with:
+          name: performance-snapshot
+          path: |
+            benchmarks/results/f2py.json
+            benchmarks/results/x2py.json
+            docs/user/performance.md
+            docs/user/assets/performance-comparison.svg
+          retention-days: 90
+
   build:
     name: Build
-    runs-on: ubuntu-latest
+    needs: benchmark
+    if: always() && (needs.benchmark.result == 'success' || needs.benchmark.result == 'skipped')
+    runs-on: ubuntu-24.04
     permissions:
       contents: read
       pages: write
@@ -52,6 +123,13 @@ jobs:
       - name: Install documentation dependencies
         run: python -m pip install -e ".[docs,qa]"
 
+      - name: Download generated Performance snapshot
+        if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
+        uses: actions/download-artifact@v4
+        with:
+          name: performance-snapshot
+          path: .
+
       - name: Run documentation tests
         run: python -m pytest -q tests/shared/docs
 

benchmarks/.gitignore

@@ -0,0 +1,2 @@
+/build/f2py/
+/results/

benchmarks/README.md

@@ -0,0 +1,62 @@
+# Binding Performance Benchmarks
+
+This suite compares the call and NumPy-array overhead of wrappers generated by
+x2py and NumPy's f2py on the same machine.
+
+The default x2py wrapper and the f2py wrappers measured here keep the GIL held,
+so the suite reports one like-for-like comparison of their normal generated
+interfaces.
+
+`X2PY_BENCHMARK_FIRST=x2py` is the default measurement order. Set it to
+`f2py` to reverse the order. The publication workflow alternates this setting
+between runs so one tool is not systematically measured first.
+
+Run the complete correctness check and rigorous benchmark with:
+
+```bash
+bash run.sh
+```
+
+The script rebuilds both extensions and applies
+`-O3 -march=native -mtune=native` to the native Fortran source, generated
+Fortran wrapper, and generated C binding. It retains f2py's generated sources
+under `build/f2py` for local inspection.
+
+To compare existing results without rebuilding:
+
+```bash
+python3 -m pyperf compare_to \
+    results/f2py.json \
+    results/x2py.json \
+    --table
+```
+
+Results are machine-specific. Compare files produced in the same run; CPU,
+compiler, Python, and NumPy differences can otherwise dominate small timings.
+The generated build directories, extensions, and result files are local
+artifacts rather than repository sources.
+
+## Publish a Documentation Snapshot
+
+After a completed run, refresh the generated sections of the public Performance
+page and its chart with:
+
+```bash
+python3 tools/generate_performance_docs.py
+```
+
+Run this command from the repository root. It reads the paired `pyperf` files,
+checks that they contain the same benchmarks and compatible platform metadata,
+records the host operating-system distribution and compiler, and updates only
+the marked result sections in `docs/user/performance.md` plus
+`docs/user/assets/performance-comparison.svg`. Explanatory prose and the
+reproduction instructions remain hand-maintained.
+
+The Documentation workflow performs the same generation after successful
+correctness checks and rigorous measurements on pushes to `main`. It keeps the
+raw `pyperf` files as a workflow artifact and overlays the generated snapshot
+only in the website build; it does not create a result commit.
+
+The publication environment pins Python 3.12, NumPy/f2py 2.5.1, pyperf 2.10.0,
+Meson 1.11.2, Ninja 1.13.0, and GNU Fortran 13. Update those versions through a
+reviewed change so published runs remain comparable.

benchmarks/build/f2py.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+benchmark_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$benchmark_dir"
+
+rm -rf bench_f2py.*.so build/f2py
+mkdir -p build/f2py
+
+CFLAGS="-O3 -march=native -mtune=native" \
+FFLAGS="-O3 -march=native -mtune=native" \
+F90FLAGS="-O3 -march=native -mtune=native" \
+python3 -m numpy.f2py \
+    -c \
+    -m bench_f2py \
+    sources/kernels.f90 \
+    --build-dir build/f2py \
+    --f90flags="-O3 -march=native -mtune=native" \
+    --opt="-O3 -march=native -mtune=native"

benchmarks/build/x2py.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+benchmark_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$benchmark_dir"
+
+rm -rf bench_x2py.*.so __x2py__
+python3 -m x2py \
+    sources/kernels.f90 \
+    --out bench_x2py \
+    --native-compile-flags="-O3 -march=native -mtune=native" \
+    --wrapper-fortran-flags="-O3 -march=native -mtune=native" \
+    --wrapper-c-flags="-O3 -march=native -mtune=native" \
+    --verbose

benchmarks/correctness.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+import importlib
+from typing import Any
+
+import numpy as np
+
+
+def load_api(module_name: str, nested_module: str | None = None) -> Any:
+    module = importlib.import_module(module_name)
+    return getattr(module, nested_module) if nested_module else module
+
+
+x2py = load_api("bench_x2py", "kernels")
+f2py = load_api("bench_f2py", "kernels")
+
+
+def check_implementation(api: Any) -> None:
+    assert api.add_scalars(np.float64(2.0), np.float64(3.0)) == np.float64(5.0)
+
+    vector = np.arange(32, dtype=np.float64)
+    expected = vector + 1.0
+    api.increment_vector(vector)
+    np.testing.assert_allclose(vector, expected)
+
+    matrix = np.asfortranarray(np.arange(128, dtype=np.float64).reshape((16, 8), order="F"))
+    expected_sum = np.sum(matrix)
+    actual_sum = api.sum_matrix(matrix)
+    np.testing.assert_allclose(actual_sum, expected_sum)
+
+    api.matrix_update(matrix, np.float64(2.0))
+    np.testing.assert_allclose(
+        matrix,
+        np.arange(128, dtype=np.float64).reshape((16, 8), order="F") + 2.0,
+    )
+
+
+check_implementation(x2py)
+check_implementation(f2py)
+
+print("All implementations passed correctness checks.")

benchmarks/run.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+benchmark_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+cd "$benchmark_dir"
+
+benchmark_first="${X2PY_BENCHMARK_FIRST:-x2py}"
+case "$benchmark_first" in
+    x2py)
+        binding_tools=(x2py f2py)
+        ;;
+    f2py)
+        binding_tools=(f2py x2py)
+        ;;
+    *)
+        echo "X2PY_BENCHMARK_FIRST must be 'x2py' or 'f2py'." >&2
+        exit 2
+        ;;
+esac
+
+rm -rf *.so __x2py__ results
+
+echo
+echo "========================================"
+echo " Building X2PY wrapper"
+echo "========================================"
+bash build/x2py.sh
+
+echo
+echo "========================================"
+echo " Building F2PY wrapper"
+echo "========================================"
+bash build/f2py.sh
+echo "========================================"
+echo "========================================"
+echo "========================================"
+
+echo
+echo "Check correctness of all shared libraries..."
+python3 correctness.py
+echo
+echo "========================================"
+echo "========================================"
+echo "========================================"
+
+mkdir -p results
+
+echo "Benchmark order: ${binding_tools[*]}"
+for binding_tool in "${binding_tools[@]}"; do
+    BINDING_TOOL="$binding_tool" \
+    OMP_NUM_THREADS=1 \
+    OPENBLAS_NUM_THREADS=1 \
+    MKL_NUM_THREADS=1 \
+    python3 runtime.py \
+        --rigorous \
+        --affinity=0 \
+        --inherit-environ=BINDING_TOOL,OMP_NUM_THREADS,OPENBLAS_NUM_THREADS,MKL_NUM_THREADS \
+        -o "results/$binding_tool.json"
+done
+
+python3 -m pyperf compare_to \
+    results/f2py.json \
+    results/x2py.json \
+    --table