From 57b0165c46275839a5d45061fff9d7c6b1d4aff0 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 12 Oct 2026 13:51:36 +0200 Subject: [PATCH 1/2] Stop timing NumPy in the ASV benchmarks A dpnp change cannot move a NumPy timing, so the numpy executor points only doubled the run time of the elementwise, linalg and random suites and let machine noise on them be reported as a dpnp regression. Keep `executor` as a single-value axis and leave the `setup`/`time_*` bodies untouched, so result names and ASV benchmark versions stay comparable with the recorded history. --- CHANGELOG.md | 1 + benchmarks/README.md | 47 +++++++++++++--------- benchmarks/benchmarks/_utils.py | 4 +- benchmarks/benchmarks/bench_elementwise.py | 2 +- benchmarks/benchmarks/bench_linalg.py | 2 +- benchmarks/benchmarks/bench_random.py | 10 ++--- 6 files changed, 37 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0240e4d5d9eb..bff240a4f870 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,7 @@ This release is compatible with NumPy 2.5. * Linked the `dpnp_backend_c` library against only the MKL SYCL domains it uses (`BLAS`, `RNG`, `VM`) [#3012](https://github.com/IntelPython/dpnp/pull/3012) * `dpnp` uses pybind11 3.1.0 [#3015](https://github.com/IntelPython/dpnp/pull/3015) * Reworked the ASV benchmarks and added end-to-end workload benchmarks derived from dpBench [#2996](https://github.com/IntelPython/dpnp/pull/2996) +* Stopped timing NumPy in the ASV benchmarks, which now measure only dpnp [#3095](https://github.com/IntelPython/dpnp/pull/3095) * Reduced allocations in `dpnp.linalg.norm` by reusing the reduction result as the `sqrt` output buffer in the 2-norm and Frobenius-norm branches [#3062](https://github.com/IntelPython/dpnp/pull/3062) * Changed `dpnp.tensor.top_k` with `mode="largest"` to no longer return `NaN` values (or complex values with a `NaN` component) ahead of finite values, matching the `NaN`-last order of `dpnp.tensor.sort` [#3066](https://github.com/IntelPython/dpnp/pull/3066) * Avoided a copy of `dpnp.einsum` result into C-order by building the product in the requested layout directly [#3069](https://github.com/IntelPython/dpnp/pull/3069) diff --git a/benchmarks/README.md b/benchmarks/README.md index 5802b6b65df3..b4a19026259b 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -8,11 +8,23 @@ Performance benchmarks for [dpnp](https://github.com/IntelPython/dpnp) using | File | API | Benchmarks | Params | Sizes | |------|-----|------------|--------|-------| | `bench_dpbench.py` | `dpnp` (end-to-end workloads) | `BlackScholes`, `L2Norm`, `PairwiseDistance`, `Rambo`, `Gpairs` | `preset`, `precision` | dpBench presets `S`, `M16Gb`, `M`, `L` | -| `bench_elementwise.py` | `dpnp` vs `numpy` | `Unary` (31 ufuncs), `Binary` (7 ufuncs) | `executor`, `ufunc`, `size`, `dtype` (float only) | 2^16, 2^20, 2^24 | -| `bench_linalg.py` | `dpnp` vs `numpy` (`dot`, `matmul`, `inner`, `einsum`; contiguous and transposed) | `MatMul` | `executor`, `order`, `dtype` (float and int) | 16 to 1024 square | -| `bench_linalg.py` | `dpnp.linalg` vs `numpy.linalg` (`det`, `norm`, `solve`, `svd`) | `Linalg` | `executor`, `order`, `dtype` (float only) | 16 to 1024 square | -| `bench_random.py` | `dpnp.random` vs `numpy.random` | `Sample` (`random_sample`, `standard_normal`) | `executor`, `size` | 2^16, 2^20, 2^24 | -| `bench_random.py` | `dpnp.random.RandomState` vs `numpy.random.default_rng` | `TypedSample` (uniform, normal) | `executor`, `size`, `dtype` (float only) | 2^16, 2^20, 2^24 | +| `bench_elementwise.py` | `dpnp` ufuncs | `Unary` (31 ufuncs), `Binary` (7 ufuncs) | `executor`, `ufunc`, `size`, `dtype` (float only) | 2^16, 2^20, 2^24 | +| `bench_linalg.py` | `dpnp` (`dot`, `matmul`, `inner`, `einsum`; contiguous and transposed) | `MatMul` | `executor`, `order`, `dtype` (float and int) | 16 to 1024 square | +| `bench_linalg.py` | `dpnp.linalg` (`det`, `norm`, `solve`, `svd`) | `Linalg` | `executor`, `order`, `dtype` (float only) | 16 to 1024 square | +| `bench_random.py` | `dpnp.random` | `Sample` (`random_sample`, `standard_normal`) | `executor`, `size` | 2^16, 2^20, 2^24 | +| `bench_random.py` | `dpnp.random.RandomState` | `TypedSample` (uniform, normal) | `executor`, `size`, `dtype` (float only) | 2^16, 2^20, 2^24 | + +### NumPy is not timed + +The suites track dpnp's own performance from commit to commit, so only dpnp is +timed. A NumPy point cannot move with a dpnp change, so it only adds run time, +and its machine noise was reported as a dpnp regression. + +`executor` is kept as a single-value axis (`'dpnp'`) so result names such as +`time_dot('dpnp', 64, 'float32')` match the recorded history. The `numpy` +branches left in some `setup` methods are kept for the same reason: ASV +versions a benchmark by the source of its `time_*` and `setup` methods, so +editing them would make every result incomparable with the recorded baseline. ### dpBench workloads @@ -70,16 +82,14 @@ declaration order. **Both precisions are benchmarked.** Devices without fp64 support (common on iGPUs) skip the `double` points via `SkipNotImplemented` rather than failing, so such a device still produces `single` results. The `float64` points of -`bench_elementwise.py` and `bench_linalg.py` skip the same way for the `dpnp` -executor; the `numpy` executor is unaffected. dpBench's own configs request -`double` throughout, and that value is kept in each workload's `PRECISION` for -reference. +`bench_elementwise.py` and `bench_linalg.py` skip the same way. dpBench's own +configs request `double` throughout, and that value is kept in each workload's +`PRECISION` for reference. -`bench_random.py`'s `Sample` skips its `dpnp` points without fp64: those -functions take no `dtype`, so dpnp would return the device's default float -against NumPy's `float64`. `TypedSample` covers the same two distributions -through dpnp's `RandomState` and NumPy's `default_rng`, which do take a `dtype`, -so `float32` is still compared there. +`bench_random.py`'s `Sample` skips without fp64: those functions take no +`dtype`, so the result dtype would depend on the device. `TypedSample` covers +the same two distributions through dpnp's `RandomState`, which does take a +`dtype`, so `float32` is still timed there. No benchmark module opens a SYCL queue at import time, so benchmark discovery and `asv check` work on a machine with no usable device; only `setup` needs one. @@ -110,14 +120,11 @@ interpreter per benchmark and avoids this entirely. **Every timed body that runs dpnp work must block on it.** dpnp enqueues to a SYCL queue and returns before the kernel has run, so a body that does not block measures submission rather than execution. On a CPU device a 1024x1024 float32 -`dot` measured **0.3 ms** unsynchronized against **18 ms** synchronized -- which -would have reported dpnp as far faster than NumPy's **10 ms** on work where it is -in fact 1.7x slower. +`dot` measured **0.3 ms** unsynchronized against **18 ms** synchronized. The dpBench workloads each end with `dpnp.synchronize_array_data`, and the -comparison suites obtain a synchronizer from `_utils.make_synchronizer` in -`setup` and pass every result through it (`self.sync(...)`). For the `numpy` -executor the synchronizer does nothing. +other suites obtain a synchronizer from `_utils.make_synchronizer` in `setup` +and pass every result through it (`self.sync(...)`). ### First-call costs diff --git a/benchmarks/benchmarks/_utils.py b/benchmarks/benchmarks/_utils.py index 787bc0139b57..a8b343e92bb6 100644 --- a/benchmarks/benchmarks/_utils.py +++ b/benchmarks/benchmarks/_utils.py @@ -29,13 +29,13 @@ """Shared helpers and parameter axes for the dpnp ASV benchmarks.""" import dpctl -import numpy from asv_runner.benchmarks.mark import SkipNotImplemented import dpnp # Keyed by name, so ASV's tables stay readable. -_EXECUTORS = {"dpnp": dpnp, "numpy": numpy} +# dpnp only: NumPy timing can't change with dpnp. The axis keeps result names. +_EXECUTORS = {"dpnp": dpnp} _EXECUTOR_NAMES = list(_EXECUTORS) # axes shared across multiple files diff --git a/benchmarks/benchmarks/bench_elementwise.py b/benchmarks/benchmarks/bench_elementwise.py index e0c48c638a61..ac09887a247e 100644 --- a/benchmarks/benchmarks/bench_elementwise.py +++ b/benchmarks/benchmarks/bench_elementwise.py @@ -26,7 +26,7 @@ # THE POSSIBILITY OF SUCH DAMAGE. # ***************************************************************************** -"""Benchmarks for elementwise ufuncs, dpnp against NumPy. +"""Benchmarks for dpnp elementwise ufuncs. The ufunc is a parameter, as in NumPy's own ``bench_ufunc.py``. """ diff --git a/benchmarks/benchmarks/bench_linalg.py b/benchmarks/benchmarks/bench_linalg.py index c3da4ed9d766..f34926a6331d 100644 --- a/benchmarks/benchmarks/bench_linalg.py +++ b/benchmarks/benchmarks/bench_linalg.py @@ -26,7 +26,7 @@ # THE POSSIBILITY OF SUCH DAMAGE. # ***************************************************************************** -"""Benchmarks for matrix products and decompositions, dpnp against NumPy.""" +"""Benchmarks for dpnp matrix products and decompositions.""" from ._utils import ( _EXECUTOR_NAMES, diff --git a/benchmarks/benchmarks/bench_random.py b/benchmarks/benchmarks/bench_random.py index 599f31a7de88..a165d2417e74 100644 --- a/benchmarks/benchmarks/bench_random.py +++ b/benchmarks/benchmarks/bench_random.py @@ -26,7 +26,7 @@ # THE POSSIBILITY OF SUCH DAMAGE. # ***************************************************************************** -"""Benchmarks for random sampling, dpnp.random against numpy.random.""" +"""Benchmarks for random sampling with dpnp.random.""" from functools import partial @@ -45,7 +45,7 @@ # One name per generator; rand, random, ranf, sample and randn are aliases. class Sample: - """Random sampling, dpnp against NumPy.""" + """Module-level random sampling functions.""" params = [_EXECUTOR_NAMES, _SIZES_1D] param_names = ["executor", "size"] @@ -76,9 +76,9 @@ def time_standard_normal(self, executor, size): class TypedSample: """Uniform and normal sampling at an explicit dtype. - dpnp reaches these through ``RandomState`` and NumPy through - ``default_rng``; the module-level functions in ``Sample`` take no dtype, so - only this class can compare float32 on a device without fp64. + dpnp reaches these through ``RandomState``; the module-level functions in + ``Sample`` take no dtype, so only this class can time float32 on a device + without fp64. """ params = [_EXECUTOR_NAMES, _SIZES_1D, _FLOAT_DTYPES] From 6f9e1f6030e107e03c8b69ef4e6a5a225b1c8f91 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 13 Oct 2026 14:32:25 +0200 Subject: [PATCH 2/2] Drop dead executor check from make_synchronizer With numpy gone from _EXECUTORS the helper is only called with "dpnp". It is not part of ASV's benchmark version hash, so benchmark versions stay unchanged. --- benchmarks/benchmarks/_utils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/benchmarks/benchmarks/_utils.py b/benchmarks/benchmarks/_utils.py index a8b343e92bb6..febab8d29d06 100644 --- a/benchmarks/benchmarks/_utils.py +++ b/benchmarks/benchmarks/_utils.py @@ -56,9 +56,8 @@ def default_queue(): def make_synchronizer(executor): - """Return a callable blocking until ``executor``'s work has finished.""" - if executor != "dpnp": - return lambda result: None + """Return a callable blocking until dpnp's work has finished.""" + # executor is always "dpnp"; still passed by the hashed setup methods def sync(result): # Some results are tuples, e.g. linalg.svd.