From 675e98244c89ea0ecbf5b48940e182713e88f97f Mon Sep 17 00:00:00 2001 From: maurycy <5383+maurycy@users.noreply.github.com> Date: Wed, 27 May 2026 19:15:43 +0200 Subject: [PATCH 1/7] metadata['phase'] --- pyperf/_metadata.py | 8 ++++++++ pyperf/_worker.py | 9 +++++++++ 2 files changed, 17 insertions(+) diff --git a/pyperf/_metadata.py b/pyperf/_metadata.py index e2ff96c5..fd8bc2b1 100644 --- a/pyperf/_metadata.py +++ b/pyperf/_metadata.py @@ -48,6 +48,12 @@ def is_tags(value): return all(isinstance(x, str) and x not in ('all', '') for x in value) +def is_phase(value): + if not isinstance(value, (tuple, list)): + return False + return len(value) == 2 and all(isinstance(x, str) and x for x in value) + + def parse_load_avg(value): if isinstance(value, NUMBER_TYPES): return value @@ -69,6 +75,7 @@ def format_noop(value): WARMUPS = _MetadataInfo(format_number, (int,), is_positive, 'integer') SECONDS = _MetadataInfo(format_seconds, NUMBER_TYPES, is_positive, 'second') TAGS = _MetadataInfo(format_generic, (list,), is_tags, 'tag') +PHASE = _MetadataInfo(format_generic, (tuple, list), is_phase, None) # Registry of metadata keys METADATA = { @@ -92,6 +99,7 @@ def format_noop(value): 'calibrate_warmups': WARMUPS, 'recalibrate_warmups': WARMUPS, 'tags': TAGS, + 'phase': PHASE, } DEFAULT_METADATA_INFO = _MetadataInfo(format_generic, METADATA_VALUE_TYPES, None, None) diff --git a/pyperf/_worker.py b/pyperf/_worker.py index 3547faeb..c9fe965b 100644 --- a/pyperf/_worker.py +++ b/pyperf/_worker.py @@ -114,6 +114,15 @@ def _compute_values(self, values, nvalue, index += 1 + if calibrate_loops: + self.metadata["phase"] = ("calibration", "loops") + elif is_warmup and (args.calibrate_warmups or args.recalibrate_warmups): + self.metadata["phase"] = ("calibration", "warmup") + elif is_warmup: + self.metadata["phase"] = ("measurement", "warmup") + else: + self.metadata["phase"] = ("measurement", "value") + for hook in hook_managers.values(): hook.teardown(self.metadata) From 05696c0b446ef9e8737cab3e7b2c5ab52001ac1d Mon Sep 17 00:00:00 2001 From: maurycy <5383+maurycy@users.noreply.github.com> Date: Wed, 27 May 2026 19:51:03 +0200 Subject: [PATCH 2/7] test --- pyperf/tests/test_bench.py | 19 +++++++++++++++++++ pyperf/tests/test_runner.py | 4 ++++ 2 files changed, 23 insertions(+) diff --git a/pyperf/tests/test_bench.py b/pyperf/tests/test_bench.py index 8fd4e2a1..3e4049f1 100644 --- a/pyperf/tests/test_bench.py +++ b/pyperf/tests/test_bench.py @@ -69,6 +69,25 @@ def test_constructor(self): collect_metadata=False) self.assertEqual(run.get_metadata()['load_avg_1min'], 0.0) + def test_phase_metadata(self): + for value in (('measurement', 'value'), ['calibration', 'loops']): + with self.subTest(value=value): + run = pyperf.Run([1.0], metadata={'phase': value}, + collect_metadata=False) + self.assertEqual(run.get_metadata()['phase'], value) + + invalid_values = ( + 'measurement:value', + ('measurement',), + ('measurement', ''), + ('measurement', 1), + ) + for value in invalid_values: + with self.subTest(value=value): + with self.assertRaises(ValueError): + pyperf.Run([1.0], metadata={'phase': value}, + collect_metadata=False) + def test_name(self): # name must be non-empty with self.assertRaises(ValueError): diff --git a/pyperf/tests/test_runner.py b/pyperf/tests/test_runner.py index c77e0251..7850c7c2 100644 --- a/pyperf/tests/test_runner.py +++ b/pyperf/tests/test_runner.py @@ -80,6 +80,8 @@ def test_worker(self): result = self.exec_runner('--worker', '-l1', '-w1') self.assertRegex(result.stdout, r'^bench: Mean \+- std dev: 1\.00 sec \+- 0\.00 sec\n$') + self.assertEqual(result.bench.get_metadata()['phase'], + ('measurement', 'value')) def test_debug_single_value(self): result = self.exec_runner('--debug-single-value', '--worker') @@ -286,6 +288,8 @@ def check_calibrate_loops(self, runner, time_func, warmups): run = runs[0] self.assertEqual(run.warmups, warmups) + self.assertEqual(run.get_metadata()['phase'], + ('calibration', 'loops')) def test_calibrate_loops(self): args = ['--worker', '-w0', '-n2', '--min-time=1.0', From e5b7e0a357c1f18a5cd22a528c46d312ec9e8627 Mon Sep 17 00:00:00 2001 From: maurycy <5383+maurycy@users.noreply.github.com> Date: Wed, 27 May 2026 23:26:11 +0200 Subject: [PATCH 3/7] teardown test --- pyperf/tests/test_runner.py | 41 +++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/pyperf/tests/test_runner.py b/pyperf/tests/test_runner.py index 7850c7c2..ca48d65c 100644 --- a/pyperf/tests/test_runner.py +++ b/pyperf/tests/test_runner.py @@ -80,8 +80,6 @@ def test_worker(self): result = self.exec_runner('--worker', '-l1', '-w1') self.assertRegex(result.stdout, r'^bench: Mean \+- std dev: 1\.00 sec \+- 0\.00 sec\n$') - self.assertEqual(result.bench.get_metadata()['phase'], - ('measurement', 'value')) def test_debug_single_value(self): result = self.exec_runner('--debug-single-value', '--worker') @@ -288,8 +286,6 @@ def check_calibrate_loops(self, runner, time_func, warmups): run = runs[0] self.assertEqual(run.warmups, warmups) - self.assertEqual(run.get_metadata()['phase'], - ('calibration', 'loops')) def test_calibrate_loops(self): args = ['--worker', '-w0', '-n2', '--min-time=1.0', @@ -523,6 +519,43 @@ def test_hook_command(self): self.assertEqual(bench.get_metadata()["hooks"], "_test_hook") + def test_hook_teardown_phase_metadata(self): + def time_func(loops): + return 1.0 + + def hook_phases(args): + phases = [] + + class PhaseHook(HookBase): + name = "phase_hook" + + @staticmethod + def load(): + return PhaseHook + + def teardown(self, metadata): + phases.append(metadata["phase"]) + + args = args + ["--hook", PhaseHook.name] + runner = self.create_runner(args, hooks=[PhaseHook]) + with tests.capture_stdout(): + runner.bench_time_func('bench', time_func) + return phases + + self.assertEqual( + hook_phases('-l1 -w1 -n1 --worker'.split()), + [('measurement', 'warmup'), ('measurement', 'value')]) + self.assertEqual( + hook_phases('--calibrate-loops -w0 -n1 --min-time=1.0 ' + '--worker'.split()), + [('calibration', 'loops')]) + + with mock.patch('pyperf._worker.WorkerTask.test_calibrate_warmups', + return_value=True): + self.assertEqual( + hook_phases('--calibrate-warmups -l1 -n1 --worker'.split()), + [('calibration', 'warmup')]) + def test_custom_hook(self): class State: inited = 0 From 466a442752af35b95fc8b882fd010d9c3c3456e5 Mon Sep 17 00:00:00 2001 From: maurycy <5383+maurycy@users.noreply.github.com> Date: Wed, 27 May 2026 23:55:30 +0200 Subject: [PATCH 4/7] better teardown test --- pyperf/tests/test_runner.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/pyperf/tests/test_runner.py b/pyperf/tests/test_runner.py index ca48d65c..181cb8de 100644 --- a/pyperf/tests/test_runner.py +++ b/pyperf/tests/test_runner.py @@ -542,19 +542,24 @@ def teardown(self, metadata): runner.bench_time_func('bench', time_func) return phases - self.assertEqual( - hook_phases('-l1 -w1 -n1 --worker'.split()), - [('measurement', 'warmup'), ('measurement', 'value')]) - self.assertEqual( - hook_phases('--calibrate-loops -w0 -n1 --min-time=1.0 ' - '--worker'.split()), - [('calibration', 'loops')]) - - with mock.patch('pyperf._worker.WorkerTask.test_calibrate_warmups', - return_value=True): - self.assertEqual( - hook_phases('--calibrate-warmups -l1 -n1 --worker'.split()), - [('calibration', 'warmup')]) + cases = ( + ('measurement', + '-l1 -w1 -n1 --worker', + [('measurement', 'warmup'), ('measurement', 'value')]), + ('calibrate loops', + '--calibrate-loops -w0 -n1 --min-time=1.0 --worker', + [('calibration', 'loops')]), + ) + for name, args, expected in cases: + with self.subTest(name=name): + self.assertEqual(hook_phases(args.split()), expected) + + with self.subTest(name='calibrate warmups'): + with mock.patch('pyperf._worker.WorkerTask.test_calibrate_warmups', + return_value=True): + self.assertEqual( + hook_phases('--calibrate-warmups -l1 -n1 --worker'.split()), + [('calibration', 'warmup')]) def test_custom_hook(self): class State: From 9ea69815dc36ab503bd9ab06eadabc5aacd65537 Mon Sep 17 00:00:00 2001 From: maurycy <5383+maurycy@users.noreply.github.com> Date: Thu, 28 May 2026 00:05:28 +0200 Subject: [PATCH 5/7] api.rst --- doc/api.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/api.rst b/doc/api.rst index cb08ebed..0b3e85d8 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -512,6 +512,7 @@ HookBase class Called when the hook is completed for a process. May add any information collected to the passed-in ``metadata`` dictionary. + The ``phase`` metadata identifies the run phase and sample type. Runner class @@ -780,6 +781,9 @@ Other: * ``perf_version``: Version of the ``pyperf`` module * ``unit``: Unit of values: ``byte``, ``integer`` or ``second`` +* ``phase`` (tuple or list of two non-empty strings): run phase and sample + type, set to one of ``("calibration", "loops")``, ``("calibration", + "warmup")``, ``("measurement", "warmup")`` or ``("measurement", "value")``. * ``calibrate_loops`` (``int >= 1``): number of loops computed in a loops calibration run * ``recalibrate_loops`` (``int >= 1``): number of loops computed in a loops From 4117e023f2008f0986fff0ed00c9d8d53473e964 Mon Sep 17 00:00:00 2001 From: maurycy <5383+maurycy@users.noreply.github.com> Date: Thu, 28 May 2026 00:12:21 +0200 Subject: [PATCH 6/7] changelog --- doc/changelog.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/changelog.rst b/doc/changelog.rst index 066beb8f..e486842a 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -1,6 +1,12 @@ Changelog ========= +* Feature: pyperf now stores ``phase`` run metadata in JSON files to identify + calibration and measurement samples. Hooks can also inspect this metadata in + :meth:`HookBase.teardown` to distinguish calibration loops, calibration + warmups, measurement warmups, and measurement values. + Patch by Maurycy Pawłowski-Wieroński. + Version 2.10.0 (2026-02-07) --------------------------- From 56c69cc97eaefb72764fd388447d3057f35fff17 Mon Sep 17 00:00:00 2001 From: maurycy <5383+maurycy@users.noreply.github.com> Date: Thu, 28 May 2026 00:19:04 +0200 Subject: [PATCH 7/7] json test --- pyperf/tests/test_bench.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/pyperf/tests/test_bench.py b/pyperf/tests/test_bench.py index 3e4049f1..48aa4439 100644 --- a/pyperf/tests/test_bench.py +++ b/pyperf/tests/test_bench.py @@ -1,6 +1,7 @@ import datetime import errno import gzip +import json import unittest import pyperf @@ -217,6 +218,40 @@ def test_dump_load(self): self.check_benchmarks_equal(bench, bench2) + def test_dump_phase_metadata(self): + runs = [ + pyperf.Run( + (), + warmups=[(1, 2.92e-7), (64, 2.65e-9)], + metadata={'name': 'bench', + 'loops': 64, + 'calibrate_loops': 64, + 'phase': ('calibration', 'loops')}, + collect_metadata=False), + pyperf.Run( + [2.5949900301497353e-9, 2.633662268304582e-9], + warmups=[(64, 2.6324198845045776e-9)], + metadata={'name': 'bench', + 'loops': 64, + 'phase': ('measurement', 'value')}, + collect_metadata=False), + ] + bench = pyperf.Benchmark(runs) + + bench_json = tests.benchmark_as_json(bench) + data = json.loads(bench_json) + json_runs = data['benchmarks'][0]['runs'] + self.assertEqual(json_runs[0]['metadata']['phase'], + ['calibration', 'loops']) + self.assertEqual(json_runs[1]['metadata']['phase'], + ['measurement', 'value']) + + bench2 = pyperf.Benchmark.loads(bench_json) + phases = [run.get_metadata()['phase'] for run in bench2.get_runs()] + self.assertEqual(phases, + [['calibration', 'loops'], + ['measurement', 'value']]) + def test_dump_replace(self): bench = self.create_dummy_benchmark()