From be428eecb8f04fc98cd84ed7a484a6eed55680e7 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Tue, 18 Aug 2026 03:08:09 +0000 Subject: [PATCH] init --- .github/workflows/bench.yaml | 67 ++++++++++ .gitignore | 2 +- README.md | 95 +++++++++++++ probe.py | 169 +++++++++++++++++++++++ report.py | 215 ++++++++++++++++++++++++++++++ run.sh | 145 ++++++++++++++++++++ scenario-helpers.sh | 172 ++++++++++++++++++++++++ scenarios/bare.sh | 6 + scenarios/docker-cgroup-parent.sh | 19 +++ scenarios/docker-cpuset-only.sh | 8 ++ scenarios/docker-host-ns.sh | 8 ++ scenarios/docker-memory-only.sh | 8 ++ scenarios/docker-private.sh | 13 ++ scenarios/k8s-limits.sh | 23 ++++ scenarios/k8s-no-limits.sh | 10 ++ scenarios/systemd-ancestor.sh | 18 +++ scenarios/systemd-own.sh | 23 ++++ wrap.py | 105 +++++++++++++++ 18 files changed, 1105 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/bench.yaml create mode 100644 probe.py create mode 100644 report.py create mode 100755 run.sh create mode 100644 scenario-helpers.sh create mode 100644 scenarios/bare.sh create mode 100644 scenarios/docker-cgroup-parent.sh create mode 100644 scenarios/docker-cpuset-only.sh create mode 100644 scenarios/docker-host-ns.sh create mode 100644 scenarios/docker-memory-only.sh create mode 100644 scenarios/docker-private.sh create mode 100644 scenarios/k8s-limits.sh create mode 100644 scenarios/k8s-no-limits.sh create mode 100644 scenarios/systemd-ancestor.sh create mode 100644 scenarios/systemd-own.sh create mode 100644 wrap.py diff --git a/.github/workflows/bench.yaml b/.github/workflows/bench.yaml new file mode 100644 index 0000000..691292e --- /dev/null +++ b/.github/workflows/bench.yaml @@ -0,0 +1,67 @@ +name: bench + +# The bench runs when a human triggers it, against whatever repo+ref they name. No schedules, no push triggers - +# this is an on-demand measurement instrument, not a monitor. + +on: + workflow_dispatch: + inputs: + repo: + description: Git repository holding the code under test + default: 'https://github.com/Mantisus/crawlee-python' + ref: + description: Branch, tag or SHA to install + default: 'container-limits' + scenarios: + description: Space-separated scenario names, or "all" + default: 'all' + cpu_budgets: + description: 'Cores for the CPU-limited scenarios, one run per value. Runner has 2 cores (private repo) or 4 (public repo)' + default: '[1, 2]' + image: + description: 'Base image the container scenarios run in. Any glibc distro works - uv is mounted in' + default: 'ghcr.io/astral-sh/uv:python3.13-bookworm' + +run-name: 'bench: ${{ inputs.ref }} / ${{ inputs.scenarios }} / cpus ${{ inputs.cpu_budgets }}' + +permissions: + contents: read + +jobs: + bench: + # One pass per CPU budget, each on its own runner, so a smaller budget's result is still there when a bigger + # one turns out to exceed the runner. + strategy: + fail-fast: false + matrix: + cpus: ${{ fromJSON(inputs.cpu_budgets) }} + name: bench (${{ matrix.cpus }} cpu) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v5 + + - name: Run scenarios + env: + CRAWLEE_REPO: ${{ inputs.repo }} + CRAWLEE_REF: ${{ inputs.ref }} + BENCH_CPUS: ${{ matrix.cpus }} + IMG: ${{ inputs.image }} + run: ./run.sh ${{ inputs.scenarios }} results/ + + - name: Report + if: always() + run: python3 report.py results/ >> "$GITHUB_STEP_SUMMARY" + + - name: Check + # The only gate: a probe that produced no data. Skips and observed values are for humans. + if: always() + run: python3 report.py results/ --check >/dev/null + + - name: Upload raw results + if: always() + uses: actions/upload-artifact@v4 + with: + name: results-${{ matrix.cpus }}cpu + path: results/ diff --git a/.gitignore b/.gitignore index 83972fa..fe0f660 100644 --- a/.gitignore +++ b/.gitignore @@ -215,4 +215,4 @@ marimo/_lsp/ __marimo__/ # Streamlit -.streamlit/secrets.toml +.streamlit/secrets.toml \ No newline at end of file diff --git a/README.md b/README.md index ece4b4d..1cfd1ef 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,97 @@ # cgroups-sensor + Utility functions to measure resource limits from cgroups in scenarios where psutils is not sufficient. + +Today the repository is a bench for that measurement: it runs a library's cgroup detection inside real +environment shapes — systemd scopes, containers, kubernetes pods — and prints what the library saw there. It +runs when a human triggers it and reports to a human; nothing is scheduled and nothing asserts on values. + +## Running it + +On GitHub: **Actions → bench → Run workflow**. Locally, on any cgroup-v2 Linux box with `uv`: + +```bash +./run.sh # every scenario, results into results/ +./run.sh bare docker-private # only these two +BENCH_CPUS=2 IMG=fedora:41 ./run.sh # a different budget, in another distro's userspace +python3 report.py results/ # the tables +python3 report.py results/ --check # same, exit 1 if a probe failed +``` + +| knob | what it sets | +| --- | --- | +| `CRAWLEE_REPO`, `CRAWLEE_REF` | what to measure; installed as a source tarball, so the image needs no `git` | +| `BENCH_CPUS` | cores the CPU-limited scenarios ask for, default 1. They derive cpuset and quota from it, so run it twice: once below the host's core count and once equal to it, where a cpuset stops restricting anything | +| `IMG` | base image for the container scenarios, default the uv one. Any glibc distro works — uv and the interpreter are mounted in | +| `BENCH_PYTHON` | interpreter to measure on, default 3.13, so the image cannot change it | + +Scenarios whose prerequisites are missing (docker, passwordless sudo, a systemd user manager, kind, enough +cores) are recorded as `skipped` with the reason; only a crashed probe reddens a run. The `k8s-*` scenarios +bring up a kind cluster named `cgroups-bench` and leave it running — `kind delete cluster --name cgroups-bench`. + +## Files + +| file | what it does | +| --- | --- | +| [run.sh](run.sh) | Runs each scenario and always writes a result, so a crash is a recorded row rather than a missing one. | +| [scenario-helpers.sh](scenario-helpers.sh) | The vocabulary scenarios are written in: prerequisite probes, cpuset derivation, the container and pod launchers. | +| [scenarios/](scenarios/) | One file per environment shape; every file here is a scenario. | +| [probe.py](probe.py) | Runs inside the prepared environment and prints one JSON object with what Crawlee sees there. | +| [wrap.py](wrap.py) | Folds the probe's output and the scenario's configuration into one result file. | +| [report.py](report.py) | Turns a results directory into the tables. `--check` is the only gate. | +| [.github/workflows/bench.yaml](.github/workflows/bench.yaml) | Manual trigger only, one job per CPU budget. | + +## Scenarios + +A scenario declares what it needs, what it configures, and how to run the probe in the environment it prepares. +The commands use the declared values rather than repeating them, so the report's `set` column is literally what +was applied. + +```bash +SCENARIO_DESC="private cgroupns (docker's default), all three axes at once" +REQUIRES="engine min${BENCH_CPUS}cpu" # unmet -> skipped, with this reason in the table + +QUOTA=$(awk "BEGIN{print $BENCH_CPUS - 0.5}") + +SET_MEMORY_BYTES=$((512 * 1024 * 1024)) # an axis left undeclared means the scenario restricts nothing +SET_CPU_CORES=$QUOTA # there, so the sensor reporting nothing is the right answer +SET_CPUSET_CORES=$BENCH_CPUS + +scenario_exec() { container_probe "$1" -m "$SET_MEMORY_BYTES" --cpus "$QUOTA" --cpuset-cpus "$(cpuset_list)"; } +``` + +`scenario_exec` must keep its stdout pure probe JSON, so setup noise goes to stderr; an optional +`scenario_cleanup` runs in a trap. A new scenario is one such file — it is picked up and reported automatically. + +| scenario | shape it exercises | +| --- | --- | +| `bare` | no limits at all, so every reading should fall back to host values | +| `systemd-own` | deep chain, limits on the process's own cgroup, cpuset delegated | +| `systemd-ancestor` | limit on an ancestor while the leaf carries no memory controller files | +| `docker-private` | private cgroupns (docker's default), all three axes at once | +| `docker-memory-only` | memory alone, CPU falls back to host | +| `docker-cpuset-only` | cpuset alone, memory falls back to host | +| `docker-host-ns` | `--cgroupns=host`, where the container sees the full chain instead of its own root | +| `docker-cgroup-parent` | limit on a parent cgroup, set outside the container (needs docker's systemd driver) | +| `k8s-limits` | a pod with container limits, as kubelet writes them | +| `k8s-no-limits` | a pod with nothing set, so every reading should fall back to the node's values | + +## Results + +One `results/.json` per scenario with the scenario's `configured` values, its `status` (`ok`, +`probe_failed`, `skipped`) and everything the probe printed: + +| section | contents | +| --- | --- | +| `sensor` | what `crawlee._utils.cgroup` reports: memory limit and working set, cpu quota, cpuset size, cpu time, and the levels it walked | +| `derived` | what `crawlee._utils.system` makes of that in `get_memory_info()` and `get_cpu_info()` | +| `evidence` | verbatim contents of the cgroup control files at every level, never interpreted | +| `host` | kernel, cores and RAM of the machine | +| `errors` | readings that raised; the value becomes `null` and the probe carries on | + +`report.py` renders two tables. In the first, each limit has a `set` column next to a `read` one: a value under +`set` with a dash under `read` is a limit the sensor missed, and dashes under both mean the scenario restricts +nothing there and the sensor agrees. `mem in use` is the memory charged against the limit excluding reclaimable +page cache — what `docker stats` shows. The second table is the same run seen through `get_memory_info()` and +`get_cpu_info()`, where a limit either reaches the caller or falls back to host values. When a reading looks +wrong, `evidence` says who is at fault: the limit is in the control files, or it never got there. diff --git a/probe.py b/probe.py new file mode 100644 index 0000000..1e0c1ec --- /dev/null +++ b/probe.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +import json +import os +import sys +from importlib.metadata import version +from pathlib import Path +from typing import Any + +SCHEMA = 1 + +_EVIDENCE_FILES = ( + 'memory.max', + 'memory.current', + 'cpu.max', + 'cpu.weight', + 'cpuset.cpus.effective', + 'cgroup.controllers', +) + +_MAX_LEVELS = 20 +"""How far up the cgroup chain the evidence dump walks. Deeper than any real hierarchy, so it only bounds a loop.""" + +_FILE_CAP_CHARS = 200 + +errors: list[str] = [] + + +def _guard(where: str, read: Any, default: Any = None) -> Any: + """Run a reading that touches the library under test, recording rather than raising when it breaks. + + The probe imports private APIs from a moving branch. A rename there should degrade one column to null, not + take the whole measurement down with it. + """ + try: + return read() + except Exception as exc: + errors.append(f'{where}: {type(exc).__name__}: {exc}') + return default + + +def _host() -> dict[str, Any]: + """Host facts, read directly - a cpuset only means something next to the machine's core count.""" + mem_total = None + try: + for line in Path('/proc/meminfo').read_text().splitlines(): + if line.startswith('MemTotal:'): + mem_total = int(line.split()[1]) * 1024 + break + except OSError: + pass + return {'kernel': os.uname().release, 'ncpu': os.cpu_count(), 'mem_total_bytes': mem_total} + + +def _read_control_file(path: Path) -> str | None: + """Read one cgroup control file, or None when it does not exist at this level.""" + try: + return path.read_text()[:_FILE_CAP_CHARS].strip() + except OSError: + return None + + +def _evidence() -> dict[str, Any]: + """Dump raw cgroup file contents along the chain, verbatim - receipts, not a verdict, never interpreted.""" + evidence: dict[str, Any] = {'proc_cgroup': None, 'levels': []} + try: + evidence['proc_cgroup'] = Path('/proc/self/cgroup').read_text().strip() + except OSError: + return evidence + + own_path = next((line[3:] for line in evidence['proc_cgroup'].splitlines() if line.startswith('0::')), None) + if own_path is None: + return evidence + + mount = Path('/sys/fs/cgroup') + directory = mount / own_path.lstrip('/') if '..' not in own_path else mount + for _ in range(_MAX_LEVELS): + files = {name: content for name in _EVIDENCE_FILES if (content := _read_control_file(directory / name))} + evidence['levels'].append({'path': str(directory), 'files': files}) + if directory in (mount, directory.parent): + break + directory = directory.parent + return evidence + + +def _sensor() -> dict[str, Any]: + """Collect everything `crawlee._utils.cgroup` reports for this process.""" + # Imported here rather than at module level: a rename on the branch under test must be recorded as an entry + # in `errors`, not kill the probe before it can report anything at all. + from crawlee._utils import cgroup + + readings: dict[str, Any] = { + 'is_v2': None, + 'memory_levels': None, + 'memory_limit_bytes': None, + 'working_set_bytes': None, + 'cpu_quota_cores': _guard('get_cpu_quota', cgroup.get_cpu_quota), + 'cpu_set_cores': _guard('get_cpu_set_size', cgroup.get_cpu_set_size), + 'cpu_usage_seconds': _guard('get_cpu_usage', cgroup.get_cpu_usage), + } + + memory_limit = _guard('get_memory_limit', cgroup.get_memory_limit) + if memory_limit is not None: + readings['memory_limit_bytes'] = memory_limit.limit + readings['working_set_bytes'] = memory_limit.working_set + + def discovery() -> dict[str, Any]: + memory = cgroup._get_controllers().memory + return { + 'is_v2': memory.is_v2 if memory is not None else None, + 'memory_levels': [str(directory) for directory in memory.dirs] if memory is not None else None, + } + + readings.update(_guard('_get_controllers', discovery, default={})) + return readings + + +def _derived() -> dict[str, Any]: + """Collect what `crawlee._utils.system` makes of those readings - the figures a crawler acts on. + + `get_memory_info()` substitutes the cgroup limit and its working set for the host totals whenever a limit + applies, and `get_cpu_info()` measures utilization against the cores the cgroup allows instead of the whole + machine. Where no limit is found both fall back to host-wide values, so these fields also show what a missed + limit would cost. + """ + from crawlee._utils.system import get_cpu_info, get_memory_info + + memory = _guard('get_memory_info', get_memory_info) + cpu = _guard('get_cpu_info', get_cpu_info) + + def allowed_cores() -> float | None: + from crawlee._utils.system import _get_allowed_cpu_cores + + return _get_allowed_cpu_cores() + + return { + 'total_size_bytes': memory.total_size.bytes if memory is not None else None, + 'system_wide_used_bytes': memory.system_wide_used_size.bytes if memory is not None else None, + 'current_size_bytes': memory.current_size.bytes if memory is not None else None, + 'allowed_cpu_cores': _guard('_get_allowed_cpu_cores', allowed_cores), + 'cpu_used_ratio': round(cpu.used_ratio, 3) if cpu is not None else None, + } + + +def main() -> None: + """Print one JSON object describing what this environment looks like to Crawlee.""" + source = sys.argv[sys.argv.index('--source') + 1] if '--source' in sys.argv else None + + report = { + 'schema': SCHEMA, + 'target': { + 'name': 'crawlee-python', + 'version': _guard('version', lambda: version('crawlee')), + 'source': source, + 'python': sys.version.split()[0], + }, + 'host': _host(), + 'evidence': _guard('evidence', _evidence, default={}), + 'sensor': _sensor(), + 'derived': _guard('derived', _derived, default={}), + 'errors': errors, + } + + json.dump(report, sys.stdout, indent=1) + sys.stdout.write('\n') + + +if __name__ == '__main__': + main() diff --git a/report.py b/report.py new file mode 100644 index 0000000..e264f75 --- /dev/null +++ b/report.py @@ -0,0 +1,215 @@ +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path +from typing import Any + +_BYTES_PER_GB = 1024**3 +_BYTES_PER_MB = 1024**2 + + +def _bytes_h(value: Any) -> str: + if not isinstance(value, (int, float)): + return '-' + if value >= _BYTES_PER_GB: + return f'{value / _BYTES_PER_GB:.2f} GB' + return f'{value / _BYTES_PER_MB:.2f} MB' + + +def _num(value: Any) -> str: + if isinstance(value, float): + return f'{value:g}' + if isinstance(value, int): + return str(value) + return '-' + + +def _bool(value: Any) -> str: + if value is True: + return 'yes' + if value is False: + return 'no' + return '-' + + +def _seconds(value: Any) -> str: + return f'{value:.2f}s' if isinstance(value, (int, float)) else '-' + + +def _percent(value: Any) -> str: + return f'{value:.0%}' if isinstance(value, (int, float)) else '-' + + +def _in_use(sensor: dict[str, Any]) -> str: + """Memory in use, with its share of the limit it is charged against.""" + in_use = sensor.get('working_set_bytes') + limit = sensor.get('memory_limit_bytes') + if not isinstance(in_use, (int, float)): + return '-' + if isinstance(limit, (int, float)) and limit > 0: + return f'{_bytes_h(in_use)} ({in_use / limit:.0%})' + return _bytes_h(in_use) + + +def _status(result: dict[str, Any]) -> str: + status = result.get('status', '?') + if status == 'probe_failed': + return '**PROBE FAILED**' + if status == 'skipped': + return f'skipped: {result.get("skip_reason", "?")}' + return status + + +def _header(results: list[dict[str, Any]]) -> list[str]: + """Identify the run: what was measured, and on what machine.""" + target = next((r.get('target') for r in results if r.get('target')), None) or {} + host = next(((r.get('probe') or {}).get('host') for r in results if (r.get('probe') or {}).get('host')), {}) or {} + + runner = [ + f'kernel `{host.get("kernel", "?")}`', + f'{host.get("ncpu", "?")} cpus', + f'{_bytes_h(host.get("mem_total_bytes"))} RAM', + ] + # GitHub spells these two in mixed case; they name the runner image the results came from. + image_os = os.environ.get('ImageOS') + image_version = os.environ.get('ImageVersion') + if image_os or image_version: + runner.append(f'image `{image_os}/{image_version}`') + + probed = next(((r.get('probe') or {}).get('target') for r in results if (r.get('probe') or {}).get('target')), {}) + installed = '' + if probed: + installed = f' → crawlee {probed.get("version")} on python {probed.get("python")}' + + lines = [ + '## cgroups-sensor bench', + '', + f'target: `{target.get("repo", "?")}` @ `{target.get("ref", "?")}`{installed}', + 'runner: ' + ', '.join(runner), + ] + budget = next((r.get('cpu_budget') for r in results if r.get('cpu_budget')), None) + if budget: + lines.append(f'cpu budget of the limited scenarios: {budget} core(s)') + + # The engine is per scenario (the podman-* rows set their own), so it belongs in the result files rather + # than the header. The image is one setting for the whole pass. + image = next((r.get('image') for r in results if r.get('image')), None) + if image: + lines.append(f'container image: `{image}`') + + lines.append('') + return lines + + +def _sensor_table(results: list[dict[str, Any]]) -> list[str]: + """Render the sensor's readings, each next to what the scenario configured.""" + lines = [ + '### What the cgroup sensor read', + '', + ( + '| scenario | status | mem limit set | mem limit read | quota set | quota read | cpuset set ' + '| cpuset read | mem in use | levels | v2 | cpu time | errors |' + ), + '| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | ---: | --- |', + ] + for result in results: + sensor = (result.get('probe') or {}).get('sensor') or {} + configured = result.get('configured') or {} + levels = sensor.get('memory_levels') + errors = (result.get('probe') or {}).get('errors') or [] + lines.append( + f'| {result.get("scenario", "?")} | {_status(result)} ' + f'| {_bytes_h(configured.get("memory_bytes"))} | {_bytes_h(sensor.get("memory_limit_bytes"))} ' + f'| {_num(configured.get("cpu_cores"))} | {_num(sensor.get("cpu_quota_cores"))} ' + f'| {_num(configured.get("cpuset_cores"))} | {_num(sensor.get("cpu_set_cores"))} ' + f'| {_in_use(sensor)} | {len(levels) if isinstance(levels, list) else "-"} ' + f'| {_bool(sensor.get("is_v2"))} | {_seconds(sensor.get("cpu_usage_seconds"))} ' + f'| {f"{len(errors)} err" if errors else ""} |' + ) + lines.append('') + return lines + + +def _derived_table(results: list[dict[str, Any]]) -> list[str]: + """Render what the library derives from those readings - the figures a crawler acts on.""" + lines = [ + '### What the library derives from them', + '', + '| scenario | memory available | used in that scope | this process | allowed cores | cpu used |', + '| --- | ---: | ---: | ---: | ---: | ---: |', + ] + for result in results: + derived = (result.get('probe') or {}).get('derived') or {} + lines.append( + f'| {result.get("scenario", "?")} | {_bytes_h(derived.get("total_size_bytes"))} ' + f'| {_bytes_h(derived.get("system_wide_used_bytes"))} | {_bytes_h(derived.get("current_size_bytes"))} ' + f'| {_num(derived.get("allowed_cpu_cores"))} | {_percent(derived.get("cpu_used_ratio"))} |' + ) + lines.append('') + return lines + + +_TAIL_LINES = 25 + + +def _failures(results: list[dict[str, Any]]) -> list[dict[str, Any]]: + """The rows that produced no data at all. + + A skip is not one of these: it is reported in the table with its reason, and a host that cannot host a + scenario is a normal outcome, not a breakage. + """ + return [result for result in results if result.get('status') == 'probe_failed'] + + +def _failure_report(failures: list[dict[str, Any]]) -> list[str]: + """Show what a failed row printed, so the run diagnoses itself instead of sending the reader to the artifact.""" + lines = ['### Failures', ''] + for result in failures: + lines.append(f'**{result.get("scenario", "?")}** — probe failed, exit {result.get("exit")}') + if result.get('probe_parse_error'): + lines.append(f'output was not JSON: {result["probe_parse_error"]}') + tail = (result.get('stderr_tail') or '').strip().splitlines()[-_TAIL_LINES:] + if tail: + lines += ['', '```', *tail, '```'] + lines.append('') + return lines + + +def render(results: list[dict[str, Any]]) -> str: + """Render one bench run as markdown.""" + lines = _header(results) + _sensor_table(results) + _derived_table(results) + + failures = _failures(results) + if failures: + lines += _failure_report(failures) + + return '\n'.join(lines) + + +def main(argv: list[str]) -> int: + """Print the report; with `--check`, exit non-zero when a probe failed.""" + check = '--check' in argv[1:] + positional = [arg for arg in argv[1:] if arg != '--check'] + results_dir = positional[0] if positional else 'results' + + results = [] + for path in sorted(Path(results_dir).glob('*.json')): + with path.open() as file: + results.append(json.load(file)) + + if not results: + print('no results found') + return 1 if check else 0 + + sys.stdout.write(render(results)) + + if check and (failures := _failures(results)): + print(f'check: {len(failures)} probe failure(s)', file=sys.stderr) + return 1 + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..d7d3a6b --- /dev/null +++ b/run.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash +# Bench driver. Runs scenarios sequentially and ALWAYS writes one result per scenario into the results +# directory - a crash becomes a recorded failure, never a missing row. Exit code is 0 unless the driver itself +# cannot start; judging the run is report.py's job (report.py --check). +# +# Usage: ./run.sh [all| [...]] [results_dir] +# A matches a scenario file with or without its NN- prefix (e.g. "bare" or "00-bare"). +# +# Environment: +# CRAWLEE_REPO / CRAWLEE_REF target to install (default: Mantisus/crawlee-python @ container-limits) +# BENCH_CPUS CPU budget for the limited scenarios, in cores (default 1). Scenarios larger +# than the host skip. Run the bench once per budget to cover more than one size. +# ENGINE container engine (default: docker) +# SCENARIO_TIMEOUT per-scenario timeout in seconds (default 300) + +set -u + +SENSOR_DIR=$(cd "$(dirname "$0")" && pwd) + +RESULTS="results" +SELECTORS=() +for arg in "$@"; do + if [ "$arg" = "all" ] || [ -f "$SENSOR_DIR/scenarios/$arg.sh" ]; then + SELECTORS+=("$arg") + else + RESULTS="$arg" + fi +done +[ ${#SELECTORS[@]} -gt 0 ] || SELECTORS=("all") + +ENGINE=${ENGINE:-docker} +IMG=${IMG:-ghcr.io/astral-sh/uv:python3.13-bookworm} +PY3=${PY3:-$(command -v python3)} +UV_BIN=${UV_BIN:-$(command -v uv || echo uv)} +BENCH_UV_CACHE=${BENCH_UV_CACHE:-/tmp/cgroups-bench-uv-cache} +# uv downloads interpreters here rather than into its cache; shared so only the first container of a run pays. +BENCH_UV_PYTHON=${BENCH_UV_PYTHON:-/tmp/cgroups-bench-uv-python} +CRAWLEE_REPO=${CRAWLEE_REPO:-https://github.com/Mantisus/crawlee-python} +CRAWLEE_REF=${CRAWLEE_REF:-container-limits} +BENCH_CPUS=${BENCH_CPUS:-1} + +export ENGINE IMG SENSOR_DIR BENCH_UV_CACHE BENCH_UV_PYTHON BENCH_CPUS UV_BIN +mkdir -p "$RESULTS" "$BENCH_UV_CACHE" "$BENCH_UV_PYTHON" + +# A git+ dependency makes uv run git inside the container, and most base images do not ship one. GitHub serves +# the same tree as a tarball, so an image needs nothing but network. Anything else falls back to git+. +case "$CRAWLEE_REPO" in + https://github.com/*) SOURCE="${CRAWLEE_REPO%.git}/archive/${CRAWLEE_REF}.tar.gz" ;; + *) SOURCE="git+${CRAWLEE_REPO}@${CRAWLEE_REF}" ;; +esac + +WITH_SPEC="crawlee @ $SOURCE" +# The interpreter is part of what gets measured, so it is pinned rather than left to whatever the image has. +PROBE_FLAGS="--source $SOURCE" + +# The probe command, in container paths (scenarios mount $SENSOR_DIR at /sensor) and in local paths. +# --refresh-package makes uv re-fetch the ref instead of trusting a cached resolution, so a re-run after pushing +# to the branch never silently measures the previous commit. +UV_ARGS="run --no-project --python ${BENCH_PYTHON:-3.13} --refresh-package crawlee --with \"$WITH_SPEC\"" +INNER_CONTAINER="uv $UV_ARGS python /sensor/probe.py $PROBE_FLAGS" +INNER_LOCAL="'$UV_BIN' $UV_ARGS python '$SENSOR_DIR/probe.py' $PROBE_FLAGS" + +TARGET_JSON=$(printf '{"repo": "%s", "ref": "%s"}' "$CRAWLEE_REPO" "$CRAWLEE_REF") + +echo "bench: $CRAWLEE_REPO @ $CRAWLEE_REF" +echo " image: $IMG, cpu budget: $BENCH_CPUS of $(nproc 2>/dev/null || echo '?') cores" + +selected() { + local name=$1 sel + for sel in "${SELECTORS[@]}"; do + [ "$sel" = "all" ] || [ "$sel" = "$name" ] && return 0 + done + return 1 +} + +run_one() { + local file=$1 + local name + name=$(basename "${file%.sh}") + local work + work=$(mktemp -d) + + ( + SCENARIO_NAME=$name + CONTAINER="bench-$name" + SCENARIO_DESC="" + REQUIRES="" + INNER_STYLE=local + # What the scenario configures, per axis. The report prints each next to what the sensor read, so a value + # left unset here means "this scenario deliberately restricts nothing on that axis". + SET_MEMORY_BYTES="" SET_CPU_CORES="" SET_CPUSET_CORES="" + + . "$SENSOR_DIR/scenario-helpers.sh" + . "$file" + + wrap() { + "$PY3" "$SENSOR_DIR/wrap.py" \ + --scenario "$name" --desc "$SCENARIO_DESC" \ + ${SET_MEMORY_BYTES:+--set-memory "$SET_MEMORY_BYTES"} \ + ${SET_CPU_CORES:+--set-cpu "$SET_CPU_CORES"} \ + ${SET_CPUSET_CORES:+--set-cpuset "$SET_CPUSET_CORES"} \ + --target-json "$TARGET_JSON" --out "$RESULTS/$name.json" "$@" + } + + unmet=$(unmet_requirement) + if [ -n "$unmet" ]; then + wrap --skip-reason "$unmet" + exit 0 + fi + + trap 'scenario_cleanup' EXIT + + INNER=$INNER_LOCAL + [ "$INNER_STYLE" = "container" ] && INNER=$INNER_CONTAINER + + # scenario_exec is a function, so `timeout` cannot wrap it directly; a watchdog in this shell kills a hang. + # The watchdog is detached from stdout/stderr - otherwise its orphaned sleep holds the pipe open and a + # `./run.sh | tee` style caller waits up to the full timeout for EOF after the run has finished. + scenario_exec "$INNER" >"$work/probe.json" 2>"$work/run.err" & + runpid=$! + ( sleep "${SCENARIO_TIMEOUT:-300}" && kill "$runpid" 2>/dev/null ) >/dev/null 2>&1 /dev/null + wait "$watchdog" 2>/dev/null + + wrap --probe-file "$work/probe.json" --exit-rc "$rc" --stderr-file "$work/run.err" + ) + + rm -rf "$work" + + # Baseline between scenarios: a leaked busy-loop would distort the next scenario's CPU readings. + leftovers=$(pgrep -f 'while :; do :; done' 2>/dev/null || true) + if [ -n "$leftovers" ]; then + echo "WARN: leftover busy-loop processes after $name, killing: $leftovers" >&2 + kill $leftovers 2>/dev/null || true + fi +} + +for file in "$SENSOR_DIR"/scenarios/*.sh; do + name=$(basename "${file%.sh}") + selected "$name" || continue + run_one "$file" +done diff --git a/scenario-helpers.sh b/scenario-helpers.sh new file mode 100644 index 0000000..481987a --- /dev/null +++ b/scenario-helpers.sh @@ -0,0 +1,172 @@ +# shellcheck shell=bash +# Shared helpers for scenario files. Sourced by run.sh inside each scenario's subshell, before the scenario file. + +BENCH_CLUSTER=${BENCH_CLUSTER:-cgroups-bench} +# A pod cannot mount the host's uv binary, so the k8s scenarios use an image that already carries one, +# independently of the $IMG the container scenarios run on. +K8S_IMAGE=${K8S_IMAGE:-ghcr.io/astral-sh/uv:python3.13-bookworm} +K8S_POD=bench-probe +K8S_CM=bench-probe + +have() { command -v "$1" >/dev/null 2>&1; } + +# $BENCH_CPUS is the run's CPU budget: how many cores the limited scenarios are given. Scenarios derive both +# their cpuset and their quota from it, so the same files stay meaningful on a 2-core runner and on a bigger +# host - and running the bench twice with different budgets is how the CPU axes get covered at more than one +# size. A budget larger than the host makes the scenario skip (see mincpu), never silently measure nothing. +cpuset_list() { [ "$BENCH_CPUS" -le 1 ] && echo '0' || echo "0-$((BENCH_CPUS - 1))"; } + +have_engine() { "$ENGINE" info >/dev/null 2>&1; } +have_sudo() { sudo -n true 2>/dev/null; } +have_systemd_user() { systemd-run --user --scope -q true 2>/dev/null; } +cgroup_driver() { "$ENGINE" info -f '{{.CgroupDriver}}' 2>/dev/null; } +ncpus() { nproc 2>/dev/null || getconf _NPROCESSORS_ONLN; } + +# Prints the first unmet requirement from $REQUIRES, or nothing when all are met. +unmet_requirement() { + local req n + for req in $REQUIRES; do + case "$req" in + engine) have_engine || { echo "container engine '$ENGINE' is not available"; return; } ;; + sudo) have_sudo || { echo "passwordless sudo is not available"; return; } ;; + systemd-user) have_systemd_user || { echo "systemd user manager is not available"; return; } ;; + systemd-driver) + [ "$(cgroup_driver)" = systemd ] || { echo "$ENGINE uses the '$(cgroup_driver)' cgroup driver, this scenario needs 'systemd'"; return; } ;; + kind) + have kind && have kubectl && have_engine || { echo "kind, kubectl and a container engine are needed to bring up a cluster"; return; } ;; + min*cpu) n=${req#min}; n=${n%cpu} + [ "$(ncpus)" -ge "$n" ] || { echo "needs at least $n CPUs, host has $(ncpus)"; return; } ;; + *) have "$req" || { echo "command '$req' is not available"; return; } ;; + esac + done +} + +# Default so run.sh can call it unconditionally. Scenarios override when they have something to clean up. +scenario_cleanup() { :; } + +# Run the probe command ($1) inside a fresh container; the remaining arguments are engine run flags. The engine +# is $ENGINE, which a scenario can set for itself, so rows from different engines can share a pass. uv and its +# downloaded interpreters are mounted in rather than baked into the image, so $IMG can be any glibc distro - +# fedora, rocky, debian - and only the first container of a run pays for the download. +container_probe() { + local inner=$1 + shift + + # A cache directory per engine. A rootful container writes into it as real root, and a rootless one - whose + # own root is this user - then cannot touch those files at all, so the two must never share. + local cache="$BENCH_UV_CACHE/$ENGINE" interpreters="$BENCH_UV_PYTHON/$ENGINE" + mkdir -p "$cache" "$interpreters" + + local mounts=( + -v "$SENSOR_DIR:/sensor:ro" + -v "$cache:/root/.cache/uv" + -v "$interpreters:/root/.local/share/uv/python" + ) + # The host's uv goes to a directory appended to PATH, never over the image's own copy: an image that ships uv + # keeps using it, and only an image without one falls back to the mounted binary. + [ -x "$UV_BIN" ] && mounts+=(-v "$UV_BIN:/opt/bench/uv:ro") + + "$ENGINE" run --rm --name "$CONTAINER" "${mounts[@]}" "$@" "$IMG" \ + sh -c 'PATH="$PATH:/opt/bench"; '"$inner" +} + +# Create the bench's kind cluster if it is not up yet, and make sure the probe image is on its node. +ensure_kind_cluster() { + if ! kind get clusters 2>/dev/null | grep -qx "$BENCH_CLUSTER"; then + kind create cluster --name "$BENCH_CLUSTER" --wait 120s + fi + "$ENGINE" image inspect "$K8S_IMAGE" >/dev/null 2>&1 || "$ENGINE" pull "$K8S_IMAGE" + kind load docker-image "$K8S_IMAGE" --name "$BENCH_CLUSTER" +} + +# The script the pod runs. `kubectl logs` merges the container's stdout and stderr, so the probe's diagnostics +# are kept in a file and printed only when it fails - otherwise uv's progress output would land in the JSON. +# It always exits 0: a pod that fails takes the full wait to report it, and a missing JSON body already says so. +k8s_run_script() { + cat </tmp/probe.json 2>/tmp/probe.err +rc=\$? +if [ \$rc -eq 0 ]; then + cat /tmp/probe.json +else + echo "probe exited \$rc" + cat /tmp/probe.err +fi +exit 0 +EOF +} + +# Wait for the pod to finish, printing its phase. Polling rather than `kubectl wait --for=jsonpath` so that a +# quirk of one kubectl version cannot silently return at once and leave the logs unread. A pod the scheduler +# cannot place would otherwise sit there for the whole timeout, so that case ends the wait immediately. +k8s_wait() { + local phase reason seconds=0 + + while [ "$seconds" -lt "${K8S_WAIT_SECONDS:-300}" ]; do + phase=$(kubectl get "pod/$K8S_POD" -o jsonpath='{.status.phase}' 2>/dev/null) + case "$phase" in + Succeeded | Failed) break ;; + esac + + if [ "$seconds" -ge 5 ]; then + reason=$(kubectl get "pod/$K8S_POD" \ + -o jsonpath='{.status.conditions[?(@.type=="PodScheduled")].reason}' 2>/dev/null) + if [ "$reason" = Unschedulable ]; then + echo "pod cannot be scheduled - its requests do not fit this node" + break + fi + fi + + sleep 1 + seconds=$((seconds + 1)) + done + + echo "pod phase: ${phase:-unknown} after ${seconds}s" + [ "$phase" = Succeeded ] || kubectl describe "pod/$K8S_POD" +} + +# Run the probe command ($1) in a one-shot pod whose container resources are the JSON object in $2, and print +# the pod's stdout. probe.py and the command itself travel as a ConfigMap, so nothing has to be quoted into +# YAML. Everything except the pod's own output goes to stderr, or it would end up mixed into the probe's JSON. +k8s_probe() { + local inner=$1 resources=$2 + + { + ensure_kind_cluster + kubectl delete pod "$K8S_POD" --ignore-not-found --now + kubectl create configmap "$K8S_CM" \ + --from-file=probe.py="$SENSOR_DIR/probe.py" --from-literal=run.sh="$(k8s_run_script "$inner")" \ + --dry-run=client -o yaml | kubectl apply -f - + + kubectl apply -f - <&2 + + # A copy of the pod's output goes to stderr, so when it is not JSON the result file still shows what it was. + kubectl logs "$K8S_POD" | tee /dev/stderr +} + +k8s_cleanup() { + kubectl delete pod "$K8S_POD" --ignore-not-found --now >/dev/null 2>&1 || true + kubectl delete configmap "$K8S_CM" --ignore-not-found >/dev/null 2>&1 || true +} diff --git a/scenarios/bare.sh b/scenarios/bare.sh new file mode 100644 index 0000000..a175792 --- /dev/null +++ b/scenarios/bare.sh @@ -0,0 +1,6 @@ +# shellcheck shell=bash +SCENARIO_DESC="no limits at all, every reading should fall back to host values" +REQUIRES="" +INNER_STYLE=local + +scenario_exec() { sh -c "$1"; } diff --git a/scenarios/docker-cgroup-parent.sh b/scenarios/docker-cgroup-parent.sh new file mode 100644 index 0000000..2b9ce0b --- /dev/null +++ b/scenarios/docker-cgroup-parent.sh @@ -0,0 +1,19 @@ +# shellcheck shell=bash +SCENARIO_DESC="limit on a parent cgroup, set outside the container" +REQUIRES="engine sudo systemd-driver" +INNER_STYLE=container + +SET_MEMORY_BYTES=$((512 * 1024 * 1024)) + +SLICE=bench-parent.slice +HOLDER=bench-parent-holder + +# Under the systemd cgroup driver --cgroup-parent takes a slice, and a slice needs a live unit in it to exist. +# The limit goes on that slice; the container itself gets none, so the sensor has to walk up to find it. +scenario_exec() { + sudo systemd-run -q --unit "$HOLDER" --slice "$SLICE" sleep infinity >&2 + sudo systemctl set-property --runtime "$SLICE" "MemoryMax=$SET_MEMORY_BYTES" >&2 + container_probe "$1" --cgroupns=host --cgroup-parent="$SLICE" +} + +scenario_cleanup() { sudo systemctl stop "$HOLDER" "$SLICE" 2>/dev/null || true; } diff --git a/scenarios/docker-cpuset-only.sh b/scenarios/docker-cpuset-only.sh new file mode 100644 index 0000000..05c40a9 --- /dev/null +++ b/scenarios/docker-cpuset-only.sh @@ -0,0 +1,8 @@ +# shellcheck shell=bash +SCENARIO_DESC="cpuset alone, memory falls back to host" +REQUIRES="engine min${BENCH_CPUS}cpu" +INNER_STYLE=container + +SET_CPUSET_CORES=$BENCH_CPUS + +scenario_exec() { container_probe "$1" --cpuset-cpus "$(cpuset_list)"; } diff --git a/scenarios/docker-host-ns.sh b/scenarios/docker-host-ns.sh new file mode 100644 index 0000000..0e1f4f9 --- /dev/null +++ b/scenarios/docker-host-ns.sh @@ -0,0 +1,8 @@ +# shellcheck shell=bash +SCENARIO_DESC="cgroupns=host, where the container sees the full chain instead of its own root" +REQUIRES="engine" +INNER_STYLE=container + +SET_MEMORY_BYTES=$((512 * 1024 * 1024)) + +scenario_exec() { container_probe "$1" -m "$SET_MEMORY_BYTES" --cgroupns=host; } diff --git a/scenarios/docker-memory-only.sh b/scenarios/docker-memory-only.sh new file mode 100644 index 0000000..6a3d86c --- /dev/null +++ b/scenarios/docker-memory-only.sh @@ -0,0 +1,8 @@ +# shellcheck shell=bash +SCENARIO_DESC="memory alone, CPU falls back to host" +REQUIRES="engine" +INNER_STYLE=container + +SET_MEMORY_BYTES=$((512 * 1024 * 1024)) + +scenario_exec() { container_probe "$1" -m "$SET_MEMORY_BYTES"; } diff --git a/scenarios/docker-private.sh b/scenarios/docker-private.sh new file mode 100644 index 0000000..c4495ac --- /dev/null +++ b/scenarios/docker-private.sh @@ -0,0 +1,13 @@ +# shellcheck shell=bash +SCENARIO_DESC="private cgroupns (docker's default), all three axes at once" +REQUIRES="engine min${BENCH_CPUS}cpu" +INNER_STYLE=container + +# Half a core tighter than the cpuset, so the row also shows which of the two CPU axes wins. +QUOTA=$(awk "BEGIN{print $BENCH_CPUS - 0.5}") + +SET_MEMORY_BYTES=$((512 * 1024 * 1024)) +SET_CPU_CORES=$QUOTA +SET_CPUSET_CORES=$BENCH_CPUS + +scenario_exec() { container_probe "$1" -m "$SET_MEMORY_BYTES" --cpus "$QUOTA" --cpuset-cpus "$(cpuset_list)"; } diff --git a/scenarios/k8s-limits.sh b/scenarios/k8s-limits.sh new file mode 100644 index 0000000..2bbd930 --- /dev/null +++ b/scenarios/k8s-limits.sh @@ -0,0 +1,23 @@ +# shellcheck shell=bash +SCENARIO_DESC="pod with container limits, read from inside its own cgroup namespace" +REQUIRES="kind" +INNER_STYLE=container +SCENARIO_TIMEOUT=900 # the first k8s scenario of a run brings the cluster up + +# Half a core tighter than the budget, as in docker-private. Requests are kept small on purpose: kubernetes +# copies the limits into the requests when none are given, and a pod requesting every core of the node is one +# the scheduler can never place. +MILLICORES=$((BENCH_CPUS * 1000 - 500)) + +SET_MEMORY_BYTES=$((512 * 1024 * 1024)) +SET_CPU_CORES=$(awk "BEGIN{print $BENCH_CPUS - 0.5}") + +# kubelet nests a pod several levels under kubepods, but the container gets a cgroup namespace of its own, so +# from inside the chain is one level deep and the limit sits on it. The walk up to an ancestor is what +# docker-cgroup-parent covers; here what matters is that kubelet's shape is read correctly at all. +scenario_exec() { + k8s_probe "$1" "$(printf '{"requests": {"cpu": "50m", "memory": "64Mi"}, "limits": {"memory": "%s", "cpu": "%sm"}}' \ + "$SET_MEMORY_BYTES" "$MILLICORES")" +} + +scenario_cleanup() { k8s_cleanup; } diff --git a/scenarios/k8s-no-limits.sh b/scenarios/k8s-no-limits.sh new file mode 100644 index 0000000..4abbfeb --- /dev/null +++ b/scenarios/k8s-no-limits.sh @@ -0,0 +1,10 @@ +# shellcheck shell=bash +SCENARIO_DESC="pod without limits, inside the same kubepods hierarchy" +REQUIRES="kind" +INNER_STYLE=container +SCENARIO_TIMEOUT=900 + +# Same nesting as k8s-limits but nothing set, so every reading should fall back to the node's values. +scenario_exec() { k8s_probe "$1" '{}'; } + +scenario_cleanup() { k8s_cleanup; } diff --git a/scenarios/systemd-ancestor.sh b/scenarios/systemd-ancestor.sh new file mode 100644 index 0000000..d401fac --- /dev/null +++ b/scenarios/systemd-ancestor.sh @@ -0,0 +1,18 @@ +# shellcheck shell=bash +SCENARIO_DESC="limit on an ancestor, leaf without memory controller files" +REQUIRES="systemd-user" +INNER_STYLE=local + +SET_MEMORY_BYTES=$((512 * 1024 * 1024)) + +# The scope carries the limit; Delegate=yes lets the probe move into a child cgroup that inherits no memory +# controller files at all, so the sensor has to walk up past a level that has nothing to read. +scenario_exec() { + INNER="$1" systemd-run --user --scope -q --unit "bench-ancestor-$$" \ + -p "MemoryMax=$SET_MEMORY_BYTES" -p Delegate=yes bash -c ' + own=$(grep "^0::" /proc/self/cgroup | cut -d: -f3) + mkdir -p "/sys/fs/cgroup$own/leaf" + echo $$ > "/sys/fs/cgroup$own/leaf/cgroup.procs" + exec sh -c "$INNER" + ' +} diff --git a/scenarios/systemd-own.sh b/scenarios/systemd-own.sh new file mode 100644 index 0000000..8cd1ae8 --- /dev/null +++ b/scenarios/systemd-own.sh @@ -0,0 +1,23 @@ +# shellcheck shell=bash +SCENARIO_DESC="deep chain, limits on the process's own cgroup, cpuset delegated" +REQUIRES="sudo systemd-run min${BENCH_CPUS}cpu" +INNER_STYLE=local + +# A system scope, not a user one: user slices do not delegate cpuset, so AllowedCPUs would be silently ignored. +# The quota is a core looser than the cpuset here, the mirror of docker-private, where it is tighter. +QUOTA_CORES=$((BENCH_CPUS + 1)) + +SET_MEMORY_BYTES=$((512 * 1024 * 1024)) +SET_CPU_CORES=$QUOTA_CORES +SET_CPUSET_CORES=$BENCH_CPUS + +UNIT="bench-systemd-own-$$" + +# sudo strips the environment, so the probe needs PATH and a cache root can write to. +scenario_exec() { + sudo systemd-run --scope -q --unit "$UNIT" \ + -p "MemoryMax=$SET_MEMORY_BYTES" -p "CPUQuota=$((QUOTA_CORES * 100))%" -p "AllowedCPUs=$(cpuset_list)" \ + env "PATH=$PATH" "HOME=/root" "UV_CACHE_DIR=$BENCH_UV_CACHE/root" sh -c "$1" +} + +scenario_cleanup() { sudo systemctl stop "$UNIT.scope" 2>/dev/null || true; } diff --git a/wrap.py b/wrap.py new file mode 100644 index 0000000..15660f0 --- /dev/null +++ b/wrap.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import json +import os +import sys +import time +from pathlib import Path +from typing import Any + +SCHEMA = 1 + +_STDERR_TAIL_CHARS = 8000 +_CONSOLE_TAIL_LINES = 15 + + +def _read_tail(path: str | None, limit: int = _STDERR_TAIL_CHARS) -> str: + if not path: + return '' + try: + return Path(path).read_text(errors='replace')[-limit:] + except OSError: + return '' + + +def _parse_args(argv: list[str]) -> dict[str, str]: + """Parse `--key value` and `--flag` pairs; run.sh is the only caller.""" + args: dict[str, str] = {} + index = 0 + while index < len(argv): + arg = argv[index] + if not arg.startswith('--'): + raise SystemExit(f'wrap.py: unexpected argument {arg!r}') + key = arg.removeprefix('--') + if index + 1 < len(argv) and not argv[index + 1].startswith('--'): + args[key] = argv[index + 1] + index += 2 + else: + args[key] = '1' + index += 1 + return args + + +def main(argv: list[str]) -> int: + """Write one scenario's result JSON and print its status line.""" + args = _parse_args(argv[1:]) + + probe: Any = None + parse_error = None + if args.get('probe-file'): + try: + probe = json.loads(Path(args['probe-file']).read_text()) + except (OSError, json.JSONDecodeError) as exc: + parse_error = str(exc) + + result: dict[str, Any] = { + 'schema': SCHEMA, + 'scenario': args['scenario'], + 'desc': args.get('desc', ''), + 'cpu_budget': os.environ.get('BENCH_CPUS'), + 'engine': os.environ.get('ENGINE'), + 'image': os.environ.get('IMG'), + # What the scenario configured. A None means it restricts nothing on that axis, so the sensor reporting + # nothing there is the right answer rather than a miss. + 'configured': { + 'memory_bytes': int(args['set-memory']) if args.get('set-memory') else None, + 'cpu_cores': float(args['set-cpu']) if args.get('set-cpu') else None, + 'cpuset_cores': int(args['set-cpuset']) if args.get('set-cpuset') else None, + }, + 'target': json.loads(args.get('target-json', 'null')), + 'probe': probe, + 'exit': int(args['exit-rc']) if args.get('exit-rc') else None, + 'stderr_tail': _read_tail(args.get('stderr-file')), + 'timestamp': time.strftime('%Y-%m-%dT%H:%M:%S%z'), + } + + if args.get('skip-reason'): + result['status'] = 'skipped' + result['skip_reason'] = args['skip-reason'] + elif result['exit'] == 0 and probe is not None: + result['status'] = 'ok' + else: + result['status'] = 'probe_failed' + if parse_error: + result['probe_parse_error'] = parse_error + + out = Path(args['out']) + out.parent.mkdir(parents=True, exist_ok=True) + with out.open('w') as file: + json.dump(result, file, indent=1) + file.write('\n') + + detail = result.get('skip_reason', '') + if result['status'] == 'probe_failed': + detail = f'exit {result["exit"]}' + (f', {parse_error}' if parse_error else '') + print(f'{args["scenario"]}: {result["status"]}' + (f' ({detail})' if detail else '')) + + # A failure is worth reading where it happens, not only in the artifact. + if result['status'] == 'probe_failed': + for line in result['stderr_tail'].strip().splitlines()[-_CONSOLE_TAIL_LINES:]: + print(f' | {line}') + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv))