From a53354a4d82d5b8c171a0e581caaea18e2c5b9a8 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Wed, 19 Aug 2026 17:38:04 +0000 Subject: [PATCH 1/3] clean analysis data --- .github/workflows/bench.yaml | 115 ------------- .gitignore | 4 +- probe.py | 212 ------------------------ report.py | 216 ------------------------- run.sh | 145 ----------------- scenario-helpers.sh | 179 -------------------- scenarios/bare.sh | 6 - scenarios/docker-cgroup-parent.sh | 19 --- scenarios/docker-cpuset-only.sh | 8 - scenarios/docker-host-ns.sh | 8 - scenarios/docker-memory-only.sh | 8 - scenarios/docker-nested-subgroup.sh | 26 --- scenarios/docker-private.sh | 13 -- scenarios/k8s-limits.sh | 23 --- scenarios/k8s-no-limits.sh | 10 -- scenarios/systemd-ancestor.sh | 18 --- scenarios/systemd-memory-above-host.sh | 16 -- scenarios/systemd-own.sh | 33 ---- scenarios/systemd-quota-above-host.sh | 18 --- v1-guest.sh | 176 -------------------- wrap.py | 105 ------------ 21 files changed, 3 insertions(+), 1355 deletions(-) delete mode 100644 .github/workflows/bench.yaml delete mode 100644 probe.py delete mode 100644 report.py delete mode 100755 run.sh delete mode 100644 scenario-helpers.sh delete mode 100644 scenarios/bare.sh delete mode 100644 scenarios/docker-cgroup-parent.sh delete mode 100644 scenarios/docker-cpuset-only.sh delete mode 100644 scenarios/docker-host-ns.sh delete mode 100644 scenarios/docker-memory-only.sh delete mode 100644 scenarios/docker-nested-subgroup.sh delete mode 100644 scenarios/docker-private.sh delete mode 100644 scenarios/k8s-limits.sh delete mode 100644 scenarios/k8s-no-limits.sh delete mode 100644 scenarios/systemd-ancestor.sh delete mode 100644 scenarios/systemd-memory-above-host.sh delete mode 100644 scenarios/systemd-own.sh delete mode 100644 scenarios/systemd-quota-above-host.sh delete mode 100755 v1-guest.sh delete mode 100644 wrap.py diff --git a/.github/workflows/bench.yaml b/.github/workflows/bench.yaml deleted file mode 100644 index 966dcfe..0000000 --- a/.github/workflows/bench.yaml +++ /dev/null @@ -1,115 +0,0 @@ -name: bench - -# The bench runs when a human triggers it, against whatever repo+ref they name. No schedules, no push triggers - -# this is an on-demand measurement instrument, not a monitor. - -on: - workflow_dispatch: - inputs: - repo: - description: Git repository holding the code under test - default: 'https://github.com/Mantisus/crawlee-python' - ref: - description: Branch, tag or SHA to install - default: 'container-limits' - scenarios: - description: Space-separated scenario names, or "all" - default: 'all' - cpu_budgets: - description: 'Cores for the CPU-limited scenarios, one run per value. Runner has 2 cores (private repo) or 4 (public repo)' - default: '[1, 2]' - image: - description: 'Base image the container scenarios run in. Any glibc distro works - uv is mounted in' - default: 'ghcr.io/astral-sh/uv:python3.13-bookworm' - cgroup_v1: - description: 'Also run the bench in a guest booted on cgroup v1 (adds a few minutes)' - type: choice - options: ['off', hybrid, legacy] - default: 'off' - -run-name: 'bench: ${{ inputs.ref }} / ${{ inputs.scenarios }} / cpus ${{ inputs.cpu_budgets }}' - -permissions: - contents: read - -jobs: - bench: - # One pass per CPU budget, each on its own runner, so a smaller budget's result is still there when a bigger - # one turns out to exceed the runner. - strategy: - fail-fast: false - matrix: - cpus: ${{ fromJSON(inputs.cpu_budgets) }} - name: bench (${{ matrix.cpus }} cpu) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: astral-sh/setup-uv@v5 - - - name: Run scenarios - env: - CRAWLEE_REPO: ${{ inputs.repo }} - CRAWLEE_REF: ${{ inputs.ref }} - BENCH_CPUS: ${{ matrix.cpus }} - IMG: ${{ inputs.image }} - run: ./run.sh ${{ inputs.scenarios }} results/ - - - name: Report - if: always() - run: python3 report.py results/ >> "$GITHUB_STEP_SUMMARY" - - - name: Check - # The only gate: a probe that produced no data. Skips and observed values are for humans. - if: always() - run: python3 report.py results/ --check >/dev/null - - - name: Upload raw results - if: always() - uses: actions/upload-artifact@v4 - with: - name: results-${{ matrix.cpus }}cpu - path: results/ - - bench-v1: - # The runner is cgroup v2 only, and a controller cannot be mounted as v1 while the unified hierarchy owns - # it, so the v1 side of the library is only reachable inside a guest booted in legacy mode. - if: inputs.cgroup_v1 != 'off' - name: bench (cgroup v1 guest) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: astral-sh/setup-uv@v5 - - - name: Allow KVM - run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ - | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - - - name: Install qemu - run: sudo apt-get install -y --no-install-recommends qemu-system-x86 cloud-image-utils - - - name: Boot the guest and run the bench inside it - env: - CRAWLEE_REPO: ${{ inputs.repo }} - CRAWLEE_REF: ${{ inputs.ref }} - V1_MODE: ${{ inputs.cgroup_v1 }} - run: ./v1-guest.sh results/ - - - name: Report - if: always() - run: python3 report.py results/ >> "$GITHUB_STEP_SUMMARY" - - - name: Check - if: always() - run: python3 report.py results/ --check >/dev/null - - - name: Upload raw results - if: always() - uses: actions/upload-artifact@v4 - with: - name: results-cgroup-v1-${{ inputs.cgroup_v1 }} - path: results/ diff --git a/.gitignore b/.gitignore index fe0f660..eb02977 100644 --- a/.gitignore +++ b/.gitignore @@ -215,4 +215,6 @@ marimo/_lsp/ __marimo__/ # Streamlit -.streamlit/secrets.toml \ No newline at end of file +.streamlit/secrets.toml +# Analysis material: the bench that validated the implementation, kept out of the module. +analysis/ diff --git a/probe.py b/probe.py deleted file mode 100644 index 7ffec8d..0000000 --- a/probe.py +++ /dev/null @@ -1,212 +0,0 @@ -from __future__ import annotations - -import json -import os -import sys -from importlib.metadata import version -from pathlib import Path -from typing import Any - -SCHEMA = 1 - -_V2_EVIDENCE_FILES = ( - 'memory.max', - 'memory.current', - 'cpu.max', - 'cpu.weight', - 'cpuset.cpus.effective', - 'cgroup.controllers', -) - -_V1_EVIDENCE_FILES = ( - 'memory.limit_in_bytes', - 'memory.usage_in_bytes', - 'cpu.cfs_quota_us', - 'cpu.cfs_period_us', - 'cpuacct.usage', - 'cpuset.cpus', -) - -_V1_CONTROLLERS = ('memory', 'cpu', 'cpuacct', 'cpuset') - -_MAX_LEVELS = 20 -"""How far up the cgroup chain the evidence dump walks. Deeper than any real hierarchy, so it only bounds a loop.""" - -_FILE_CAP_CHARS = 200 - -errors: list[str] = [] - - -def _guard(where: str, read: Any, default: Any = None) -> Any: - """Run a reading that touches the library under test, recording rather than raising when it breaks. - - The probe imports private APIs from a moving branch. A rename there should degrade one column to null, not - take the whole measurement down with it. - """ - try: - return read() - except Exception as exc: - errors.append(f'{where}: {type(exc).__name__}: {exc}') - return default - - -def _host() -> dict[str, Any]: - """Host facts, read directly - a cpuset only means something next to the machine's core count.""" - mem_total = None - try: - for line in Path('/proc/meminfo').read_text().splitlines(): - if line.startswith('MemTotal:'): - mem_total = int(line.split()[1]) * 1024 - break - except OSError: - pass - return {'kernel': os.uname().release, 'ncpu': os.cpu_count(), 'mem_total_bytes': mem_total} - - -def _read_control_file(path: Path) -> str | None: - """Read one cgroup control file, or None when it does not exist at this level.""" - try: - return path.read_text()[:_FILE_CAP_CHARS].strip() - except OSError: - return None - - -def _walk_levels(mount: Path, own_path: str, names: tuple[str, ...]) -> list[dict[str, Any]]: - """Read the named control files at the process's own cgroup and at every level above it, leaf first.""" - levels = [] - directory = mount / own_path.lstrip('/') if '..' not in own_path else mount - for _ in range(_MAX_LEVELS): - files = {name: content for name in names if (content := _read_control_file(directory / name))} - levels.append({'path': str(directory), 'files': files}) - if directory in (mount, directory.parent): - break - directory = directory.parent - return levels - - -def _evidence() -> dict[str, Any]: - """Dump raw cgroup file contents along the chain, verbatim - receipts, not a verdict, never interpreted. - - Both cgroup versions are dumped the same way, from the standard mount points: the unified hierarchy sits at - /sys/fs/cgroup, while a v1 controller has a directory of its own under it. Resolving mounts properly is the - library's job, and this deliberately does not repeat it - the receipts only have to be readable next to the - values the library reported. - """ - evidence: dict[str, Any] = {'proc_cgroup': None, 'levels': []} - try: - evidence['proc_cgroup'] = Path('/proc/self/cgroup').read_text().strip() - except OSError: - return evidence - - mount = Path('/sys/fs/cgroup') - seen: set[str] = set() - - for line in evidence['proc_cgroup'].splitlines(): - parts = line.split(':', 2) - if len(parts) != 3: - continue - _hierarchy_id, controllers, own_path = parts - - if not controllers: - levels = _walk_levels(mount, own_path, _V2_EVIDENCE_FILES) - else: - # A v1 line can name several controllers sharing one mount, as `cpu,cpuacct` usually does. - names = [name for name in controllers.split(',') if name in _V1_CONTROLLERS] - levels = [ - level - for name in names - for level in _walk_levels(mount / name, own_path, _V1_EVIDENCE_FILES) - ] - - for level in levels: - if level['path'] not in seen: - seen.add(level['path']) - evidence['levels'].append(level) - - return evidence - - -def _sensor() -> dict[str, Any]: - """Collect everything `crawlee._utils.cgroup` reports for this process.""" - # Imported here rather than at module level: a rename on the branch under test must be recorded as an entry - # in `errors`, not kill the probe before it can report anything at all. - from crawlee._utils import cgroup - - readings: dict[str, Any] = { - 'is_v2': None, - 'memory_levels': None, - 'memory_limit_bytes': None, - 'working_set_bytes': None, - 'cpu_quota_cores': _guard('get_cpu_quota', cgroup.get_cpu_quota), - 'cpu_set_cores': _guard('get_cpu_set_size', cgroup.get_cpu_set_size), - 'cpu_usage_seconds': _guard('get_cpu_usage', cgroup.get_cpu_usage), - } - - memory_limit = _guard('get_memory_limit', cgroup.get_memory_limit) - if memory_limit is not None: - readings['memory_limit_bytes'] = memory_limit.limit - readings['working_set_bytes'] = memory_limit.working_set - - def discovery() -> dict[str, Any]: - memory = cgroup._get_controllers().memory - return { - 'is_v2': memory.is_v2 if memory is not None else None, - 'memory_levels': [str(directory) for directory in memory.dirs] if memory is not None else None, - } - - readings.update(_guard('_get_controllers', discovery, default={})) - return readings - - -def _derived() -> dict[str, Any]: - """Collect what `crawlee._utils.system` makes of those readings - the figures a crawler acts on. - - `get_memory_info()` substitutes the cgroup limit and its working set for the host totals whenever a limit - applies, and `get_cpu_info()` measures utilization against the cores the cgroup allows instead of the whole - machine. Where no limit is found both fall back to host-wide values, so these fields also show what a missed - limit would cost. - """ - from crawlee._utils.system import get_cpu_info, get_memory_info - - memory = _guard('get_memory_info', get_memory_info) - cpu = _guard('get_cpu_info', get_cpu_info) - - def allowed_cores() -> float | None: - from crawlee._utils.system import _get_allowed_cpu_cores - - return _get_allowed_cpu_cores() - - return { - 'total_size_bytes': memory.total_size.bytes if memory is not None else None, - 'system_wide_used_bytes': memory.system_wide_used_size.bytes if memory is not None else None, - 'current_size_bytes': memory.current_size.bytes if memory is not None else None, - 'allowed_cpu_cores': _guard('_get_allowed_cpu_cores', allowed_cores), - 'cpu_used_ratio': round(cpu.used_ratio, 3) if cpu is not None else None, - } - - -def main() -> None: - """Print one JSON object describing what this environment looks like to Crawlee.""" - source = sys.argv[sys.argv.index('--source') + 1] if '--source' in sys.argv else None - - report = { - 'schema': SCHEMA, - 'target': { - 'name': 'crawlee-python', - 'version': _guard('version', lambda: version('crawlee')), - 'source': source, - 'python': sys.version.split()[0], - }, - 'host': _host(), - 'evidence': _guard('evidence', _evidence, default={}), - 'sensor': _sensor(), - 'derived': _guard('derived', _derived, default={}), - 'errors': errors, - } - - json.dump(report, sys.stdout, indent=1) - sys.stdout.write('\n') - - -if __name__ == '__main__': - main() diff --git a/report.py b/report.py deleted file mode 100644 index d195851..0000000 --- a/report.py +++ /dev/null @@ -1,216 +0,0 @@ -from __future__ import annotations - -import json -import os -import sys -from pathlib import Path -from typing import Any - -_BYTE_UNITS = (('EB', 1024**6), ('PB', 1024**5), ('TB', 1024**4), ('GB', 1024**3), ('MB', 1024**2)) - - -def _bytes_h(value: Any) -> str: - """Format a byte count. The largest units only ever show up as the v1 sentinel for "no limit at all".""" - if not isinstance(value, (int, float)): - return '-' - for unit, size in _BYTE_UNITS: - if value >= size: - return f'{value / size:.2f} {unit}' - return f'{value / _BYTE_UNITS[-1][1]:.2f} MB' - - -def _num(value: Any) -> str: - if isinstance(value, float): - return f'{value:g}' - if isinstance(value, int): - return str(value) - return '-' - - -def _bool(value: Any) -> str: - if value is True: - return 'yes' - if value is False: - return 'no' - return '-' - - -def _seconds(value: Any) -> str: - return f'{value:.2f}s' if isinstance(value, (int, float)) else '-' - - -def _percent(value: Any) -> str: - return f'{value:.0%}' if isinstance(value, (int, float)) else '-' - - -def _in_use(sensor: dict[str, Any]) -> str: - """Memory in use, with its share of the limit it is charged against.""" - in_use = sensor.get('working_set_bytes') - limit = sensor.get('memory_limit_bytes') - if not isinstance(in_use, (int, float)): - return '-' - if isinstance(limit, (int, float)) and limit > 0: - return f'{_bytes_h(in_use)} ({in_use / limit:.0%})' - return _bytes_h(in_use) - - -def _status(result: dict[str, Any]) -> str: - status = result.get('status', '?') - if status == 'probe_failed': - return '**PROBE FAILED**' - if status == 'skipped': - return f'skipped: {result.get("skip_reason", "?")}' - return status - - -def _header(results: list[dict[str, Any]]) -> list[str]: - """Identify the run: what was measured, and on what machine.""" - target = next((r.get('target') for r in results if r.get('target')), None) or {} - host = next(((r.get('probe') or {}).get('host') for r in results if (r.get('probe') or {}).get('host')), {}) or {} - - runner = [ - f'kernel `{host.get("kernel", "?")}`', - f'{host.get("ncpu", "?")} cpus', - f'{_bytes_h(host.get("mem_total_bytes"))} RAM', - ] - # GitHub spells these two in mixed case; they name the runner image the results came from. - image_os = os.environ.get('ImageOS') - image_version = os.environ.get('ImageVersion') - if image_os or image_version: - runner.append(f'image `{image_os}/{image_version}`') - - probed = next(((r.get('probe') or {}).get('target') for r in results if (r.get('probe') or {}).get('target')), {}) - installed = '' - if probed: - installed = f' → crawlee {probed.get("version")} on python {probed.get("python")}' - - lines = [ - '## cgroups-sensor bench', - '', - f'target: `{target.get("repo", "?")}` @ `{target.get("ref", "?")}`{installed}', - 'runner: ' + ', '.join(runner), - ] - budget = next((r.get('cpu_budget') for r in results if r.get('cpu_budget')), None) - if budget: - lines.append(f'cpu budget of the limited scenarios: {budget} core(s)') - - # The engine is per scenario (the podman-* rows set their own), so it belongs in the result files rather - # than the header. The image is one setting for the whole pass. - image = next((r.get('image') for r in results if r.get('image')), None) - if image: - lines.append(f'container image: `{image}`') - - lines.append('') - return lines - - -def _sensor_table(results: list[dict[str, Any]]) -> list[str]: - """Render the sensor's readings, each next to what the scenario configured.""" - lines = [ - '### What the cgroup sensor read', - '', - ( - '| scenario | status | mem limit set | mem limit read | quota set | quota read | cpuset set ' - '| cpuset read | mem in use | levels | v2 | cpu time | errors |' - ), - '| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | ---: | --- |', - ] - for result in results: - sensor = (result.get('probe') or {}).get('sensor') or {} - configured = result.get('configured') or {} - levels = sensor.get('memory_levels') - errors = (result.get('probe') or {}).get('errors') or [] - lines.append( - f'| {result.get("scenario", "?")} | {_status(result)} ' - f'| {_bytes_h(configured.get("memory_bytes"))} | {_bytes_h(sensor.get("memory_limit_bytes"))} ' - f'| {_num(configured.get("cpu_cores"))} | {_num(sensor.get("cpu_quota_cores"))} ' - f'| {_num(configured.get("cpuset_cores"))} | {_num(sensor.get("cpu_set_cores"))} ' - f'| {_in_use(sensor)} | {len(levels) if isinstance(levels, list) else "-"} ' - f'| {_bool(sensor.get("is_v2"))} | {_seconds(sensor.get("cpu_usage_seconds"))} ' - f'| {f"{len(errors)} err" if errors else ""} |' - ) - lines.append('') - return lines - - -def _derived_table(results: list[dict[str, Any]]) -> list[str]: - """Render what the library derives from those readings - the figures a crawler acts on.""" - lines = [ - '### What the library derives from them', - '', - '| scenario | memory available | used in that scope | this process | allowed cores | cpu used |', - '| --- | ---: | ---: | ---: | ---: | ---: |', - ] - for result in results: - derived = (result.get('probe') or {}).get('derived') or {} - lines.append( - f'| {result.get("scenario", "?")} | {_bytes_h(derived.get("total_size_bytes"))} ' - f'| {_bytes_h(derived.get("system_wide_used_bytes"))} | {_bytes_h(derived.get("current_size_bytes"))} ' - f'| {_num(derived.get("allowed_cpu_cores"))} | {_percent(derived.get("cpu_used_ratio"))} |' - ) - lines.append('') - return lines - - -_TAIL_LINES = 25 - - -def _failures(results: list[dict[str, Any]]) -> list[dict[str, Any]]: - """The rows that produced no data at all. - - A skip is not one of these: it is reported in the table with its reason, and a host that cannot host a - scenario is a normal outcome, not a breakage. - """ - return [result for result in results if result.get('status') == 'probe_failed'] - - -def _failure_report(failures: list[dict[str, Any]]) -> list[str]: - """Show what a failed row printed, so the run diagnoses itself instead of sending the reader to the artifact.""" - lines = ['### Failures', ''] - for result in failures: - lines.append(f'**{result.get("scenario", "?")}** — probe failed, exit {result.get("exit")}') - if result.get('probe_parse_error'): - lines.append(f'output was not JSON: {result["probe_parse_error"]}') - tail = (result.get('stderr_tail') or '').strip().splitlines()[-_TAIL_LINES:] - if tail: - lines += ['', '```', *tail, '```'] - lines.append('') - return lines - - -def render(results: list[dict[str, Any]]) -> str: - """Render one bench run as markdown.""" - lines = _header(results) + _sensor_table(results) + _derived_table(results) - - failures = _failures(results) - if failures: - lines += _failure_report(failures) - - return '\n'.join(lines) - - -def main(argv: list[str]) -> int: - """Print the report; with `--check`, exit non-zero when a probe failed.""" - check = '--check' in argv[1:] - positional = [arg for arg in argv[1:] if arg != '--check'] - results_dir = positional[0] if positional else 'results' - - results = [] - for path in sorted(Path(results_dir).glob('*.json')): - with path.open() as file: - results.append(json.load(file)) - - if not results: - print('no results found') - return 1 if check else 0 - - sys.stdout.write(render(results)) - - if check and (failures := _failures(results)): - print(f'check: {len(failures)} probe failure(s)', file=sys.stderr) - return 1 - return 0 - - -if __name__ == '__main__': - sys.exit(main(sys.argv)) diff --git a/run.sh b/run.sh deleted file mode 100755 index d7d3a6b..0000000 --- a/run.sh +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env bash -# Bench driver. Runs scenarios sequentially and ALWAYS writes one result per scenario into the results -# directory - a crash becomes a recorded failure, never a missing row. Exit code is 0 unless the driver itself -# cannot start; judging the run is report.py's job (report.py --check). -# -# Usage: ./run.sh [all| [...]] [results_dir] -# A matches a scenario file with or without its NN- prefix (e.g. "bare" or "00-bare"). -# -# Environment: -# CRAWLEE_REPO / CRAWLEE_REF target to install (default: Mantisus/crawlee-python @ container-limits) -# BENCH_CPUS CPU budget for the limited scenarios, in cores (default 1). Scenarios larger -# than the host skip. Run the bench once per budget to cover more than one size. -# ENGINE container engine (default: docker) -# SCENARIO_TIMEOUT per-scenario timeout in seconds (default 300) - -set -u - -SENSOR_DIR=$(cd "$(dirname "$0")" && pwd) - -RESULTS="results" -SELECTORS=() -for arg in "$@"; do - if [ "$arg" = "all" ] || [ -f "$SENSOR_DIR/scenarios/$arg.sh" ]; then - SELECTORS+=("$arg") - else - RESULTS="$arg" - fi -done -[ ${#SELECTORS[@]} -gt 0 ] || SELECTORS=("all") - -ENGINE=${ENGINE:-docker} -IMG=${IMG:-ghcr.io/astral-sh/uv:python3.13-bookworm} -PY3=${PY3:-$(command -v python3)} -UV_BIN=${UV_BIN:-$(command -v uv || echo uv)} -BENCH_UV_CACHE=${BENCH_UV_CACHE:-/tmp/cgroups-bench-uv-cache} -# uv downloads interpreters here rather than into its cache; shared so only the first container of a run pays. -BENCH_UV_PYTHON=${BENCH_UV_PYTHON:-/tmp/cgroups-bench-uv-python} -CRAWLEE_REPO=${CRAWLEE_REPO:-https://github.com/Mantisus/crawlee-python} -CRAWLEE_REF=${CRAWLEE_REF:-container-limits} -BENCH_CPUS=${BENCH_CPUS:-1} - -export ENGINE IMG SENSOR_DIR BENCH_UV_CACHE BENCH_UV_PYTHON BENCH_CPUS UV_BIN -mkdir -p "$RESULTS" "$BENCH_UV_CACHE" "$BENCH_UV_PYTHON" - -# A git+ dependency makes uv run git inside the container, and most base images do not ship one. GitHub serves -# the same tree as a tarball, so an image needs nothing but network. Anything else falls back to git+. -case "$CRAWLEE_REPO" in - https://github.com/*) SOURCE="${CRAWLEE_REPO%.git}/archive/${CRAWLEE_REF}.tar.gz" ;; - *) SOURCE="git+${CRAWLEE_REPO}@${CRAWLEE_REF}" ;; -esac - -WITH_SPEC="crawlee @ $SOURCE" -# The interpreter is part of what gets measured, so it is pinned rather than left to whatever the image has. -PROBE_FLAGS="--source $SOURCE" - -# The probe command, in container paths (scenarios mount $SENSOR_DIR at /sensor) and in local paths. -# --refresh-package makes uv re-fetch the ref instead of trusting a cached resolution, so a re-run after pushing -# to the branch never silently measures the previous commit. -UV_ARGS="run --no-project --python ${BENCH_PYTHON:-3.13} --refresh-package crawlee --with \"$WITH_SPEC\"" -INNER_CONTAINER="uv $UV_ARGS python /sensor/probe.py $PROBE_FLAGS" -INNER_LOCAL="'$UV_BIN' $UV_ARGS python '$SENSOR_DIR/probe.py' $PROBE_FLAGS" - -TARGET_JSON=$(printf '{"repo": "%s", "ref": "%s"}' "$CRAWLEE_REPO" "$CRAWLEE_REF") - -echo "bench: $CRAWLEE_REPO @ $CRAWLEE_REF" -echo " image: $IMG, cpu budget: $BENCH_CPUS of $(nproc 2>/dev/null || echo '?') cores" - -selected() { - local name=$1 sel - for sel in "${SELECTORS[@]}"; do - [ "$sel" = "all" ] || [ "$sel" = "$name" ] && return 0 - done - return 1 -} - -run_one() { - local file=$1 - local name - name=$(basename "${file%.sh}") - local work - work=$(mktemp -d) - - ( - SCENARIO_NAME=$name - CONTAINER="bench-$name" - SCENARIO_DESC="" - REQUIRES="" - INNER_STYLE=local - # What the scenario configures, per axis. The report prints each next to what the sensor read, so a value - # left unset here means "this scenario deliberately restricts nothing on that axis". - SET_MEMORY_BYTES="" SET_CPU_CORES="" SET_CPUSET_CORES="" - - . "$SENSOR_DIR/scenario-helpers.sh" - . "$file" - - wrap() { - "$PY3" "$SENSOR_DIR/wrap.py" \ - --scenario "$name" --desc "$SCENARIO_DESC" \ - ${SET_MEMORY_BYTES:+--set-memory "$SET_MEMORY_BYTES"} \ - ${SET_CPU_CORES:+--set-cpu "$SET_CPU_CORES"} \ - ${SET_CPUSET_CORES:+--set-cpuset "$SET_CPUSET_CORES"} \ - --target-json "$TARGET_JSON" --out "$RESULTS/$name.json" "$@" - } - - unmet=$(unmet_requirement) - if [ -n "$unmet" ]; then - wrap --skip-reason "$unmet" - exit 0 - fi - - trap 'scenario_cleanup' EXIT - - INNER=$INNER_LOCAL - [ "$INNER_STYLE" = "container" ] && INNER=$INNER_CONTAINER - - # scenario_exec is a function, so `timeout` cannot wrap it directly; a watchdog in this shell kills a hang. - # The watchdog is detached from stdout/stderr - otherwise its orphaned sleep holds the pipe open and a - # `./run.sh | tee` style caller waits up to the full timeout for EOF after the run has finished. - scenario_exec "$INNER" >"$work/probe.json" 2>"$work/run.err" & - runpid=$! - ( sleep "${SCENARIO_TIMEOUT:-300}" && kill "$runpid" 2>/dev/null ) >/dev/null 2>&1 /dev/null - wait "$watchdog" 2>/dev/null - - wrap --probe-file "$work/probe.json" --exit-rc "$rc" --stderr-file "$work/run.err" - ) - - rm -rf "$work" - - # Baseline between scenarios: a leaked busy-loop would distort the next scenario's CPU readings. - leftovers=$(pgrep -f 'while :; do :; done' 2>/dev/null || true) - if [ -n "$leftovers" ]; then - echo "WARN: leftover busy-loop processes after $name, killing: $leftovers" >&2 - kill $leftovers 2>/dev/null || true - fi -} - -for file in "$SENSOR_DIR"/scenarios/*.sh; do - name=$(basename "${file%.sh}") - selected "$name" || continue - run_one "$file" -done diff --git a/scenario-helpers.sh b/scenario-helpers.sh deleted file mode 100644 index bc87487..0000000 --- a/scenario-helpers.sh +++ /dev/null @@ -1,179 +0,0 @@ -# shellcheck shell=bash -# Shared helpers for scenario files. Sourced by run.sh inside each scenario's subshell, before the scenario file. - -BENCH_CLUSTER=${BENCH_CLUSTER:-cgroups-bench} -# A pod cannot mount the host's uv binary, so the k8s scenarios use an image that already carries one, -# independently of the $IMG the container scenarios run on. -K8S_IMAGE=${K8S_IMAGE:-ghcr.io/astral-sh/uv:python3.13-bookworm} -K8S_POD=bench-probe -K8S_CM=bench-probe - -have() { command -v "$1" >/dev/null 2>&1; } - -# $BENCH_CPUS is the run's CPU budget: how many cores the limited scenarios are given. Scenarios derive both -# their cpuset and their quota from it, so the same files stay meaningful on a 2-core runner and on a bigger -# host - and running the bench twice with different budgets is how the CPU axes get covered at more than one -# size. A budget larger than the host makes the scenario skip (see mincpu), never silently measure nothing. -cpuset_list() { [ "$BENCH_CPUS" -le 1 ] && echo '0' || echo "0-$((BENCH_CPUS - 1))"; } - -have_engine() { "$ENGINE" info >/dev/null 2>&1; } -# True when the unified hierarchy is the one carrying the controllers. Under cgroup v1 - and under the hybrid -# layout, where a controller-less cgroup2 sits next to the v1 mounts - systemd puts resource limits only on -# system units, and properties that exist solely for the unified hierarchy are accepted and then ignored. -# Read it rather than test its size: every file in cgroupfs reports zero bytes, so -s is always false here. -have_unified() { [ -n "$(cat /sys/fs/cgroup/cgroup.controllers 2>/dev/null)" ]; } -have_sudo() { sudo -n true 2>/dev/null; } -have_systemd_user() { systemd-run --user --scope -q true 2>/dev/null; } -cgroup_driver() { "$ENGINE" info -f '{{.CgroupDriver}}' 2>/dev/null; } -ncpus() { nproc 2>/dev/null || getconf _NPROCESSORS_ONLN; } -mem_total_bytes() { awk '/^MemTotal:/ {print $2 * 1024}' /proc/meminfo; } - -# Prints the first unmet requirement from $REQUIRES, or nothing when all are met. -unmet_requirement() { - local req n - for req in $REQUIRES; do - case "$req" in - engine) have_engine || { echo "container engine '$ENGINE' is not available"; return; } ;; - sudo) have_sudo || { echo "passwordless sudo is not available"; return; } ;; - systemd-user) have_systemd_user || { echo "systemd user manager is not available"; return; } ;; - unified) have_unified || { echo "the controllers are not on the unified hierarchy, and systemd gives user scopes no cgroup of their own under cgroup v1"; return; } ;; - systemd-driver) - [ "$(cgroup_driver)" = systemd ] || { echo "$ENGINE uses the '$(cgroup_driver)' cgroup driver, this scenario needs 'systemd'"; return; } ;; - kind) - have kind && have kubectl && have_engine || { echo "kind, kubectl and a container engine are needed to bring up a cluster"; return; } ;; - min*cpu) n=${req#min}; n=${n%cpu} - [ "$(ncpus)" -ge "$n" ] || { echo "needs at least $n CPUs, host has $(ncpus)"; return; } ;; - *) have "$req" || { echo "command '$req' is not available"; return; } ;; - esac - done -} - -# Default so run.sh can call it unconditionally. Scenarios override when they have something to clean up. -scenario_cleanup() { :; } - -# Run the probe command ($1) inside a fresh container; the remaining arguments are engine run flags. The engine -# is $ENGINE, which a scenario can set for itself, so rows from different engines can share a pass. uv and its -# downloaded interpreters are mounted in rather than baked into the image, so $IMG can be any glibc distro - -# fedora, rocky, debian - and only the first container of a run pays for the download. -container_probe() { - local inner=$1 - shift - - # A cache directory per engine. A rootful container writes into it as real root, and a rootless one - whose - # own root is this user - then cannot touch those files at all, so the two must never share. - local cache="$BENCH_UV_CACHE/$ENGINE" interpreters="$BENCH_UV_PYTHON/$ENGINE" - mkdir -p "$cache" "$interpreters" - - local mounts=( - -v "$SENSOR_DIR:/sensor:ro" - -v "$cache:/root/.cache/uv" - -v "$interpreters:/root/.local/share/uv/python" - ) - # The host's uv goes to a directory appended to PATH, never over the image's own copy: an image that ships uv - # keeps using it, and only an image without one falls back to the mounted binary. - [ -x "$UV_BIN" ] && mounts+=(-v "$UV_BIN:/opt/bench/uv:ro") - - "$ENGINE" run --rm --name "$CONTAINER" "${mounts[@]}" "$@" "$IMG" \ - sh -c 'PATH="$PATH:/opt/bench"; '"$inner" -} - -# Create the bench's kind cluster if it is not up yet, and make sure the probe image is on its node. -ensure_kind_cluster() { - if ! kind get clusters 2>/dev/null | grep -qx "$BENCH_CLUSTER"; then - kind create cluster --name "$BENCH_CLUSTER" --wait 120s - fi - "$ENGINE" image inspect "$K8S_IMAGE" >/dev/null 2>&1 || "$ENGINE" pull "$K8S_IMAGE" - kind load docker-image "$K8S_IMAGE" --name "$BENCH_CLUSTER" -} - -# The script the pod runs. `kubectl logs` merges the container's stdout and stderr, so the probe's diagnostics -# are kept in a file and printed only when it fails - otherwise uv's progress output would land in the JSON. -# It always exits 0: a pod that fails takes the full wait to report it, and a missing JSON body already says so. -k8s_run_script() { - cat </tmp/probe.json 2>/tmp/probe.err -rc=\$? -if [ \$rc -eq 0 ]; then - cat /tmp/probe.json -else - echo "probe exited \$rc" - cat /tmp/probe.err -fi -exit 0 -EOF -} - -# Wait for the pod to finish, printing its phase. Polling rather than `kubectl wait --for=jsonpath` so that a -# quirk of one kubectl version cannot silently return at once and leave the logs unread. A pod the scheduler -# cannot place would otherwise sit there for the whole timeout, so that case ends the wait immediately. -k8s_wait() { - local phase reason seconds=0 - - while [ "$seconds" -lt "${K8S_WAIT_SECONDS:-300}" ]; do - phase=$(kubectl get "pod/$K8S_POD" -o jsonpath='{.status.phase}' 2>/dev/null) - case "$phase" in - Succeeded | Failed) break ;; - esac - - if [ "$seconds" -ge 5 ]; then - reason=$(kubectl get "pod/$K8S_POD" \ - -o jsonpath='{.status.conditions[?(@.type=="PodScheduled")].reason}' 2>/dev/null) - if [ "$reason" = Unschedulable ]; then - echo "pod cannot be scheduled - its requests do not fit this node" - break - fi - fi - - sleep 1 - seconds=$((seconds + 1)) - done - - echo "pod phase: ${phase:-unknown} after ${seconds}s" - [ "$phase" = Succeeded ] || kubectl describe "pod/$K8S_POD" -} - -# Run the probe command ($1) in a one-shot pod whose container resources are the JSON object in $2, and print -# the pod's stdout. probe.py and the command itself travel as a ConfigMap, so nothing has to be quoted into -# YAML. Everything except the pod's own output goes to stderr, or it would end up mixed into the probe's JSON. -k8s_probe() { - local inner=$1 resources=$2 - - { - ensure_kind_cluster - kubectl delete pod "$K8S_POD" --ignore-not-found --now - kubectl create configmap "$K8S_CM" \ - --from-file=probe.py="$SENSOR_DIR/probe.py" --from-literal=run.sh="$(k8s_run_script "$inner")" \ - --dry-run=client -o yaml | kubectl apply -f - - - kubectl apply -f - <&2 - - # A copy of the pod's output goes to stderr, so when it is not JSON the result file still shows what it was. - kubectl logs "$K8S_POD" | tee /dev/stderr -} - -k8s_cleanup() { - kubectl delete pod "$K8S_POD" --ignore-not-found --now >/dev/null 2>&1 || true - kubectl delete configmap "$K8S_CM" --ignore-not-found >/dev/null 2>&1 || true -} diff --git a/scenarios/bare.sh b/scenarios/bare.sh deleted file mode 100644 index a175792..0000000 --- a/scenarios/bare.sh +++ /dev/null @@ -1,6 +0,0 @@ -# shellcheck shell=bash -SCENARIO_DESC="no limits at all, every reading should fall back to host values" -REQUIRES="" -INNER_STYLE=local - -scenario_exec() { sh -c "$1"; } diff --git a/scenarios/docker-cgroup-parent.sh b/scenarios/docker-cgroup-parent.sh deleted file mode 100644 index 2b9ce0b..0000000 --- a/scenarios/docker-cgroup-parent.sh +++ /dev/null @@ -1,19 +0,0 @@ -# shellcheck shell=bash -SCENARIO_DESC="limit on a parent cgroup, set outside the container" -REQUIRES="engine sudo systemd-driver" -INNER_STYLE=container - -SET_MEMORY_BYTES=$((512 * 1024 * 1024)) - -SLICE=bench-parent.slice -HOLDER=bench-parent-holder - -# Under the systemd cgroup driver --cgroup-parent takes a slice, and a slice needs a live unit in it to exist. -# The limit goes on that slice; the container itself gets none, so the sensor has to walk up to find it. -scenario_exec() { - sudo systemd-run -q --unit "$HOLDER" --slice "$SLICE" sleep infinity >&2 - sudo systemctl set-property --runtime "$SLICE" "MemoryMax=$SET_MEMORY_BYTES" >&2 - container_probe "$1" --cgroupns=host --cgroup-parent="$SLICE" -} - -scenario_cleanup() { sudo systemctl stop "$HOLDER" "$SLICE" 2>/dev/null || true; } diff --git a/scenarios/docker-cpuset-only.sh b/scenarios/docker-cpuset-only.sh deleted file mode 100644 index 05c40a9..0000000 --- a/scenarios/docker-cpuset-only.sh +++ /dev/null @@ -1,8 +0,0 @@ -# shellcheck shell=bash -SCENARIO_DESC="cpuset alone, memory falls back to host" -REQUIRES="engine min${BENCH_CPUS}cpu" -INNER_STYLE=container - -SET_CPUSET_CORES=$BENCH_CPUS - -scenario_exec() { container_probe "$1" --cpuset-cpus "$(cpuset_list)"; } diff --git a/scenarios/docker-host-ns.sh b/scenarios/docker-host-ns.sh deleted file mode 100644 index 0e1f4f9..0000000 --- a/scenarios/docker-host-ns.sh +++ /dev/null @@ -1,8 +0,0 @@ -# shellcheck shell=bash -SCENARIO_DESC="cgroupns=host, where the container sees the full chain instead of its own root" -REQUIRES="engine" -INNER_STYLE=container - -SET_MEMORY_BYTES=$((512 * 1024 * 1024)) - -scenario_exec() { container_probe "$1" -m "$SET_MEMORY_BYTES" --cgroupns=host; } diff --git a/scenarios/docker-memory-only.sh b/scenarios/docker-memory-only.sh deleted file mode 100644 index 6a3d86c..0000000 --- a/scenarios/docker-memory-only.sh +++ /dev/null @@ -1,8 +0,0 @@ -# shellcheck shell=bash -SCENARIO_DESC="memory alone, CPU falls back to host" -REQUIRES="engine" -INNER_STYLE=container - -SET_MEMORY_BYTES=$((512 * 1024 * 1024)) - -scenario_exec() { container_probe "$1" -m "$SET_MEMORY_BYTES"; } diff --git a/scenarios/docker-nested-subgroup.sh b/scenarios/docker-nested-subgroup.sh deleted file mode 100644 index 81d17dd..0000000 --- a/scenarios/docker-nested-subgroup.sh +++ /dev/null @@ -1,26 +0,0 @@ -# shellcheck shell=bash -SCENARIO_DESC="a tighter cgroup created inside the container, so two levels carry different limits" -REQUIRES="engine" -INNER_STYLE=container - -OUTER_MEMORY_BYTES=$((512 * 1024 * 1024)) -SET_MEMORY_BYTES=$((256 * 1024 * 1024)) - -# The container gets one limit and the probe then puts itself under a tighter one, the shape kubelet produces -# and the only scenario here where two levels hold different limits - so the tightest one has to win. -# -# The dance is the kernel's "no internal processes" rule: a cgroup cannot both hold processes and delegate -# controllers to its children, so the shell first vacates the namespace root, then enables the memory -# controller for children, and only then moves into the tighter cgroup it created. -NESTED_SETUP=' -mount -o remount,rw /sys/fs/cgroup 2>/dev/null || true -mkdir -p /sys/fs/cgroup/init /sys/fs/cgroup/inner -echo $$ > /sys/fs/cgroup/init/cgroup.procs -echo +memory > /sys/fs/cgroup/cgroup.subtree_control -echo '"$SET_MEMORY_BYTES"' > /sys/fs/cgroup/inner/memory.max -echo $$ > /sys/fs/cgroup/inner/cgroup.procs -' - -scenario_exec() { - container_probe "$NESTED_SETUP $1" --privileged -m "$OUTER_MEMORY_BYTES" -} diff --git a/scenarios/docker-private.sh b/scenarios/docker-private.sh deleted file mode 100644 index c4495ac..0000000 --- a/scenarios/docker-private.sh +++ /dev/null @@ -1,13 +0,0 @@ -# shellcheck shell=bash -SCENARIO_DESC="private cgroupns (docker's default), all three axes at once" -REQUIRES="engine min${BENCH_CPUS}cpu" -INNER_STYLE=container - -# Half a core tighter than the cpuset, so the row also shows which of the two CPU axes wins. -QUOTA=$(awk "BEGIN{print $BENCH_CPUS - 0.5}") - -SET_MEMORY_BYTES=$((512 * 1024 * 1024)) -SET_CPU_CORES=$QUOTA -SET_CPUSET_CORES=$BENCH_CPUS - -scenario_exec() { container_probe "$1" -m "$SET_MEMORY_BYTES" --cpus "$QUOTA" --cpuset-cpus "$(cpuset_list)"; } diff --git a/scenarios/k8s-limits.sh b/scenarios/k8s-limits.sh deleted file mode 100644 index 2bbd930..0000000 --- a/scenarios/k8s-limits.sh +++ /dev/null @@ -1,23 +0,0 @@ -# shellcheck shell=bash -SCENARIO_DESC="pod with container limits, read from inside its own cgroup namespace" -REQUIRES="kind" -INNER_STYLE=container -SCENARIO_TIMEOUT=900 # the first k8s scenario of a run brings the cluster up - -# Half a core tighter than the budget, as in docker-private. Requests are kept small on purpose: kubernetes -# copies the limits into the requests when none are given, and a pod requesting every core of the node is one -# the scheduler can never place. -MILLICORES=$((BENCH_CPUS * 1000 - 500)) - -SET_MEMORY_BYTES=$((512 * 1024 * 1024)) -SET_CPU_CORES=$(awk "BEGIN{print $BENCH_CPUS - 0.5}") - -# kubelet nests a pod several levels under kubepods, but the container gets a cgroup namespace of its own, so -# from inside the chain is one level deep and the limit sits on it. The walk up to an ancestor is what -# docker-cgroup-parent covers; here what matters is that kubelet's shape is read correctly at all. -scenario_exec() { - k8s_probe "$1" "$(printf '{"requests": {"cpu": "50m", "memory": "64Mi"}, "limits": {"memory": "%s", "cpu": "%sm"}}' \ - "$SET_MEMORY_BYTES" "$MILLICORES")" -} - -scenario_cleanup() { k8s_cleanup; } diff --git a/scenarios/k8s-no-limits.sh b/scenarios/k8s-no-limits.sh deleted file mode 100644 index 4abbfeb..0000000 --- a/scenarios/k8s-no-limits.sh +++ /dev/null @@ -1,10 +0,0 @@ -# shellcheck shell=bash -SCENARIO_DESC="pod without limits, inside the same kubepods hierarchy" -REQUIRES="kind" -INNER_STYLE=container -SCENARIO_TIMEOUT=900 - -# Same nesting as k8s-limits but nothing set, so every reading should fall back to the node's values. -scenario_exec() { k8s_probe "$1" '{}'; } - -scenario_cleanup() { k8s_cleanup; } diff --git a/scenarios/systemd-ancestor.sh b/scenarios/systemd-ancestor.sh deleted file mode 100644 index 0f84827..0000000 --- a/scenarios/systemd-ancestor.sh +++ /dev/null @@ -1,18 +0,0 @@ -# shellcheck shell=bash -SCENARIO_DESC="limit on an ancestor, leaf without memory controller files" -REQUIRES="systemd-user unified" -INNER_STYLE=local - -SET_MEMORY_BYTES=$((512 * 1024 * 1024)) - -# The scope carries the limit; Delegate=yes lets the probe move into a child cgroup that inherits no memory -# controller files at all, so the sensor has to walk up past a level that has nothing to read. -scenario_exec() { - INNER="$1" systemd-run --user --scope -q --unit "bench-ancestor-$$" \ - -p "MemoryMax=$SET_MEMORY_BYTES" -p Delegate=yes bash -c ' - own=$(grep "^0::" /proc/self/cgroup | cut -d: -f3) - mkdir -p "/sys/fs/cgroup$own/leaf" - echo $$ > "/sys/fs/cgroup$own/leaf/cgroup.procs" - exec sh -c "$INNER" - ' -} diff --git a/scenarios/systemd-memory-above-host.sh b/scenarios/systemd-memory-above-host.sh deleted file mode 100644 index 2f3a095..0000000 --- a/scenarios/systemd-memory-above-host.sh +++ /dev/null @@ -1,16 +0,0 @@ -# shellcheck shell=bash -SCENARIO_DESC="a memory limit larger than the machine has RAM" -REQUIRES="systemd-user unified" -INNER_STYLE=local - -# The memory twin of systemd-quota-above-host, and the axis where the library already guards itself: a limit at -# or above the host's total is treated as no limit, so the derived figures fall back to host values while the -# sensor still reports the number the file holds. Realistic wherever a container or pod is given a limit larger -# than the machine it landed on. -SET_MEMORY_BYTES=$(($(mem_total_bytes) + 1024 * 1024 * 1024)) - -UNIT="bench-memory-above-host-$$" - -scenario_exec() { - systemd-run --user --scope -q --unit "$UNIT" -p "MemoryMax=$SET_MEMORY_BYTES" sh -c "$1" -} diff --git a/scenarios/systemd-own.sh b/scenarios/systemd-own.sh deleted file mode 100644 index 86e8951..0000000 --- a/scenarios/systemd-own.sh +++ /dev/null @@ -1,33 +0,0 @@ -# shellcheck shell=bash -SCENARIO_DESC="deep chain, limits on the process's own cgroup, cpuset delegated" -REQUIRES="sudo systemd-run min${BENCH_CPUS}cpu" -INNER_STYLE=local - -# A system scope, not a user one: user slices do not delegate cpuset, so AllowedCPUs would be silently ignored. -# The quota is a core looser than the cpuset here, the mirror of docker-private, where it is tighter. It stops -# at the machine's core count: a quota above that is a different question, asked on purpose by -# systemd-quota-above-host, and letting this row drift into it would just duplicate that one. -QUOTA_CORES=$((BENCH_CPUS + 1)) -[ "$QUOTA_CORES" -gt "$(ncpus)" ] && QUOTA_CORES=$(ncpus) - -SET_MEMORY_BYTES=$((512 * 1024 * 1024)) -SET_CPU_CORES=$QUOTA_CORES - -# AllowedCPUs= exists only for the unified hierarchy: under cgroup v1 systemd accepts it and silently applies -# nothing, so the ask is left out there rather than recorded as a limit that was never set. -CPUSET_PROPS=() -if have_unified; then - SET_CPUSET_CORES=$BENCH_CPUS - CPUSET_PROPS=(-p "AllowedCPUs=$(cpuset_list)") -fi - -UNIT="bench-systemd-own-$$" - -# sudo strips the environment, so the probe needs PATH and a cache root can write to. -scenario_exec() { - sudo systemd-run --scope -q --unit "$UNIT" \ - -p "MemoryMax=$SET_MEMORY_BYTES" -p "CPUQuota=$((QUOTA_CORES * 100))%" "${CPUSET_PROPS[@]}" \ - env "PATH=$PATH" "HOME=/root" "UV_CACHE_DIR=$BENCH_UV_CACHE/root" sh -c "$1" -} - -scenario_cleanup() { sudo systemctl stop "$UNIT.scope" 2>/dev/null || true; } diff --git a/scenarios/systemd-quota-above-host.sh b/scenarios/systemd-quota-above-host.sh deleted file mode 100644 index e28f8bb..0000000 --- a/scenarios/systemd-quota-above-host.sh +++ /dev/null @@ -1,18 +0,0 @@ -# shellcheck shell=bash -SCENARIO_DESC="a cpu quota larger than the machine has cores" -REQUIRES="systemd-user unified" -INNER_STYLE=local - -# Nothing rejects a quota above the machine's core count: systemd writes it as asked, and a kubernetes limit -# above node capacity does the same whenever the requests are small enough to schedule. Docker is the exception, -# it validates --cpus against nproc. The cores that exist are the real ceiling, so a derived "allowed cores" -# above them means the utilization ratio is divided by more than can ever be consumed. -QUOTA_CORES=$(($(ncpus) + 1)) - -SET_CPU_CORES=$QUOTA_CORES - -UNIT="bench-quota-above-host-$$" - -scenario_exec() { - systemd-run --user --scope -q --unit "$UNIT" -p "CPUQuota=$((QUOTA_CORES * 100))%" sh -c "$1" -} diff --git a/v1-guest.sh b/v1-guest.sh deleted file mode 100755 index f5a0923..0000000 --- a/v1-guest.sh +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/env bash -# Run the whole bench inside a guest booted on cgroup v1, and bring its results back. -# -# cgroup v1 cannot be produced on a modern host: a controller belongs to exactly one hierarchy, and on a -# unified system they all belong to v2 - mounting them as v1 fails with EBUSY even as root, and inside a user -# namespace v1 cannot be mounted at all. A guest kernel booted in legacy mode is the only way to see those code -# paths, so this is a lane rather than a scenario: the same run.sh runs inside, and every scenario that can run -# there exercises the v1 side of the library. -# -# Usage: ./v1-guest.sh [results_dir] -# -# Environment: -# V1_IMAGE_URL / V1_KERNEL_URL / V1_INITRD_URL guest to boot (default: Ubuntu 22.04 cloud image) -# V1_CMDLINE kernel command line (default: legacy cgroup mode) -# V1_SCENARIOS what to run inside (default: everything the guest can do) -# V1_MEMORY / V1_CPUS guest size (default 4096 MB, 2 cores) -# plus CRAWLEE_REPO, CRAWLEE_REF, BENCH_CPUS, which are passed through to the bench - -set -eu - -SENSOR_DIR=$(cd "$(dirname "$0")" && pwd) -RESULTS=${1:-results} - -WORK=${V1_WORK:-/tmp/cgroups-bench-v1} -SSH_PORT=${V1_SSH_PORT:-2222} -GUEST_USER=bench - -BASE=https://cloud-images.ubuntu.com/releases/22.04/release -V1_IMAGE_URL=${V1_IMAGE_URL:-$BASE/ubuntu-22.04-server-cloudimg-amd64.img} -V1_KERNEL_URL=${V1_KERNEL_URL:-$BASE/unpacked/ubuntu-22.04-server-cloudimg-amd64-vmlinuz-generic} -V1_INITRD_URL=${V1_INITRD_URL:-$BASE/unpacked/ubuntu-22.04-server-cloudimg-amd64-initrd-generic} - -# systemd honours these up to v255; they are ignored from v256 and cgroup v1 is gone in v258, which is why the -# guest is Ubuntu 22.04 (systemd 249) rather than something newer. Booting the kernel directly, rather than -# through the image's own bootloader, is what lets the line be set from here without editing the image first. -# -# hybrid (the default) mounts the v1 controllers and a controller-less cgroup2 next to them, so the library has -# to notice that the unified hierarchy carries nothing and fall back per controller. legacy is plain v1. -case ${V1_MODE:-hybrid} in - hybrid) CGROUP_ARGS="systemd.unified_cgroup_hierarchy=0" ;; - legacy) CGROUP_ARGS="systemd.unified_cgroup_hierarchy=0 systemd.legacy_systemd_cgroup_controller=1" ;; - *) echo "v1-guest: V1_MODE must be hybrid or legacy" >&2; exit 1 ;; -esac -V1_CMDLINE=${V1_CMDLINE:-"root=/dev/vda1 console=ttyS0 $CGROUP_ARGS"} - -# kind needs a container runtime and a lot of memory; the k8s shapes are already covered on the v2 host. -V1_SCENARIOS=${V1_SCENARIOS:-"bare systemd-own systemd-ancestor systemd-quota-above-host systemd-memory-above-host docker-private docker-memory-only docker-cpuset-only docker-host-ns docker-nested-subgroup"} - -CRAWLEE_REPO=${CRAWLEE_REPO:-https://github.com/Mantisus/crawlee-python} -CRAWLEE_REF=${CRAWLEE_REF:-container-limits} -BENCH_CPUS=${BENCH_CPUS:-1} - -need() { command -v "$1" >/dev/null 2>&1 || { echo "v1-guest: $1 is required (apt install $2)" >&2; exit 1; }; } -need qemu-system-x86_64 qemu-system-x86 -need cloud-localds cloud-image-utils -need ssh openssh-client -# Emulation works without any privileges but is slow enough to be useful only for checking the plumbing, so it -# has to be asked for by name rather than silently turning a five minute lane into a thirty minute one. -V1_ACCEL=${V1_ACCEL:-kvm} -if [ "$V1_ACCEL" = kvm ] && ! { [ -r /dev/kvm ] && [ -w /dev/kvm ]; }; then - echo "v1-guest: /dev/kvm is not usable. On a GitHub runner:" >&2 - echo " echo 'KERNEL==\"kvm\", GROUP=\"kvm\", MODE=\"0666\", OPTIONS+=\"static_node=kvm\"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules" >&2 - echo " sudo udevadm control --reload-rules && sudo udevadm trigger --name-match=kvm" >&2 - echo " locally: sudo usermod -aG kvm \$USER (then log in again), or run with V1_ACCEL=tcg to emulate" >&2 - exit 1 -fi -[ "$V1_ACCEL" = kvm ] || echo "v1-guest: no KVM, emulating - expect this to be several times slower" >&2 - -mkdir -p "$WORK" "$RESULTS" - -fetch() { # url -> cached file, downloaded once per machine - local url=$1 dest="$WORK/$(basename "$1")" - [ -s "$dest" ] || { echo "v1-guest: downloading $(basename "$url")" >&2; curl -fsSL -o "$dest" "$url"; } - echo "$dest" -} - -IMAGE=$(fetch "$V1_IMAGE_URL") -KERNEL=$(fetch "$V1_KERNEL_URL") -INITRD=$(fetch "$V1_INITRD_URL") - -# A throwaway overlay, so a run never dirties the downloaded image and every run starts clean. -DISK=$WORK/run.qcow2 -rm -f "$DISK" -qemu-img create -q -f qcow2 -F qcow2 -b "$IMAGE" "$DISK" 8G - -KEY=$WORK/id_ed25519 -[ -s "$KEY" ] || ssh-keygen -q -t ed25519 -N '' -f "$KEY" - -cat > "$WORK/user-data" </dev/null - rm -f "$WORK/qemu.pid" -} -trap cleanup EXIT - -# scp spells the port -P and reads -p as "preserve timestamps", so the two need separate option strings. -SSH_OPTS="-i $KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR" -# shellcheck disable=SC2086 -in_guest() { ssh $SSH_OPTS -p "$SSH_PORT" "$GUEST_USER@127.0.0.1" "$@"; } -# shellcheck disable=SC2086 -copy_in() { scp -q $SSH_OPTS -P "$SSH_PORT" -r "$@" "$GUEST_USER@127.0.0.1:bench/"; } -# shellcheck disable=SC2086 -copy_out() { scp -q $SSH_OPTS -P "$SSH_PORT" "$GUEST_USER@127.0.0.1:$1" "$2"; } - -echo -n "v1-guest: waiting for ssh" -for _ in $(seq 1 120); do - in_guest true 2>/dev/null && break - echo -n . - sleep 2 -done -echo -in_guest true 2>/dev/null || { - echo "v1-guest: the guest never came up; the last of its console output:" >&2 - tail -30 "$WORK/console.log" >&2 - exit 1 -} - -# v1 controllers each get a mount of their own, so counting both kinds says which mode the guest came up in: -# only cgroup mounts is legacy, both kinds is hybrid, only cgroup2 means the kernel line did not take. -echo -n 'v1-guest: guest is up, ' -in_guest 'printf "cgroup mounts: %s v1, %s v2\n" \ - "$(grep -c " - cgroup " /proc/self/mountinfo || true)" "$(grep -c " - cgroup2 " /proc/self/mountinfo || true)"' - -# ssh answers as soon as sshd is up, while cloud-init is still installing packages behind it. -echo "v1-guest: waiting for cloud-init to finish" -in_guest 'cloud-init status --wait >/dev/null 2>&1 || true' - -in_guest 'mkdir -p bench' -copy_in "$SENSOR_DIR/run.sh" "$SENSOR_DIR/probe.py" "$SENSOR_DIR/wrap.py" "$SENSOR_DIR/report.py" \ - "$SENSOR_DIR/scenario-helpers.sh" "$SENSOR_DIR/scenarios" "$(command -v uv)" -in_guest 'sudo install -m 0755 bench/uv /usr/local/bin/uv' -in_guest 'getent group docker >/dev/null && sudo usermod -aG docker '"$GUEST_USER"' || true' - -echo "v1-guest: running the bench inside" -BENCH_CMD="cd bench && CRAWLEE_REPO='$CRAWLEE_REPO' CRAWLEE_REF='$CRAWLEE_REF' BENCH_CPUS='$BENCH_CPUS' \ - ./run.sh $V1_SCENARIOS results/" -# The freshly granted docker group only applies to new logins, so borrow it for this command when it exists. -if in_guest 'getent group docker >/dev/null'; then - in_guest "sg docker -c \"$BENCH_CMD\"" -else - in_guest "$BENCH_CMD" -fi - -# shellcheck disable=SC2086 -copy_out 'bench/results/*.json' "$RESULTS/" -in_guest 'sudo poweroff' 2>/dev/null || true - -echo "v1-guest: results in $RESULTS" diff --git a/wrap.py b/wrap.py deleted file mode 100644 index 15660f0..0000000 --- a/wrap.py +++ /dev/null @@ -1,105 +0,0 @@ -from __future__ import annotations - -import json -import os -import sys -import time -from pathlib import Path -from typing import Any - -SCHEMA = 1 - -_STDERR_TAIL_CHARS = 8000 -_CONSOLE_TAIL_LINES = 15 - - -def _read_tail(path: str | None, limit: int = _STDERR_TAIL_CHARS) -> str: - if not path: - return '' - try: - return Path(path).read_text(errors='replace')[-limit:] - except OSError: - return '' - - -def _parse_args(argv: list[str]) -> dict[str, str]: - """Parse `--key value` and `--flag` pairs; run.sh is the only caller.""" - args: dict[str, str] = {} - index = 0 - while index < len(argv): - arg = argv[index] - if not arg.startswith('--'): - raise SystemExit(f'wrap.py: unexpected argument {arg!r}') - key = arg.removeprefix('--') - if index + 1 < len(argv) and not argv[index + 1].startswith('--'): - args[key] = argv[index + 1] - index += 2 - else: - args[key] = '1' - index += 1 - return args - - -def main(argv: list[str]) -> int: - """Write one scenario's result JSON and print its status line.""" - args = _parse_args(argv[1:]) - - probe: Any = None - parse_error = None - if args.get('probe-file'): - try: - probe = json.loads(Path(args['probe-file']).read_text()) - except (OSError, json.JSONDecodeError) as exc: - parse_error = str(exc) - - result: dict[str, Any] = { - 'schema': SCHEMA, - 'scenario': args['scenario'], - 'desc': args.get('desc', ''), - 'cpu_budget': os.environ.get('BENCH_CPUS'), - 'engine': os.environ.get('ENGINE'), - 'image': os.environ.get('IMG'), - # What the scenario configured. A None means it restricts nothing on that axis, so the sensor reporting - # nothing there is the right answer rather than a miss. - 'configured': { - 'memory_bytes': int(args['set-memory']) if args.get('set-memory') else None, - 'cpu_cores': float(args['set-cpu']) if args.get('set-cpu') else None, - 'cpuset_cores': int(args['set-cpuset']) if args.get('set-cpuset') else None, - }, - 'target': json.loads(args.get('target-json', 'null')), - 'probe': probe, - 'exit': int(args['exit-rc']) if args.get('exit-rc') else None, - 'stderr_tail': _read_tail(args.get('stderr-file')), - 'timestamp': time.strftime('%Y-%m-%dT%H:%M:%S%z'), - } - - if args.get('skip-reason'): - result['status'] = 'skipped' - result['skip_reason'] = args['skip-reason'] - elif result['exit'] == 0 and probe is not None: - result['status'] = 'ok' - else: - result['status'] = 'probe_failed' - if parse_error: - result['probe_parse_error'] = parse_error - - out = Path(args['out']) - out.parent.mkdir(parents=True, exist_ok=True) - with out.open('w') as file: - json.dump(result, file, indent=1) - file.write('\n') - - detail = result.get('skip_reason', '') - if result['status'] == 'probe_failed': - detail = f'exit {result["exit"]}' + (f', {parse_error}' if parse_error else '') - print(f'{args["scenario"]}: {result["status"]}' + (f' ({detail})' if detail else '')) - - # A failure is worth reading where it happens, not only in the artifact. - if result['status'] == 'probe_failed': - for line in result['stderr_tail'].strip().splitlines()[-_CONSOLE_TAIL_LINES:]: - print(f' | {line}') - return 0 - - -if __name__ == '__main__': - sys.exit(main(sys.argv)) From 076040e1550dc41478c5f1b1fc4c60237561b247 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Thu, 20 Aug 2026 21:11:07 +0000 Subject: [PATCH 2/3] convert to package --- .github/workflows/ci.yaml | 39 + .github/workflows/e2e.yaml | 136 +++ CHANGELOG.md | 5 + LICENSE | 2 +- README.md | 204 ++-- pyproject.toml | 141 +++ src/cgroups_sensor/__init__.py | 61 ++ src/cgroups_sensor/_cgroup.py | 976 +++++++++++++++++++ src/cgroups_sensor/_cpu_list.py | 30 + src/cgroups_sensor/_sensor.py | 842 +++++++++++++++++ src/cgroups_sensor/py.typed | 0 tests/__init__.py | 0 tests/e2e/__init__.py | 0 tests/e2e/conftest.py | 201 ++++ tests/e2e/harness.py | 399 ++++++++ tests/e2e/probe.py | 82 ++ tests/e2e/scripts/guest.sh | 253 +++++ tests/e2e/test_docker.py | 185 ++++ tests/e2e/test_kubernetes.py | 209 +++++ tests/e2e/test_machine.py | 186 ++++ tests/unit/__init__.py | 0 tests/unit/conftest.py | 94 ++ tests/unit/test_cgroup.py | 1543 +++++++++++++++++++++++++++++++ tests/unit/test_cpu_list.py | 26 + tests/unit/test_sensor.py | 1217 ++++++++++++++++++++++++ uv.lock | 301 ++++++ 26 files changed, 7036 insertions(+), 96 deletions(-) create mode 100644 .github/workflows/ci.yaml create mode 100644 .github/workflows/e2e.yaml create mode 100644 CHANGELOG.md create mode 100644 pyproject.toml create mode 100644 src/cgroups_sensor/__init__.py create mode 100644 src/cgroups_sensor/_cgroup.py create mode 100644 src/cgroups_sensor/_cpu_list.py create mode 100644 src/cgroups_sensor/_sensor.py create mode 100644 src/cgroups_sensor/py.typed create mode 100644 tests/__init__.py create mode 100644 tests/e2e/__init__.py create mode 100644 tests/e2e/conftest.py create mode 100644 tests/e2e/harness.py create mode 100644 tests/e2e/probe.py create mode 100755 tests/e2e/scripts/guest.sh create mode 100644 tests/e2e/test_docker.py create mode 100644 tests/e2e/test_kubernetes.py create mode 100644 tests/e2e/test_machine.py create mode 100644 tests/unit/__init__.py create mode 100644 tests/unit/conftest.py create mode 100644 tests/unit/test_cgroup.py create mode 100644 tests/unit/test_cpu_list.py create mode 100644 tests/unit/test_sensor.py create mode 100644 uv.lock diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..d7948cd --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,39 @@ +name: CI + +on: + push: + branches: + - main + pull_request: + +permissions: + contents: read + +# One run per branch. A push that supersedes another cancels it, except on main, where every commit is checked. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + # One interpreter for both: ruff takes the target version from its config and ty takes `python-version` from + # `[tool.ty.environment]`, so what the runner runs on decides nothing. The unit tests below are where the + # interpreter is a real axis. + lint_check: + name: Lint check + uses: apify/workflows/.github/workflows/python_lint_check.yaml@main + with: + python_versions: '["3.13"]' + + type_check: + name: Type check + uses: apify/workflows/.github/workflows/python_type_check.yaml@main + with: + python_versions: '["3.13"]' + + unit_tests: + name: Unit tests + uses: apify/workflows/.github/workflows/python_unit_tests.yaml@main + with: + python_versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' + operating_systems: '["ubuntu-latest"]' + run_tests_command: uv run poe unit-tests diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml new file mode 100644 index 0000000..1b6e04f --- /dev/null +++ b/.github/workflows/e2e.yaml @@ -0,0 +1,136 @@ +name: E2E + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +permissions: + contents: read + +# One run per branch. A superseded push is cancelled, except on main, where every commit is checked. These +# lanes boot virtual machines, so a queue of stale runs is expensive. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + # The interpreter is a real axis only for what the probe runs on: the container tests below bring their own + # interpreters with uv, and running them once per version would repeat identical work five times. + machine: + name: machine (python ${{ matrix.python-version }}) + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + python-version: ['3.10', '3.11', '3.12', '3.13', '3.14'] + env: + # The suite itself, and with it the probe on this machine, runs on the version of the matrix. + UV_PYTHON: ${{ matrix.python-version }} + E2E_REQUIRE: sudo,systemd + E2E_INTERFACE: cgroup-v2 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Set up uv package manager + uses: astral-sh/setup-uv@v10.0.1 + + - name: Install dependencies + run: uv run poe install-dev + + # pytest directly, not `poe e2e-tests`: that task names the whole suite, and a path passed to it would + # be collected on top of the directory rather than instead of it. + - name: Run the machine tests + run: uv run pytest tests/e2e/test_machine.py + + containers: + name: containers + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + # What this runner is here to prove. Without it a runner that lost docker would report a green job + # while every container test quietly skipped. + E2E_REQUIRE: docker,sudo,systemd + E2E_INTERFACE: cgroup-v2 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Set up uv package manager + uses: astral-sh/setup-uv@v10.0.1 + + - name: Install dependencies + run: uv run poe install-dev + + - name: Run the container tests + # These parametrize the interpreter themselves, inside the containers, so they run once. + run: uv run pytest tests/e2e/test_docker.py + + kubernetes: + name: kubernetes + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Set up uv package manager + uses: astral-sh/setup-uv@v10.0.1 + + - name: Install kind + uses: helm/kind-action@v1.14.0 + with: + cluster_name: cgroups-sensor-e2e + + - name: Install dependencies + run: uv run poe install-dev + + - name: Run the kubernetes tests + env: + E2E_REQUIRE: kubernetes + run: uv run pytest tests/e2e/test_kubernetes.py + + guest: + name: guest (${{ matrix.profile }}) + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + # The Ubuntu profiles are the only way to reach cgroup v1. The Fedora one runs the same suite on a + # second distribution, with a much newer systemd. + profile: [ubuntu-v1-hybrid, ubuntu-v1-legacy, fedora-v2] + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Set up uv package manager + uses: astral-sh/setup-uv@v10.0.1 + + - name: Install QEMU + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends qemu-system-x86 cloud-image-utils + + - name: Let this user reach /dev/kvm + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ + | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + + - name: Cache the guest image + uses: actions/cache@v4 + with: + # The images only, not the keys and overlays a run creates next to them. + path: ~/.cache/cgroups-sensor-guest + key: guest-images-${{ hashFiles('tests/e2e/scripts/guest.sh') }} + + - name: Run the e2e suite inside the guest + env: + GUEST_PROFILE: ${{ matrix.profile }} + run: ./tests/e2e/scripts/guest.sh diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..8190135 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +# Changelog + +## [Unreleased] + +Nothing is released yet. diff --git a/LICENSE b/LICENSE index 261eeb9..7fe035a 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright 2026 Apify Technologies s.r.o. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index 2000942..5304967 100644 --- a/README.md +++ b/README.md @@ -1,114 +1,128 @@ # cgroups-sensor -Utility functions to measure resource limits from cgroups in scenarios where psutils is not sufficient. +Reports the CPU and memory limits that actually apply to the running process, read from cgroups. -Today the repository is a bench for that measurement: it runs a library's cgroup detection inside real -environment shapes — systemd scopes, containers, kubernetes pods — and prints what the library saw there. It -runs when a human triggers it and reports to a human; nothing is scheduled and nothing asserts on values. +## Why -## Running it +Inside a container the usual answers describe the machine, not the process. `psutil.virtual_memory().total` reports the memory of the node, `os.cpu_count()` reports every core of it. A process that sizes a budget from those numbers keeps growing until the kernel kills it. -On GitHub: **Actions → bench → Run workflow**. Locally, on any cgroup-v2 Linux box with `uv`: +Reading the cgroup files directly is not enough either. An unrestricted cgroup does not leave its limit empty: cgroup v1 spells it as a sentinel near 2\*\*63, a CPU quota can exceed the cores of the machine, and a CPU set can cover every core. A consumer that takes those at face value believes it has 8 EB of memory. -```bash -./run.sh # every scenario, results into results/ -./run.sh bare docker-private # only these two -BENCH_CPUS=2 IMG=fedora:41 ./run.sh # a different budget, in another distro's userspace -python3 report.py results/ # the tables -python3 report.py results/ --check # same, exit 1 if a probe failed +This package reads the files, walks the levels, and reports only what restricts the process. `None` means nothing restricts it - the machine is then the honest answer, and `get_machine_cpu_count()` and `get_machine_memory_bytes()` below give it. + +Linux only, Python 3.10 or newer, and no dependencies. Off Linux every limit reads as `None`. Some examples below pair it with `psutil`, which this package does not require - it is there to answer what the machine is using, which is not a question about limits. + +## Use + +Size a memory budget from the limit, and from the machine only when nothing limits you: + +```python +import psutil + +import cgroups_sensor + + +def memory_budget() -> tuple[int, int]: + """The total and the used memory a budget should be derived from, in bytes.""" + budget = cgroups_sensor.get_memory_budget() + if budget is not None: + return budget.limit, budget.working_set + + memory = psutil.virtual_memory() + return memory.total, memory.total - memory.available ``` -| knob | what it sets | -| --- | --- | -| `CRAWLEE_REPO`, `CRAWLEE_REF` | what to measure; installed as a source tarball, so the image needs no `git` | -| `BENCH_CPUS` | cores the CPU-limited scenarios ask for, default 1. They derive cpuset and quota from it, so run it twice: once below the host's core count and once equal to it, where a cpuset stops restricting anything | -| `IMG` | base image for the container scenarios, default the uv one. Any glibc distro works — uv and the interpreter are mounted in | -| `BENCH_PYTHON` | interpreter to measure on, default 3.13, so the image cannot change it | - -Scenarios whose prerequisites are missing (docker, passwordless sudo, a systemd user manager, kind, enough -cores) are recorded as `skipped` with the reason; only a crashed probe reddens a run. The `k8s-*` scenarios -bring up a kind cluster named `cgroups-bench` and leave it running — `kind delete cluster --name cgroups-bench`. - -**cgroup v1** cannot be produced on a modern host: a controller belongs to one hierarchy at a time, and on a -unified system it cannot be mounted as v1 even by root — inside a user namespace, not at all. So the v1 half of -the library is reachable only from a guest kernel, which is what `./v1-guest.sh` does: it boots Ubuntu 22.04 -(systemd 249, the last releases that still honour the flags), runs the same scenarios inside and copies the -results back into the same report. `V1_MODE` picks what the guest boots into — `hybrid` puts the v1 controllers -next to a controller-less cgroup2, so the library has to notice the unified hierarchy is empty and fall back -per controller, while `legacy` is plain v1. In the workflow both are the `cgroup_v1` input. - -It needs `qemu-system-x86 cloud-image-utils` and a usable `/dev/kvm` (`sudo usermod -aG kvm $USER` locally, a -udev rule on a runner); `V1_ACCEL=tcg` emulates instead, which is slow but needs no privileges. The report's -`v2` column says what the guest actually came up as, and a v1 host with no limit reports the sentinel that -shows up in the table as `8.00 EB`. - -## Files - -| file | what it does | -| --- | --- | -| [run.sh](run.sh) | Runs each scenario and always writes a result, so a crash is a recorded row rather than a missing one. | -| [scenario-helpers.sh](scenario-helpers.sh) | The vocabulary scenarios are written in: prerequisite probes, cpuset derivation, the container and pod launchers. | -| [scenarios/](scenarios/) | One file per environment shape; every file here is a scenario. | -| [probe.py](probe.py) | Runs inside the prepared environment and prints one JSON object with what Crawlee sees there. | -| [wrap.py](wrap.py) | Folds the probe's output and the scenario's configuration into one result file. | -| [report.py](report.py) | Turns a results directory into the tables. `--check` is the only gate. | -| [v1-guest.sh](v1-guest.sh) | Boots a guest on cgroup v1 and runs the bench inside it, bringing the results back. | -| [.github/workflows/bench.yaml](.github/workflows/bench.yaml) | Manual trigger only, one job per CPU budget. | +Size a worker pool from the cores you may use, and from the machine when nothing limits you: + +```python +import cgroups_sensor + +cores = cgroups_sensor.get_cpu_limit() or cgroups_sensor.get_machine_cpu_count() or 1 +workers = max(1, round(cores)) +``` + +`get_machine_cpu_count()` rather than `os.cpu_count()`: the latter honours the `PYTHON_CPU_COUNT` override, and under musl it reports the affinity of the process instead of the machine. `psutil.cpu_count()` has the same two problems. This one asks the kernel. + +Measure the CPU load against what you may use, rather than against the machine. A cgroup reports consumed CPU time as a counter, so a rate needs two readings. `CpuLoad` keeps the previous one, which makes the window as long as the interval between calls and costs no waiting: + +```python +import time -## Scenarios +import psutil -A scenario declares what it needs, what it configures, and how to run the probe in the environment it prepares. -The commands use the declared values rather than repeating them, so the report's `set` column is literally what -was applied. +import cgroups_sensor -```bash -SCENARIO_DESC="private cgroupns (docker's default), all three axes at once" -REQUIRES="engine min${BENCH_CPUS}cpu" # unmet -> skipped, with this reason in the table +load = cgroups_sensor.CpuLoad() -QUOTA=$(awk "BEGIN{print $BENCH_CPUS - 0.5}") +while True: + used_ratio = load.sample() + if used_ratio is None: + used_ratio = psutil.cpu_percent() / 100 + ... + time.sleep(5) +``` + +The loop has to pace itself, as the `sleep` above does: every call here returns at once where there is nothing to measure, and `sample()` never waits at all. For a single measurement there is `get_cpu_used_ratio(interval)`, which waits out the window itself, and `get_cpu_used_ratio_async(interval)` for asyncio. Keep the interval generous - the kernel updates the counter in coarse steps, so a tenth of a second can report a busy process as idle, and anything below 0.01 seconds is refused outright. + +Log what applies when a service starts, so a surprising number can be explained later: + +```python +import logging -SET_MEMORY_BYTES=$((512 * 1024 * 1024)) # an axis left undeclared means the scenario restricts nothing -SET_CPU_CORES=$QUOTA # there, so the sensor reporting nothing is the right answer -SET_CPUSET_CORES=$BENCH_CPUS +import cgroups_sensor -scenario_exec() { container_probe "$1" -m "$SET_MEMORY_BYTES" --cpus "$QUOTA" --cpuset-cpus "$(cpuset_list)"; } +logger = logging.getLogger(__name__) +logger.info('resource limits: %s', cgroups_sensor.snapshot()) + +for notice in cgroups_sensor.describe().notices: + logger.info('%s: %s', notice.code, notice.message) ``` -`scenario_exec` must keep its stdout pure probe JSON, so setup noise goes to stderr; an optional -`scenario_cleanup` runs in a trap. A new scenario is one such file — it is picked up and reported automatically. +## What it reports + +Which call for which job: `get_memory_budget().available` to size a memory budget, `get_cpu_limit()` to size a pool, `CpuLoad().sample()` or `get_cpu_used_ratio()` for a load, `describe()` when a number looks wrong. `get_cpu_usage()` is a raw counter - do not divide it by `get_cpu_limit()` yourself, because the limit can come from a level above this process and the two would describe different scopes; that is what `CpuLoad` is for. -| scenario | shape it exercises | +| Call | Answers | | --- | --- | -| `bare` | no limits at all, so every reading should fall back to host values | -| `systemd-own` | deep chain, limits on the process's own cgroup, cpuset delegated | -| `systemd-ancestor` | limit on an ancestor while the leaf carries no memory controller files | -| `systemd-quota-above-host` | a cpu quota larger than the machine has cores, which is a permission, not a resource | -| `systemd-memory-above-host` | the same on the memory axis: a limit larger than the machine has RAM | -| `docker-private` | private cgroupns (docker's default), all three axes at once | -| `docker-memory-only` | memory alone, CPU falls back to host | -| `docker-cpuset-only` | cpuset alone, memory falls back to host | -| `docker-host-ns` | `--cgroupns=host`, where the container sees the full chain instead of its own root | -| `docker-nested-subgroup` | a tighter cgroup made inside the container: two levels, two different limits | -| `docker-cgroup-parent` | limit on a parent cgroup, set outside the container (needs docker's systemd driver) | -| `k8s-limits` | a pod with container limits, as kubelet writes them | -| `k8s-no-limits` | a pod with nothing set, so every reading should fall back to the node's values | - -## Results - -One `results/.json` per scenario with the scenario's `configured` values, its `status` (`ok`, -`probe_failed`, `skipped`) and everything the probe printed: - -| section | contents | +| `get_memory_budget()` | the memory limit and the memory charged against it, or `None` | +| `get_cpu_limit()` | how many cores may be used, or `None` | +| `get_cpu_usage()` | CPU seconds consumed so far, a counter that only grows | +| `CpuLoad().sample()` | the share of the allowed cores used since the previous call | +| `get_cpu_used_ratio(interval)` | the same, measured across one window it waits out | +| `get_cpu_used_ratio_async(interval)` | the same, for asyncio | +| `snapshot()` | all of the above except the ratio, taken at once | +| `describe()` | the readings again, where they came from, the raw values, and why any are missing | +| `get_machine_cpu_count()` | the cores the kernel lists as online, whatever this process may use | +| `get_machine_memory_bytes()` | the total memory of the machine | +| `clear_cache()` | forget the discovered files after the process was moved to another cgroup | + +`MemoryBudget` carries `limit` and `working_set`, and reports `available` and `used_ratio` derived from them. `available` is the memory this process can still allocate before something kills it, and it is the number to size a budget from. Where several levels hold a limit, that distance is the smallest one along the chain, moved onto the tightest limit so the pair stays comparable. An out-of-memory kill follows from that distance rather than from a ratio, which is why it is the number kept exact. + +`limit` and `working_set` describe the cgroup the limit was found at, and that is not always the cgroup of this process: a limit on a slice or a Kubernetes pod restricts everything under it, and the memory of everything under it is charged against it. Where that happens `used_ratio` is the share of that whole level, not of this process. `describe().memory_limit_level` names the level. `available` is unaffected: the room left there is the room left here. + +The working set excludes reclaimable file cache, the same way `docker stats` and `kubectl top` do. Those tools read one cgroup and never walk up, so they agree with this only while a single level is visible. + +The CPU works the same way. A quota often sits above the process - a systemd scope carries none of its own, and the slice above it does - and the kernel then throttles that whole level, siblings included. The load is therefore measured where the quota binds, not in the group of this process, which would report an idle service inside a saturated slice. + +## What it does not do + +It reports two facts about the machine and no more: the online cores and the total memory, which are the numbers the filters compare against and the ones a consumer needs when a reading is `None`. Anything else about the machine - free memory, load, per-process figures - is what `psutil` is for. It does not read the affinity of the process, because `taskset` narrows one process without narrowing the cgroup its CPU time is accounted to. It never logs: everything that was dropped, and why, is available from `describe()`. + +## Diagnostics + +`describe()` explains a reading that looks wrong. It carries the readings themselves, so one dump answers what was reported as well as why, and around them a `Source` per metric - the mechanism it was read through and the levels searched - the raw values before filtering, the machine it compared against, the levels the memory and the CPU limit actually came from, and a notice for every reading that is not there. A reading of `None` with no notice about it means the mechanism was there and nothing limited this process in a way that kills it - only hard limits are read, and `memory.high` throttles reclaim instead. The levels searched are not the levels a reading came from: a level carries no files until a limit is written there, and it is kept in the chain regardless. + +`Source.interface` is an `Interface` member, and a notice carries a `NoticeCode`. Branch on those rather than on the strings they print as: + +| `NoticeCode` | Meaning | | --- | --- | -| `sensor` | what `crawlee._utils.cgroup` reports: memory limit and working set, cpu quota, cpuset size, cpu time, and the levels it walked | -| `derived` | what `crawlee._utils.system` makes of that in `get_memory_info()` and `get_cpu_info()` | -| `evidence` | verbatim contents of the cgroup control files at every level, never interpreted | -| `host` | kernel, cores and RAM of the machine | -| `errors` | readings that raised; the value becomes `null` and the probe carries on | - -`report.py` renders two tables. In the first, each limit has a `set` column next to a `read` one: a value under -`set` with a dash under `read` is a limit the sensor missed, and dashes under both mean the scenario restricts -nothing there and the sensor agrees. `mem in use` is the memory charged against the limit excluding reclaimable -page cache — what `docker stats` shows. The second table is the same run seen through `get_memory_info()` and -`get_cpu_info()`, where a limit either reaches the caller or falls back to host values. When a reading looks -wrong, `evidence` says who is at fault: the limit is in the control files, or it never got there. +| `MEMORY_LIMIT_COVERS_MACHINE` | the limit is at least the memory of the machine | +| `MEMORY_USAGE_UNAVAILABLE` | a limit was found, but no usage to pair it with | +| `MACHINE_MEMORY_UNKNOWN` | the memory of the machine cannot be read, so a sentinel cannot be told apart | +| `CPU_QUOTA_COVERS_MACHINE` | the quota is at least the cores of the machine | +| `CPU_SET_COVERS_MACHINE` | the set covers every core of the machine | +| `CPU_USAGE_SCOPE_MISMATCH` | the level the CPU limit applies to counts no CPU time, so no rate can be measured there | +| `MEMORY_METRICS_UNAVAILABLE` | nothing here carries a memory limit at all, which is what a machine without cgroups looks like | +| `CPU_METRICS_UNAVAILABLE` | nothing here carries a CPU limit at all, for the same reasons | +| `MEMORY_LIMIT_UNREADABLE` | a level holds a memory limit that says nothing usable, so what it enforces is unknown | +| `CPU_LIMIT_UNREADABLE` | the same for a CPU limit | diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..5559eda --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,141 @@ +[build-system] +requires = ["uv_build>=0.12.0,<0.13.0"] +build-backend = "uv_build" + +[project] +name = "cgroups-sensor" +version = "0.1.0" +description = "Resource limits that actually apply to the running process, read from cgroups" +authors = [ + { name = "Apify Technologies s.r.o.", email = "support@apify.com" }, + { name = "Max Bohomolov"}, + { name = "Josef Procházka"}, + ] +license = "Apache-2.0" +license-files = ["LICENSE"] +readme = "README.md" +requires-python = ">=3.10" +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Software Development :: Libraries", + "Typing :: Typed", +] +keywords = [ + "apify", + "cgroups", + "container", + "cpu", + "docker", + "kubernetes", + "limits", + "memory", +] +dependencies = [] + +[project.urls] +"Apify Homepage" = "https://apify.com" +"Issue Tracker" = "https://github.com/apify/cgroups-sensor/issues" +"Source Code" = "https://github.com/apify/cgroups-sensor" + +[dependency-groups] +dev = [ + "poethepoet<1.0.0", + "pytest<10.0.0", + "ruff~=0.16.0", + # Pre-1.0 and released often. Bounded so that a nightly cannot change what CI enforces on its own; the + # lockfile pins the exact build. + "ty>=0.0.72,<0.1.0", +] + +[tool.ruff] +line-length = 120 +include = [ + "**/*.py", + # Ruff formats Python code blocks embedded in Markdown files. + "**/*.md", +] + +[tool.ruff.lint] +select = ["ALL"] +ignore = [ + "COM812", # Conflicts with the formatter + "CPY001", # Missing copyright notice at top of file + "D100", # Missing docstring in public module + "D104", # Missing docstring in public package + "D203", # One blank line required before class docstring + "D213", # Multi-line docstring summary should start at the second line + "D413", # Missing blank line after last section + "EM102", # Exception must not use an f-string literal + "ISC001", # Conflicts with the formatter + "TRY003", # Avoid specifying long messages outside the exception class +] + +[tool.ruff.format] +quote-style = "single" +indent-style = "space" + +[tool.ruff.lint.per-file-ignores] +"tests/**" = [ + "D", # Docstring style is relaxed in tests + "PLR2004", # Magic value used in comparison + "S101", # Use of assert detected + "S603", # subprocess call: check for execution of untrusted input + "S607", # Starting a process with a partial executable path + "SLF001", # Private member accessed + "T20", # The e2e probe prints its result, that is its whole contract +] + +[tool.ruff.lint.flake8-quotes] +docstring-quotes = "double" +inline-quotes = "single" + +[tool.ruff.lint.isort] +known-first-party = ["cgroups_sensor"] + +[tool.pytest.ini_options] +# `--strict-markers` because the gates of the e2e suite are `usefixtures` marks: a mistyped one would run the +# test without its gate. `--strict-config` and `filterwarnings` fail on a typo here and on a deprecation in +# what the package is read through, which for a stdlib-only package is a real signal. +addopts = "-r a --verbose --strict-markers --strict-config" +filterwarnings = ["error"] +# The e2e readings carry a dozen fields, and a truncated one hides the number that explains the failure. +verbosity_assertions = 2 + +[tool.ty.environment] +python-version = "3.10" + +[tool.ty.src] +include = ["src", "tests"] + +[tool.ty.rules] +unused-ignore-comment = "error" + +# Run tasks with: uv run poe +[tool.poe.tasks] +clean = "rm -rf .pytest_cache .ruff_cache build dist" +# `--locked` so a lock that no longer matches `pyproject.toml` fails here rather than resolving anew, which +# is what makes a guest run and a CI run install the same versions. +install-dev = "uv sync --group dev --locked" +build = "uv build --verbose" +type-check = "uv run ty check" +unit-tests = "uv run pytest tests/unit" +# Both name a whole suite. Anything passed to them is appended, so a subset is `uv run pytest ` instead: +# a path given here would be collected next to the directory rather than in place of it. +e2e-tests = "uv run pytest tests/e2e" +check-code = ["lint", "type-check", "unit-tests"] + +[tool.poe.tasks.lint] +shell = "uv run ruff format --check && uv run ruff check" + +[tool.poe.tasks.format] +shell = "uv run ruff check --fix && uv run ruff format" + +[tool.uv.build-backend] +source-include = ["CHANGELOG.md"] diff --git a/src/cgroups_sensor/__init__.py b/src/cgroups_sensor/__init__.py new file mode 100644 index 0000000..5e91420 --- /dev/null +++ b/src/cgroups_sensor/__init__.py @@ -0,0 +1,61 @@ +from ._sensor import ( + CpuLoad, + Description, + Interface, + MemoryBudget, + Notice, + NoticeCode, + Snapshot, + Source, + clear_cache, + describe, + get_cpu_limit, + get_cpu_usage, + get_cpu_used_ratio, + get_cpu_used_ratio_async, + get_machine_cpu_count, + get_machine_memory_bytes, + get_memory_budget, + snapshot, +) + + +def __getattr__(name: str) -> str: + """Read `__version__` from the installed metadata, the first time anything asks for it. + + Reading it costs more than everything else this package does at import time, and a consumer that only + wants the limits never asks. `importlib.metadata` is therefore imported here rather than above. + """ + if name != '__version__': + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') + + from importlib import metadata # noqa: PLC0415 + + try: + return metadata.version('cgroups-sensor') + except metadata.PackageNotFoundError: + # Imported from a source tree nothing installed, which is how the end-to-end tests run it. + return 'unknown' + + +__all__ = [ + 'CpuLoad', + 'Description', + 'Interface', + 'MemoryBudget', + 'Notice', + 'NoticeCode', + 'Snapshot', + 'Source', + '__version__', + 'clear_cache', + 'describe', + 'get_cpu_limit', + 'get_cpu_usage', + 'get_cpu_used_ratio', + 'get_cpu_used_ratio_async', + 'get_machine_cpu_count', + 'get_machine_memory_bytes', + 'get_memory_budget', + 'snapshot', +] diff --git a/src/cgroups_sensor/_cgroup.py b/src/cgroups_sensor/_cgroup.py new file mode 100644 index 0000000..cb850e6 --- /dev/null +++ b/src/cgroups_sensor/_cgroup.py @@ -0,0 +1,976 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path, PurePosixPath + +from ._cpu_list import count_cpu_list + +_PROC_SELF_CGROUP = Path('/proc/self/cgroup') +"""Lists the cgroup this process belongs to. One line per mounted hierarchy.""" + +_PROC_SELF_MOUNTINFO = Path('/proc/self/mountinfo') +"""Lists the mounted filesystems. Read to locate the hierarchies instead of assuming `/sys/fs/cgroup`.""" + +_MICROSECONDS_PER_SECOND = 1_000_000 +_NANOSECONDS_PER_SECOND = 1_000_000_000 + +_V1_CONTROLLER_NAMES = frozenset({'memory', 'cpu', 'cpuacct', 'cpuset'}) +"""The controllers worth recording.""" + +_CONVENTIONAL_MOUNT_POINT = Path('/sys/fs/cgroup') +"""Where a hierarchy is mounted unless someone chose otherwise. Used only to break a tie.""" + + +@dataclass(frozen=True) +class _FileNames: + """What one cgroup interface calls the files this module reads. + + The two interfaces differ in the names far more than in the meaning, so the names live here and nowhere + else. Each file is also the probe that says its controller is present at a level. + """ + + memory_limit: str + """Holds the hard memory limit.""" + + memory_usage: str + """Holds the memory charged to the cgroup, page cache included.""" + + memory_stat: str + """Holds the breakdown of that memory, as ` ` lines.""" + + inactive_file: str + """The `memory_stat` key holding the reclaimable file cache.""" + + cpu_quota: str + """Holds the CPU bandwidth quota.""" + + cpu_usage: str + """Holds the consumed CPU time.""" + + cpu_set: str + """Holds the cores the cgroup may run on.""" + + +_V2 = _FileNames( + memory_limit='memory.max', + memory_usage='memory.current', + memory_stat='memory.stat', + inactive_file='inactive_file', + cpu_quota='cpu.max', + cpu_usage='cpu.stat', + cpu_set='cpuset.cpus.effective', +) + +_V1 = _FileNames( + memory_limit='memory.limit_in_bytes', + memory_usage='memory.usage_in_bytes', + memory_stat='memory.stat', + inactive_file='total_inactive_file', + cpu_quota='cpu.cfs_quota_us', + cpu_usage='cpuacct.usage', + cpu_set='cpuset.cpus', +) + +_V2_UNLIMITED = 'max' +"""How cgroup v2 spells "no limit" in every file that can hold one. cgroup v1 has no such word: its memory +limit carries a sentinel near 2**63, and its CPU quota a negative number.""" + +_V1_CPU_PERIOD = 'cpu.cfs_period_us' +"""The period a cgroup v1 quota is spelled against. It has no place in the table above: cgroup v2 keeps the +quota and the period in one file.""" + +_V1_CPU_SET_EFFECTIVE = 'cpuset.effective_cpus' +"""What a cgroup v1 cpuset may really run on, inheritance resolved. `cpuset.cpus` holds what was configured +there, and an empty one means "whatever the parent allows".""" + +_V2_CONTROLLERS = 'cgroup.controllers' +"""Lists the controllers bound to a cgroup v2 group. It carries no metric, so it has no place in the table +either - it is read to tell a real unified hierarchy from the controller-less one a hybrid machine mounts.""" + + +class _UnreadableFileError(Exception): + """A control file is there, and says nothing this module can use. + + No consumer ever sees this: it travels from the level that could not be read up to the reader, which turns + it into a reading of nothing plus the name of that level. Skipping the level instead would answer with a + looser limit from an ancestor - the one wrong answer this module must never give, because a consumer sizes + a budget from it and the kernel then enforces something tighter. + """ + + def __init__(self, directory: Path) -> None: + super().__init__(f'{directory} holds a control file that says nothing usable') + self.directory = directory + """The level that could not be read.""" + + +@dataclass(frozen=True) +class _Mount: + """One line of the mount table, as far as this module cares.""" + + root: str + """The subtree of the filesystem this mount exposes.""" + + point: Path + """The directory it is mounted at.""" + + filesystem: str + """The kind of filesystem, e.g. `cgroup2`.""" + + options: str + """The options of the filesystem itself. A cgroup v1 mount names its controllers among them.""" + + +@dataclass(frozen=True) +class _Hierarchy: + """A mounted cgroup hierarchy and the cgroup this process belongs to in it.""" + + mount_point: Path + """The directory the hierarchy is mounted at.""" + + mount_root: str + """The subtree of the hierarchy the mount exposes. Spelled the same way as `own_path`.""" + + own_path: str + """The cgroup this process belongs to, as `/proc/self/cgroup` spells it.""" + + +@dataclass(frozen=True) +class Controller: + """A located controller: its interface and the directories holding its control files.""" + + is_v2: bool + """Whether the controller provides the cgroup v2 interface. It spells the control files differently.""" + + dirs: tuple[Path, ...] + """The cgroup of this process first, then its ancestors up to the mount point. + + A limit on an ancestor caps everything below it. Under Kubernetes the container, the pod and the QoS class + each get a level of their own. Levels that carry none of the controller's files are kept: a limit can be + written to one of them at any time, and discovery is only made once. + """ + + @property + def names(self) -> _FileNames: + """How this controller's interface spells its files.""" + return _V2 if self.is_v2 else _V1 + + +@dataclass(frozen=True) +class Controllers: + """The controllers that carry resource metrics, as located for this process.""" + + memory: Controller | None + """Carries the memory limit and the memory charged against it.""" + + cpu_quota: Controller | None + """Carries the CPU bandwidth quota.""" + + cpu_usage: Controller | None + """Carries the consumed CPU time. Under cgroup v1 that is `cpuacct`, a controller of its own.""" + + cpu_set: Controller | None + """Carries the set of cores the cgroup may run on.""" + + +@dataclass(frozen=True) +class RawCpuQuota: + """A CPU bandwidth quota and the level it binds at.""" + + cores: float + """The number of cores the quota allows, possibly fractional.""" + + limit_directory: Path + """The cgroup holding the quota. + + The kernel throttles that level as a whole, siblings included. It is often not the group of this process: + a systemd scope carries no quota of its own, and the slice above it does. + """ + + usage_directory: Path | None + """Where the CPU time of that level is counted, or `None` when nothing counts it. + + A rate measured anywhere else would divide the time of one scope by the limit of another, so `None` means + no rate can be measured against this quota at all. + """ + + +@dataclass(frozen=True) +class RawCpuSet: + """A set of allowed cores, the level it was read at, and where the CPU time it restricts is counted.""" + + cores: int + """The number of cores the set allows.""" + + limit_directory: Path + """The cgroup holding the set. + + Not necessarily the group of this process: a set is inherited, so the closest level carrying one restricts + everything below it. + """ + + usage_directory: Path | None + """Where the CPU time of the level the set applies to is counted, or `None` when nothing counts it. + + As for a quota, a rate measured anywhere else would divide the time of one cgroup by the cores allowed to + another, so `None` means no rate can be measured against this set at all. + """ + + +@dataclass(frozen=True) +class RawCpu: + """Everything the CPU control files say, read in one go. + + The two restrictions are read together because a level that says nothing usable concerns both: reporting + one of them while the other is unknown would hand out a limit that is not the tightest. + """ + + quota: RawCpuQuota | None + """The bandwidth quota, or `None` when no level sets one.""" + + cpu_set: RawCpuSet | None + """The set of allowed cores, or `None` when no level sets one.""" + + unreadable_directory: Path | None + """The level whose control file says nothing usable, or `None` when every level answered. + + Both readings are dropped when this is set. The file exists, so what it holds is unknown rather than + absent, and an ancestor's looser number is not a substitute for it. + """ + + +@dataclass(frozen=True) +class _MemoryLevel: + """One level of the chain that holds a memory limit.""" + + limit: int + """The limit that level holds, in bytes.""" + + usage: int | None + """The memory charged to that level, in bytes, or `None` when it cannot be read.""" + + directory: Path + """The cgroup the two were read from.""" + + +@dataclass(frozen=True) +class RawMemory: + """The memory limit and usage as the control files spell them.""" + + limit: int | None + """The tightest limit along the chain, in bytes. `None` when no level holds one. + + Taken as read: cgroup v1 spells "no limit" as a sentinel near 2**63, and it passes through here. + """ + + working_set: int | None + """The memory charged against `limit`, in bytes. Excludes reclaimable file cache. + + Not the content of one file. Where several levels hold a limit, `limit - working_set` is the memory that + can still be allocated before the tightest of them is reached, whichever level that is. An out-of-memory + kill follows from that distance, so it is the quantity worth preserving. `None` when the tightest limit + has no usage to pair it with. + """ + + limit_directory: Path | None + """The cgroup holding `limit`, or `None` when no level holds one. + + The whole chain is walked and the tightest limit wins, which is often not the one of this process. The + number alone therefore does not say which level it came from. + """ + + unreadable_directory: Path | None + """The level whose limit file says nothing usable, or `None` when every level answered. + + The reading is dropped when this is set, rather than falling back to a level that did answer: every such + level is looser, and the kernel enforces the one that did not. + """ + + +_NO_MEMORY = RawMemory(limit=None, working_set=None, limit_directory=None, unreadable_directory=None) +"""What is reported where no level holds a memory limit at all.""" + + +def read_memory() -> RawMemory: + """Read the tightest memory limit along the chain, with a usage that keeps the distance to it. + + An out-of-memory kill follows from how much memory can still be allocated, not from a ratio. The smallest + distance to a limit anywhere along the chain is therefore the quantity carried over, expressed against the + tightest limit so that a consumer sees one comparable pair. + """ + controller = locate_controllers().memory + if controller is None: + return _NO_MEMORY + + try: + levels = _read_memory_levels(controller) + except _UnreadableFileError as unreadable: + return RawMemory( + limit=None, + working_set=None, + limit_directory=None, + unreadable_directory=unreadable.directory, + ) + + if not levels: + return _NO_MEMORY + + tightest = min(levels, key=lambda level: level.limit) + + # Any level can be the closest to its own limit: memory is charged up the whole chain, so an ancestor + # counts what its other children use as well. A pod at 96 of its 100 bytes can sit under a QoS class at + # 498 of 500 - the tighter limit is the class's, the tighter distance the pod's. Which is why every level + # is measured below, and not just this one. + # + # Any level whose usage cannot be read leaves one distance unknown, and an unknown may be the smallest. + # Taking the minimum of what is left would then promise memory the kernel will not give, so the pair is + # dropped and the ceiling reported alone. + if any(level.usage is None for level in levels): + return RawMemory( + limit=tightest.limit, + working_set=None, + limit_directory=tightest.directory, + unreadable_directory=None, + ) + + distances = [level.limit - level.usage for level in levels if level.usage is not None] + + # The tightest level is among these, so the smallest distance never exceeds its limit and the working set + # never goes negative. It can still exceed the limit the other way: a cgroup sits above its limit while + # the kernel reclaims, which makes that level's distance negative. + used = min(tightest.limit - min(distances), tightest.limit) + + return RawMemory( + limit=tightest.limit, + working_set=used, + limit_directory=tightest.directory, + unreadable_directory=None, + ) + + +def _read_memory_levels(controller: Controller) -> list[_MemoryLevel]: + """Read the limit and the usage of every level along the chain that holds a limit. + + A level with no readable usage is kept: its limit still caps everything below it, and dropping it would + raise the ceiling above what the kernel enforces. + + Raises: + _UnreadableFileError: If a level holds a limit file that says nothing usable. + """ + levels = [] + + for directory in controller.dirs: + # Only the hard limit counts. `memory.high` throttles reclaim instead of killing, so a cgroup can sit + # above it indefinitely. + limit = _read_limit(directory / controller.names.memory_limit) + if limit is None: + continue + + # Zero is a limit, and the tightest one a cgroup can hold. A negative number of bytes is not a limit + # at all, and no kernel writes one - so it is a file this module cannot use rather than an absent one. + if limit < 0: + raise _UnreadableFileError(directory) + + levels.append(_MemoryLevel(limit=limit, usage=_read_working_set(controller, directory), directory=directory)) + + return levels + + +def read_cpu() -> RawCpu: + """Read both CPU restrictions, and the level that said nothing usable if there was one.""" + try: + return RawCpu(quota=read_cpu_quota(), cpu_set=read_cpu_set_size(), unreadable_directory=None) + except _UnreadableFileError as unreadable: + return RawCpu(quota=None, cpu_set=None, unreadable_directory=unreadable.directory) + + +def read_cpu_quota() -> RawCpuQuota | None: + """Read the tightest CPU bandwidth quota along the chain, and the level it binds at. + + Returns: + The quota, or `None` when no level sets one. Taken as read: a quota can exceed the cores of the + machine. + + Raises: + _UnreadableFileError: If a level holds a quota file that says nothing usable. `read_cpu` turns that into a + reading of nothing, which is why it is the call the sensor layer makes. + """ + controller = locate_controllers().cpu_quota + if controller is None: + return None + + read_quota = _read_cpu_quota_v2 if controller.is_v2 else _read_cpu_quota_v1 + + # Levels without a readable quota drop out. An unlimited ancestor must not hide a quota set below it. + quotas = [(quota, directory) for directory in controller.dirs if (quota := read_quota(directory)) is not None] + if not quotas: + return None + + cores, directory = min(quotas, key=lambda found: found[0]) + + return RawCpuQuota( + cores=cores, + limit_directory=directory, + usage_directory=_cpu_usage_dir(directory, controller), + ) + + +def read_cpu_set_size() -> RawCpuSet | None: + """Read the set of CPU cores the cgroup of this process may run on. + + The cgroup is read, not the process affinity. A `taskset` narrows one process without narrowing the cgroup. + + Returns: + The set, or `None` when no level sets one. Taken as read: a set can cover every core of the machine. + + Raises: + _UnreadableFileError: If the level holds a set that says nothing usable. `read_cpu` turns that into a + reading of nothing. + """ + controller = locate_controllers().cpu_set + if controller is None: + return None + + # Only the closest level carrying a set is read. Under cgroup v2 the effective set already accounts for + # the ancestors, and under cgroup v1 the effective file below does the same. + directory = _first_with(controller.dirs, controller.names.cpu_set) + if directory is None: + return None + + cores = _read_cpu_set(directory, controller) + if cores is None: + return None + + # The set restricts the level it was read at, so that is the level whose time it has to be compared + # against. It is looked up exactly as a quota's is: taking whatever level happens to carry a counter would + # pair a set of two cores with the CPU time of an ancestor, or of the whole machine. + return RawCpuSet( + cores=cores, + limit_directory=directory, + usage_directory=_cpu_usage_dir(directory, controller), + ) + + +def _read_cpu_set(directory: Path, controller: Controller) -> int | None: + """Count the cores one level allows. `None` when it sets none. + + Under cgroup v1 the configured file is empty where the set is inherited, and the effective file resolves + that - so it is read first, and the configured one answers on kernels too old to have it. cgroup v2 spells + only the effective set, which already accounts for the ancestors. + + Raises: + _UnreadableFileError: If a file is there and holds no list of cores. + """ + names = (controller.names.cpu_set,) if controller.is_v2 else (_V1_CPU_SET_EFFECTIVE, controller.names.cpu_set) + + for name in names: + cpu_list = _read_control_file(directory / name) + if cpu_list is None: + continue + + # An empty file is how cgroup v1 spells an inherited set. The effective file above answers for it. + if not cpu_list: + continue + + cores = count_cpu_list(cpu_list) + if cores is None: + raise _UnreadableFileError(directory) + + return cores + + return None + + +def read_cpu_usage(directory: Path | None = None) -> float | None: + """Read the CPU time consumed since the cgroup was created, in seconds. + + Args: + directory: The level to read. Defaults to the closest level along the chain that counts any time. A + quota that binds on an ancestor has to be paired with the time consumed at that ancestor, siblings + included. + + Returns: + The cumulative CPU time. `None` when it cannot be read. + """ + controller = locate_controllers().cpu_usage + if controller is None: + return None + + level = directory if directory is not None else _first_with(controller.dirs, controller.names.cpu_usage) + if level is None: + return None + + path = level / controller.names.cpu_usage + + # cgroup v2 keeps the time among other counters, in microseconds. cgroup v1 gives it a file of its own, + # in nanoseconds. + if controller.is_v2: + microseconds = _read_stat_value(path, 'usage_usec') + return microseconds / _MICROSECONDS_PER_SECOND if microseconds is not None else None + + nanoseconds = _read_counter(path) + return nanoseconds / _NANOSECONDS_PER_SECOND if nanoseconds is not None else None + + +@lru_cache(maxsize=1) +def locate_controllers() -> Controllers: + """Locate the control files carrying the resource metrics of this process. + + This is the whole discovery walk, and the rest of this module reads what it finds. It goes: + + 1. `/proc/self/cgroup` says which cgroup this process belongs to, once per hierarchy. The unified + hierarchy is the line with no controller named, and each cgroup v1 hierarchy has a line of its own. + 2. `/proc/self/mountinfo` says where those hierarchies are mounted. Nothing here assumes `/sys/fs/cgroup`, + and a mount can expose a subtree rather than the whole tree. + 3. A mount counts as ours only where it exposes the cgroup from step 1 and that cgroup exists under it. + The same hierarchy can be mounted several times, and the other mounts belong to somebody else. + 4. Each controller is then located per metric, because a machine can serve different metrics through + different interfaces. The unified hierarchy wins where it carries the metric, and cgroup v1 answers + where it does not. The counter of consumed CPU time takes one more test, for the reason + `_locate_cpu_usage` gives. + 5. What is recorded per controller is a chain of directories: the cgroup of this process first, then its + ancestors up to the mount point. `Controller.dirs` says why the whole chain is kept. + + Discovery walks `/proc` and is cached for the lifetime of the process. The control files themselves are + read again on every sample. + """ + try: + unified, v1 = _read_hierarchies() + except (OSError, ValueError): + # Not Linux, `/proc` is not mounted, or its content does not decode. Nothing to read either way. + return Controllers(memory=None, cpu_quota=None, cpu_usage=None, cpu_set=None) + + return Controllers( + memory=_locate_controller(unified, v1, 'memory', v2_probe=_V2.memory_usage, v1_probe=_V1.memory_usage), + cpu_quota=_locate_controller(unified, v1, 'cpu', v2_probe=_V2.cpu_quota, v1_probe=_V1.cpu_quota), + cpu_usage=_locate_cpu_usage(unified, v1), + cpu_set=_locate_controller(unified, v1, 'cpuset', v2_probe=_V2.cpu_set, v1_probe=_V1.cpu_set), + ) + + +def clear_cache() -> None: + """Forget the located controllers. The next reading discovers them again.""" + locate_controllers.cache_clear() + + +# A child of `fork` inherits what its parent discovered. A pre-fork server whose supervisor puts each worker +# into a cgroup of its own would then have every child reading the parent's levels. Discovery is two files. +if hasattr(os, 'register_at_fork'): + os.register_at_fork(after_in_child=clear_cache) + + +def _locate_cpu_usage(unified: _Hierarchy | None, v1: dict[str, _Hierarchy]) -> Controller | None: + """Locate the counter of consumed CPU time. The hierarchy carrying the limits wins. + + Every other metric is probed by a file that exists only where its controller does. `cpu.stat` is not such + a file: a cgroup v2 group has one whether or not the CPU controller is enabled for it, so its presence + says nothing about where the metrics live. A hybrid machine mounts a controller-less cgroup2 next to the + cgroup v1 controllers, and names this process as belonging to another cgroup there - counting the time of + somebody else's group. So the unified hierarchy answers here only where it carries controllers at all, or + where nothing else counts anything. Under cgroup v1 the accounting is `cpuacct`, a controller of its own, + which can be mounted apart from the quota. + """ + with_controllers = unified if unified is not None and _carries_controllers(unified) else None + counter = _locate_controller(with_controllers, v1, 'cpuacct', v2_probe=_V2.cpu_usage, v1_probe=_V1.cpu_usage) + + return counter or _controller_in(unified, probe=_V2.cpu_usage, is_v2=True) + + +def _carries_controllers(hierarchy: _Hierarchy) -> bool: + """Whether any controller is bound to this cgroup2 hierarchy. + + The mount lists them at its top. A hybrid machine leaves that list empty, because every controller is + mounted on a cgroup v1 hierarchy of its own instead. + """ + try: + return bool((hierarchy.mount_point / _V2_CONTROLLERS).read_text().strip()) + except (OSError, ValueError): + return False + + +def _cpu_usage_dir(directory: Path, source: Controller) -> Path | None: + """Find where the CPU time of one level is counted, given that level in another hierarchy. + + Under cgroup v1 the quota and the accounting can be two hierarchies with two mount points, so the level + is translated by its path below the mount point. Every chain ends at its own mount point, which is what + makes that path comparable. + + The translation is textual, so the result has to be one of the levels this process belongs to. Two + hierarchies can name different cgroups for the same process, and a group of the same name in the other + hierarchy then belongs to somebody else - its counter would be read as ours. `None` in that case, and + `None` where the level carries no counter: no rate can be measured either way. + """ + usage = locate_controllers().cpu_usage + if usage is None: + return None + + try: + relative = directory.relative_to(source.dirs[-1]) + except ValueError: + return None + + translated = usage.dirs[-1] / relative + if translated not in usage.dirs: + return None + + return translated if _exists(translated / usage.names.cpu_usage) else None + + +def _read_working_set(controller: Controller, directory: Path) -> int | None: + """Read the memory charged to one cgroup, in bytes. Excludes reclaimable file cache. + + The raw usage counts the page cache, which the kernel drops on demand. Subtracting the inactive file cache + gives the working set - the figure `docker stats` and `kubectl top` report. + """ + current = _read_counter(directory / controller.names.memory_usage) + if current is None: + return None + + # No fallback to the raw usage. That would count reclaimable cache as usage. + inactive_file = _read_stat_value(directory / controller.names.memory_stat, controller.names.inactive_file) + if inactive_file is None: + return None + + return max(current - inactive_file, 0) + + +def _locate_controller( + unified: _Hierarchy | None, + v1: dict[str, _Hierarchy], + v1_name: str, + *, + v2_probe: str, + v1_probe: str, +) -> Controller | None: + """Locate the directories of one controller. The cgroup v2 unified hierarchy wins. + + A hybrid system can mount both interfaces with only some controllers on the unified hierarchy, so each + candidate counts only where the file it is probed by exists. + """ + return _controller_in(unified, probe=v2_probe, is_v2=True) or _controller_in( + v1.get(v1_name), probe=v1_probe, is_v2=False + ) + + +def _controller_in(hierarchy: _Hierarchy | None, *, probe: str, is_v2: bool) -> Controller | None: + """Locate one controller in one hierarchy, if that hierarchy carries it at all. + + The whole chain is kept, not only the levels that carry the controller today. A cgroup gets a controller's + files once its parent enables that controller for its children, and `systemctl set-property` does exactly + that at runtime. Discovery happens once per process, so a chain trimmed at startup would hide such a limit + for the lifetime of the process. + """ + if hierarchy is None: + return None + + dirs = _candidate_dirs(hierarchy) + + return Controller(is_v2=is_v2, dirs=dirs) if _first_with(dirs, probe) is not None else None + + +def _first_with(dirs: tuple[Path, ...], file_name: str) -> Path | None: + """Find the closest level that carries one file, or `None` when no level does.""" + return next((directory for directory in dirs if _exists(directory / file_name)), None) + + +def _exists(path: Path) -> bool: + """Whether a path exists. + + `Path.exists()` raises on a directory this process may not traverse. Only Python 3.14 turns that into + `False`, and a cgroup chain can hold such a directory. + """ + try: + return path.exists() + except OSError: + return False + + +def _candidate_dirs(hierarchy: _Hierarchy) -> tuple[Path, ...]: + """List the directories a controller's files can be read from. The cgroup of this process first.""" + parts = _own_path_within_mount(hierarchy) + + # Descend from the mount point one level at a time, keeping each. The walk starts there because nothing + # above the mount belongs to the hierarchy. + chain = [hierarchy.mount_point] + for part in parts: + chain.append(chain[-1] / part) + + # The cgroup of this process first, then its ancestors. + return tuple(reversed(chain)) + + +def _own_path_within_mount(hierarchy: _Hierarchy) -> tuple[str, ...]: + """Spell the cgroup of this process as the path components below the mount point.""" + path = PurePosixPath(hierarchy.own_path) + + # A mount can expose just a subtree. The paths in `/proc/self/cgroup` then carry the mount root as a + # prefix, which has to come off. + if hierarchy.mount_root != '/': + try: + path = PurePosixPath('/') / path.relative_to(hierarchy.mount_root) + except ValueError: + # The mount does not cover the cgroup of this process. Only the top of the mount is readable. + path = PurePosixPath('/') + + return path.parts[1:] if path.is_absolute() else path.parts + + +def _read_hierarchies() -> tuple[_Hierarchy | None, dict[str, _Hierarchy]]: + """Locate the mounted cgroup hierarchies and the cgroup this process belongs to in each. + + Returns: + The cgroup v2 unified hierarchy, and the cgroup v1 hierarchies keyed by controller. + + Raises: + OSError: If `/proc/self/mountinfo` or `/proc/self/cgroup` cannot be read. + """ + unified_path, controller_paths = _read_own_paths() + mounts = _read_mounts() + + controllers: dict[str, _Hierarchy] = {} + + for mount in mounts: + if mount.filesystem != 'cgroup': + continue + + # A cgroup v1 mount names its controllers among its options, e.g. `rw,cpu,cpuacct`. + for option in mount.options.split(','): + own_path = controller_paths.get(option) + if option not in _V1_CONTROLLER_NAMES or own_path is None: + continue + + # A controller can be mounted more than once, so the same rule as for the unified hierarchy: a + # mount that does not expose the cgroup of this process belongs to somebody else. The first mount + # that does wins, because mounts are listed in the order they were made. + hierarchy = _Hierarchy(mount_point=mount.point, mount_root=mount.root, own_path=own_path) + if _covers_own_cgroup(hierarchy): + controllers.setdefault(option, hierarchy) + + unified = _pick_unified(mounts, unified_path) if unified_path is not None else None + + return unified, controllers + + +def _pick_unified(mounts: list[_Mount], own_path: str) -> _Hierarchy | None: + """Pick the cgroup2 mount that exposes the cgroup of this process. + + The same hierarchy can be mounted more than once. An agent that watches the machine from inside a + container bind-mounts the whole of it somewhere else, and a runtime can expose another subtree entirely. + Taking the first line of the mount table would then report another cgroup as this one, so a mount counts + only where it covers the cgroup of this process and that cgroup exists under it. + + Where several still qualify, the conventional mount point wins, and after that the order the mounts were + made in. Where none does, there is no unified hierarchy to read: the cgroup v1 hierarchies are left to + answer instead, and where there are none every reading is `None`. Both are better answers than the numbers + of a cgroup that is not this one. + """ + hierarchies = ( + _Hierarchy(mount_point=mount.point, mount_root=mount.root, own_path=own_path) + for mount in mounts + if mount.filesystem == 'cgroup2' + ) + + covering = [hierarchy for hierarchy in hierarchies if _covers_own_cgroup(hierarchy)] + if not covering: + return None + + conventional = [hierarchy for hierarchy in covering if hierarchy.mount_point == _CONVENTIONAL_MOUNT_POINT] + + return (conventional or covering)[0] + + +def _covers_own_cgroup(hierarchy: _Hierarchy) -> bool: + """Whether one mount exposes the cgroup of this process, and that cgroup exists under it.""" + mount_root = PurePosixPath(hierarchy.mount_root) + + # A mount of a subtree only covers the cgroups below that subtree. + if mount_root != PurePosixPath('/') and not PurePosixPath(hierarchy.own_path).is_relative_to(mount_root): + return False + + return _exists(_candidate_dirs(hierarchy)[0]) + + +def _read_mounts() -> list[_Mount]: + """Read the mount table, skipping the lines that are not mount entries. + + Raises: + OSError: If `/proc/self/mountinfo` cannot be read. + """ + # `errors='replace'` because a foreign mount point with non-UTF-8 bytes must not hide our own cgroup. + # Such a path decodes to something that matches no file, so only that mount drops out. + lines = _PROC_SELF_MOUNTINFO.read_text(errors='replace').splitlines() + + return [mount for line in lines if (mount := _parse_mount(line)) is not None] + + +def _parse_mount(line: str) -> _Mount | None: + """Read one line of the mount table, or `None` when it is not a mount entry.""" + # A variable number of optional fields sits before the ` - ` separator. Split on it first. + before, separator, after = line.partition(' - ') + if not separator: + return None + + try: + _mount_id, _parent_id, _device, root, point, *_ = before.split(' ') + filesystem, _source, options, *_ = after.split(' ') + except ValueError: + return None + + # `/proc/self/cgroup` spells the same paths unescaped. Without this the two cannot be compared, and the + # mount point cannot be opened either. + return _Mount(root=_unescape(root), point=Path(_unescape(point)), filesystem=filesystem, options=options) + + +def _read_own_paths() -> tuple[str | None, dict[str, str]]: + """Read the cgroup this process belongs to in the unified hierarchy and in each cgroup v1 one. + + Raises: + OSError: If `/proc/self/cgroup` cannot be read. + """ + unified: str | None = None + controllers: dict[str, str] = {} + + for line in _PROC_SELF_CGROUP.read_text(errors='replace').splitlines(): + try: + _hierarchy_id, controller_list, cgroup_path = line.split(':', 2) + except ValueError: + continue + + # The unified hierarchy is the entry with no controllers listed, spelled `0::`. + if not controller_list: + unified = cgroup_path + continue + + for controller in controller_list.split(','): + # A cgroup v1 hierarchy without a controller carries a name instead, e.g. `name=systemd`. + controllers[controller.removeprefix('name=')] = cgroup_path + + return unified, controllers + + +def _unescape(field: str) -> str: + """Decode the octal escapes in a path field of `/proc/self/mountinfo`.""" + # The backslash goes last. Undoing it first would decode `\134040` into a space. + return field.replace('\\040', ' ').replace('\\011', '\t').replace('\\012', '\n').replace('\\134', '\\') + + +def _read_cpu_quota_v2(directory: Path) -> float | None: + """Read the cores allowed by a cgroup v2 `cpu.max` file, which holds the quota and its period. + + Raises: + _UnreadableFileError: If the file is there and describes no bandwidth this module can use. + """ + content = _read_control_file(directory / _V2.cpu_quota) + if content is None: + return None + + # An unlimited cgroup spells the quota as `max`, and keeps the period next to it. + written, _separator, period_written = content.partition(' ') + if written == _V2_UNLIMITED: + return None + + try: + quota, period = int(written), int(period_written) + except ValueError: + raise _UnreadableFileError(directory) from None + + # Neither can be zero or negative on a real kernel, and cgroup v2 has `max` for "no quota". Reading one as + # a limit would hand out a quota of no cores at all, which every consumer would then divide by. + if quota <= 0 or period <= 0: + raise _UnreadableFileError(directory) + + return quota / period + + +def _read_cpu_quota_v1(directory: Path) -> float | None: + """Read the cores allowed by the cgroup v1 quota and period files. + + Raises: + _UnreadableFileError: If a file is there and describes no bandwidth this module can use. + """ + quota = _read_limit(directory / _V1.cpu_quota) + period = _read_limit(directory / _V1_CPU_PERIOD) + if quota is None or period is None: + return None + + # Unlike cgroup v2, this interface has no word for "no quota": it writes a negative number instead. + if quota < 0: + return None + + if quota == 0 or period <= 0: + raise _UnreadableFileError(directory) + + return quota / period + + +def _read_text(path: Path) -> str | None: + """Read a control file. `None` when it cannot be read at all.""" + try: + return path.read_text().strip() + except (OSError, ValueError): + return None + + +def _read_control_file(path: Path) -> str | None: + """Read a control file, telling an absent one from an unreadable one. `None` when the level has no such file. + + Every reader of a limit goes through here, so the rule lives in one place: a level that carries no such + file simply does not limit anything, while one that carries a file nobody can read limits something by an + amount nothing here can name. + + Raises: + _UnreadableFileError: If the file is there and cannot be read. + """ + content = _read_text(path) + if content is None and _exists(path): + raise _UnreadableFileError(path.parent) + + return content + + +def _read_counter(path: Path) -> int | None: + """Read a counter, best effort. `None` when it is missing or says anything but a number. + + A counter that cannot be read costs a rate, and nothing else: there is no looser value to fall back to, + which is what makes best effort right here and wrong for a limit. + """ + content = _read_text(path) + if content is None: + return None + + try: + return int(content) + except ValueError: + return None + + +def _read_limit(path: Path) -> int | None: + """Read a file holding a limit. `None` when the level sets none. + + Raises: + _UnreadableFileError: If the file is there and holds no number this module can use. That is not the same + as no limit, and `_UnreadableFileError` says why. + """ + content = _read_control_file(path) + if content is None or content == _V2_UNLIMITED: + return None + + try: + return int(content) + except ValueError: + raise _UnreadableFileError(path.parent) from None + + +def _read_stat_value(path: Path, key: str) -> int | None: + """Read one entry of a control file holding ` ` lines.""" + try: + with path.open() as file: + for line in file: + entry_key, _separator, value = line.partition(' ') + if entry_key == key: + return int(value) + except (OSError, ValueError): + return None + + return None diff --git a/src/cgroups_sensor/_cpu_list.py b/src/cgroups_sensor/_cpu_list.py new file mode 100644 index 0000000..caf2fef --- /dev/null +++ b/src/cgroups_sensor/_cpu_list.py @@ -0,0 +1,30 @@ +from __future__ import annotations + + +def count_cpu_list(cpu_list: str) -> int | None: + """Count the CPUs in a list of ranges and single numbers, e.g. `0-3,8`. + + The kernel writes this format in more than one place: `cpuset.cpus` inside a cgroup, and + `/sys/devices/system/cpu/online` for the machine. It belongs to neither layer, so it lives here. + + Returns: + The number of cores, or `None` when the list does not parse. A reversed range does not parse: a real + kernel never writes one, but an emulated cgroupfs can, and counting it would yield zero cores. + """ + count = 0 + + try: + for part in cpu_list.split(','): + first, separator, last = part.partition('-') + # A single core carries no separator. One that carries it has to carry both ends too. + if separator and not last: + return None + + span = int(last or first) - int(first) + 1 + if span <= 0: + return None + count += span + except ValueError: + return None + + return count diff --git a/src/cgroups_sensor/_sensor.py b/src/cgroups_sensor/_sensor.py new file mode 100644 index 0000000..35413e8 --- /dev/null +++ b/src/cgroups_sensor/_sensor.py @@ -0,0 +1,842 @@ +from __future__ import annotations + +import os +import threading +import time +from dataclasses import dataclass +from enum import Enum +from pathlib import Path + +from . import _cgroup +from ._cpu_list import count_cpu_list + +_PROC_MEMINFO = Path('/proc/meminfo') +"""Holds the total memory of the machine.""" + +_SYS_CPU_ONLINE = Path('/sys/devices/system/cpu/online') +"""Lists the cores of the machine that are online, e.g. `0-3`.""" + +_SHORTEST_WINDOW_SECONDS = 0.01 +"""Below this a rate says more about the counter's own resolution than about the load. + +The kernel advances the consumed time in steps of a scheduler tick or so. Two readings taken closer together +than that usually differ by nothing, which would read as an idle process however busy it is. +""" + + +@dataclass(frozen=True) +class MemoryBudget: + """A memory limit that actually restricts this process, with the usage charged against it. + + `available` is what this is for: the memory this process can still allocate. That is the number to size a + budget from, and the one this pair is built to keep exact. + + The other two describe the cgroup the limit was found at, which is not always the cgroup of this process - + a limit on a slice or a pod restricts everything under it. Where that happens, `working_set` counts the + memory of everything under that level and `used_ratio` is its share, not this process's. + `describe().memory_limit_level` names the level, and `available` stays the honest answer either way: the + room left there is the room left here. + """ + + limit: int + """The tightest limit in bytes. Always below the memory of the machine.""" + + working_set: int + """The memory charged against the limit, in bytes. Excludes reclaimable file cache. + + Where several levels hold a limit, this is the smallest distance to one along the chain, moved onto the + tightest limit so the pair stays comparable. An out-of-memory kill follows from that distance rather than + from a ratio, which is why `available` is the number kept exact. + """ + + @property + def available(self) -> int: + """The memory this process can still allocate before something kills it, in bytes.""" + return self.limit - self.working_set + + @property + def used_ratio(self) -> float: + """The share of the limit in use, between 0 and 1. + + Of the limit, not of this process: where the limit belongs to a level above, sibling cgroups are + charged against it too. + """ + return self.working_set / self.limit if self.limit > 0 else 1.0 + + +@dataclass(frozen=True) +class Snapshot: + """Every reading the sensor takes, taken at once. + + Each reading answers on its own. The CPU counter belongs to the group of this process and the CPU limit + can belong to a level above it, so the three are not a set of numbers to combine. + """ + + memory_budget: MemoryBudget | None + """The memory budget, or `None` when nothing restricts the memory.""" + + cpu_limit: float | None + """The number of usable CPU cores, or `None` when nothing restricts the CPU.""" + + cpu_usage: float | None + """The consumed CPU time in seconds, or `None` when it cannot be read. + + Counted in the group of this process, which need not be the level `cpu_limit` applies to. Dividing one by + the other is therefore not a load: `CpuLoad` and `get_cpu_used_ratio()` pair the two properly. + """ + + +class NoticeCode(str, Enum): + """Why a reading was dropped or a rate could not be measured. + + Compare against these rather than against the strings they carry: the strings are what a log line shows, + the members are what code should branch on. + """ + + # Without this a member prints as `NoticeCode.MEMORY_LIMIT_COVERS_MACHINE`, and an f-string of it differs + # between the supported Python versions. `StrEnum` would do the same, and arrives only in 3.11. + __str__ = str.__str__ + + MEMORY_LIMIT_COVERS_MACHINE = 'memory-limit-covers-machine' + """The limit is at least the memory of the machine, which is how an unrestricted group spells "no limit".""" + + MEMORY_USAGE_UNAVAILABLE = 'memory-usage-unavailable' + """A limit was found, but the level holding it reports no usage to pair with it.""" + + MACHINE_MEMORY_UNKNOWN = 'machine-memory-unknown' + """The memory of the machine cannot be read, so a limit cannot be told apart from a sentinel.""" + + CPU_QUOTA_COVERS_MACHINE = 'cpu-quota-covers-machine' + """The quota is at least the cores of the machine, so it restricts nothing.""" + + CPU_SET_COVERS_MACHINE = 'cpu-set-covers-machine' + """The set of allowed cores covers every core of the machine, so it restricts nothing.""" + + CPU_USAGE_SCOPE_MISMATCH = 'cpu-usage-scope-mismatch' + """The level the CPU limit applies to counts no CPU time, so no rate can be measured against it.""" + + MEMORY_METRICS_UNAVAILABLE = 'memory-metrics-unavailable' + """Nothing here carries a memory limit at all: not Linux, or no cgroup filesystem is mounted.""" + + CPU_METRICS_UNAVAILABLE = 'cpu-metrics-unavailable' + """Nothing here carries a CPU limit at all: not Linux, or no cgroup filesystem is mounted.""" + + MEMORY_LIMIT_UNREADABLE = 'memory-limit-unreadable' + """A level holds a memory limit that says nothing usable, so what it enforces is unknown.""" + + CPU_LIMIT_UNREADABLE = 'cpu-limit-unreadable' + """A level holds a CPU limit that says nothing usable, so what it enforces is unknown.""" + + +@dataclass(frozen=True) +class Notice: + """One reason a reading was dropped.""" + + code: NoticeCode + """What happened, as a member of `NoticeCode`. It carries its own string, so an f-string of it, a `%s` in + a log line and `json.dumps` all show that string rather than the member.""" + + message: str + """A human-readable explanation with the values involved.""" + + +class Interface(str, Enum): + """The mechanism a reading comes from. + + A machine can serve different metrics through different interfaces, so this is reported per metric rather + than once. Compare against these members rather than against the strings they carry. + """ + + # As in `NoticeCode`, so that a member prints as its string on every supported Python version. + __str__ = str.__str__ + + CGROUP_V2 = 'cgroup-v2' + """The unified hierarchy, where one cgroup carries every controller.""" + + CGROUP_V1 = 'cgroup-v1' + """The older interface, where each controller is a hierarchy of its own.""" + + +@dataclass(frozen=True) +class Source: + """Where one metric is read from.""" + + interface: Interface + """The mechanism providing the metric. It carries its own string, as `Notice.code` does.""" + + levels: tuple[str, ...] + """The levels the metric is looked for in. The cgroup of this process first, then its ancestors. + + A level here need not carry the metric's files today. They appear the moment a limit is written there, and + discovery happens only once, so the whole chain is kept. `memory_limit_level` and + `cpu_limit_level` name the levels the readings actually came from. + """ + + +@dataclass(frozen=True) +class Description: + """How the sensor arrived at its readings. + + This is the diagnostic counterpart of `snapshot()`, and it carries the readings themselves, so that one + dump answers what was reported as well as why. When a reading looks wrong, the description shows the + mechanism, the levels, the raw values and the rejections. + + A reading of `None` next to no notice about it means the mechanism was there and nothing limited this + process in a way that kills it. Only hard limits are read: `memory.high` throttles reclaim instead, so a + cgroup can sit above it indefinitely and nothing here reports it. + """ + + memory_budget: MemoryBudget | None + """What `get_memory_budget()` reports, taken at the same moment as everything below.""" + + cpu_limit: float | None + """What `get_cpu_limit()` reports, taken at the same moment as everything below.""" + + cpu_usage: float | None + """What `get_cpu_usage()` reports, taken at the same moment as everything below. + + Counted in the group of this process. A rate is measured at `cpu_rate_level` instead, so where that is + another level, this counter and a rate that looks wrong do not describe the same scope. That is the first + thing to check when they disagree. + """ + + memory_source: Source | None + """Where the memory limit and usage are read from. `None` when no mechanism carries them.""" + + cpu_quota_source: Source | None + """Where the CPU bandwidth quota is read from. `None` when no mechanism carries it.""" + + cpu_set_source: Source | None + """Where the set of allowed cores is read from. `None` when no mechanism carries it.""" + + cpu_usage_source: Source | None + """Where the consumed CPU time is read from, as a mechanism and not as a level. `cpu_usage` says which + group is counted and `cpu_rate_level` where a rate is measured. `None` when no mechanism carries it.""" + + raw_memory_limit: int | None + """The tightest memory limit before filtering. Sentinels included.""" + + raw_memory_working_set: int | None + """The usage paired with the raw limit, in bytes. + + Unlike the other `raw_` fields this one is computed, not read: `raw_memory_limit - raw_memory_working_set` + is the smallest distance to a limit along the chain, moved onto that limit. It matches no single file. The + reclaimable file cache comes off `memory.current`, and where several levels are visible the distance comes + from whichever level is closest to its own limit. `None` when the tightest level has no usage to pair with + it. + """ + + raw_cpu_quota: float | None + """The CPU quota in cores before filtering. It can exceed the machine.""" + + raw_cpu_set_size: int | None + """The number of allowed cores before filtering. It can cover the whole machine.""" + + memory_limit_level: str | None + """The level holding `raw_memory_limit`. `None` when no level holds one. + + The tightest limit of the chain wins, which is often not the cgroup of this process. Under Kubernetes it + is regularly the pod or the QoS class above it. The level is named even where the filters drop that limit, + and a notice then says why. + """ + + cpu_limit_level: str | None + """The level the reported CPU limit was read at. `None` when nothing restricts the CPU. + + Often not the cgroup of this process: a systemd scope carries no quota of its own and the slice above it + does, and a CPU set on an ancestor restricts everything below it. + """ + + cpu_rate_level: str | None + """The level whose consumed CPU time belongs to that limit, which is where a rate is measured. + + Under cgroup v2 it is `cpu_limit_level` itself. Under cgroup v1 the accounting is a controller of its own + and can be mounted elsewhere, so it is the same cgroup under the other mount point. `None` when no level + counts that time, and a notice then says so - no rate can be measured at all in that case. + """ + + machine_memory_bytes: int | None + """The machine memory the filters compared against. `None` when it cannot be read.""" + + machine_cpu_count: int | None + """The machine core count the filters compared against. `None` when it cannot be read.""" + + notices: tuple[Notice, ...] + """Why readings were rejected. Empty when every reading passed.""" + + +def get_memory_budget() -> MemoryBudget | None: + """Get the memory budget that actually restricts this process. + + A limit that covers the whole machine is not reported. That is how an unrestricted group spells "no limit". + A limit is also not reported when the machine memory is unknown, or when no usage metric can be paired with + it. `describe()` explains every rejection. + + Returns: + The budget, or `None` when nothing restricts the memory of this process. + """ + return _evaluate_memory(get_machine_memory_bytes()).effective + + +def get_cpu_limit() -> float | None: + """Get the number of CPU cores this process may actually use. + + A bandwidth quota and a CPU set restrict the CPU independently. The tighter one wins. A reading that covers + the whole machine is ignored. Process affinity (`taskset`) is out of scope - it narrows one process, not its + group. `describe()` shows the readings separately. + + Returns: + The number of cores, possibly fractional. `None` when nothing restricts the CPU of this process. + """ + return _evaluate_cpu_limit(get_machine_cpu_count()).effective + + +def get_cpu_usage() -> float | None: + """Get the CPU time this process's group has consumed, in seconds. + + The value is a counter that only grows, read in the group of this process. Use it to see how much CPU time + has been spent - not to compute a load: `get_cpu_limit()` can come from a level above this process, and + dividing this counter by that limit compares two different scopes. `CpuLoad` and `get_cpu_used_ratio()` + pair both correctly, and they are what a rate should come from. + + Returns: + The cumulative CPU time. `None` when it cannot be read. + """ + return _cgroup.read_cpu_usage() + + +def get_cpu_used_ratio(interval: float = 1.0) -> float | None: + """Measure the CPU usage relative to the cores this process may use. + + Blocks the calling thread while measuring, but only while measuring: where there is nothing to measure it + returns at once, so a loop around it has to pace itself. In asyncio code use `get_cpu_used_ratio_async()`. + + The kernel updates the counter in coarse steps, so a short window is noisy: a tenth of a second can report + an idle process as busy or a busy one as idle. The default is long enough for that to average out. When + sampling repeatedly, use `CpuLoad` instead - it measures across the time between calls and waits for + nothing. + + Args: + interval: How long to measure, in seconds. A tenth of a second is already noisy, and anything below + 0.01 is refused: the counter does not move that fast, so such a window can only report nothing. + + Returns: + The ratio between 0 and 1. `None` when nothing restricts the CPU, when the counter cannot be read, or + when the limit changed while measuring, in value or in level. + + Raises: + ValueError: If `interval` is shorter than a measurable window. An argument is the caller's to get + right, so it is refused rather than answered with `None`. `CpuLoad.sample()` returns `None` for a + window that turned out too short, because there the window is what happened, not what was asked + for. + """ + _check_interval(interval) + + start = _read_cpu() + if start is None: + return None + + time.sleep(interval) + + end = _read_cpu() + + return _used_ratio(start, end) if end is not None else None + + +async def get_cpu_used_ratio_async(interval: float = 1.0) -> float | None: + """Measure the CPU usage relative to the cores this process may use. + + The asyncio variant of `get_cpu_used_ratio()`. Waits with `asyncio.sleep`, so the event loop stays free. + It waits only while measuring: where there is nothing to measure it returns at once, so a loop around it + has to pace itself, or it spins. The same accuracy applies, and `CpuLoad` avoids the wait entirely. + + Args: + interval: How long to measure, in seconds. The same floor of 0.01 applies. + + Returns: + The ratio between 0 and 1. `None` when nothing restricts the CPU, when the counter cannot be read, or + when the limit changed while measuring, in value or in level. + + Raises: + ValueError: If `interval` is shorter than a measurable window. + """ + _check_interval(interval) + + # Imported here: it costs a third of what this package costs to import, and only this one call needs it. + import asyncio # noqa: PLC0415 + + start = _read_cpu() + if start is None: + return None + + await asyncio.sleep(interval) + + end = _read_cpu() + + return _used_ratio(start, end) if end is not None else None + + +def snapshot() -> Snapshot: + """Take every reading at once. + + Returns: + The memory budget, the CPU limit and the CPU usage counter. The rate is not included - it needs a + measurement window. + """ + return Snapshot( + memory_budget=get_memory_budget(), + cpu_limit=get_cpu_limit(), + cpu_usage=get_cpu_usage(), + ) + + +def describe() -> Description: + """Explain how the sensor arrives at its readings. + + Every control file is read again, so a limit changed between two calls shows up here. Where those files + are was discovered once - `clear_cache()` is what forgets that. + + Returns: + The readings, the source of each metric, the raw values before filtering, the machine facts, and a + notice for every reading that is not there. + """ + controllers = _cgroup.locate_controllers() + + # One read of the machine facts for both the report and the filtering. They cannot disagree this way. + machine_memory = get_machine_memory_bytes() + machine_cpus = get_machine_cpu_count() + memory = _evaluate_memory(machine_memory) + cpu = _evaluate_cpu_limit(machine_cpus) + + return Description( + memory_budget=memory.effective, + cpu_limit=cpu.effective, + cpu_usage=get_cpu_usage(), + memory_source=_source(controllers.memory), + cpu_quota_source=_source(controllers.cpu_quota), + cpu_set_source=_source(controllers.cpu_set), + cpu_usage_source=_source(controllers.cpu_usage), + raw_memory_limit=memory.raw.limit, + raw_memory_working_set=memory.raw.working_set, + raw_cpu_quota=cpu.raw_quota, + raw_cpu_set_size=cpu.raw_set_size, + memory_limit_level=str(memory.raw.limit_directory) if memory.raw.limit_directory is not None else None, + cpu_limit_level=str(cpu.limit_directory) if cpu.limit_directory is not None else None, + cpu_rate_level=str(cpu.usage_directory) if cpu.usage_directory is not None else None, + machine_memory_bytes=machine_memory, + machine_cpu_count=machine_cpus, + notices=memory.notices + cpu.notices, + ) + + +def clear_cache() -> None: + """Forget the discovered metric sources. + + Discovery is cached for the lifetime of the process. Call this after the process was moved into another + group. The next reading then locates the sources again. + """ + _cgroup.clear_cache() + + +@dataclass(frozen=True) +class _CpuReading: + """One reading of the CPU counter, with what it has to be compared against.""" + + limit: float + usage: float + taken_at: float + usage_directory: Path | None + """The level the counter was read at, so a later reading can tell that the limit moved.""" + + +def _check_interval(interval: float) -> None: + """Reject a measurement window the counter cannot resolve. + + Raises: + ValueError: If `interval` is shorter than a measurable window. The public callers document it. + """ + # Anything shorter would sleep and then report nothing, because the counter has not moved yet. + if interval < _SHORTEST_WINDOW_SECONDS: + raise ValueError( + f'interval must be at least {_SHORTEST_WINDOW_SECONDS} seconds, got {interval}. ' + 'The kernel advances the consumed time in coarser steps than that.' + ) + + +def _read_cpu() -> _CpuReading | None: + """Read the CPU limit and the consumed time at the level that limit binds at.""" + evaluation = _evaluate_cpu_limit(get_machine_cpu_count()) + if evaluation.effective is None or evaluation.usage_directory is None: + return None + + usage = _cgroup.read_cpu_usage(evaluation.usage_directory) + if usage is None: + return None + + return _CpuReading( + limit=evaluation.effective, + usage=usage, + taken_at=time.monotonic(), + usage_directory=evaluation.usage_directory, + ) + + +def _used_ratio(start: _CpuReading, end: _CpuReading) -> float | None: + """Turn two readings into the share of the allowed cores that was used between them.""" + # A limit that moved level, or changed in place, makes the two readings incomparable: the counters then + # belong to two scopes, or the same consumption divides by two different numbers. + if end.usage_directory != start.usage_directory or end.limit != start.limit: + return None + + elapsed = end.taken_at - start.taken_at + if elapsed < _SHORTEST_WINDOW_SECONDS or end.limit <= 0: + return None + + used_ratio = (end.usage - start.usage) / (elapsed * end.limit) + + # A counter restart makes the difference negative. Clamp both ends. + return min(max(used_ratio, 0.0), 1.0) + + +class CpuLoad: + """Measures the CPU load between calls, without blocking. + + A cgroup reports consumed CPU time as a counter, so a rate needs two readings. This keeps the previous one + and measures against it, which makes the window as long as the interval between calls. That is what makes + it accurate: a short window is dominated by how coarsely the kernel updates the counter, and a sampler + called every few seconds does not pay for that. + + Give each caller a sampler of its own. Two callers sharing one measure each other's windows, and a window + of nearly no time reports nothing at all. + + Safe to call from several threads, which is what makes the sampling of a worker thread safe next to a + `clear_cache()` elsewhere. + """ + + def __init__(self) -> None: + self._previous: _CpuReading | None = None + self._lock = threading.Lock() + + def sample(self) -> float | None: + """Measure the load since the previous call. + + Returns: + The ratio between 0 and 1. `None` on the first call, when nothing restricts the CPU, when the + counter cannot be read, when the limit changed since the previous call, or when that call was too + recent for the counter to have moved. + """ + reading = _read_cpu() + + with self._lock: + previous, self._previous = self._previous, reading if reading is not None else self._previous + + if reading is None or previous is None: + return None + + return _used_ratio(previous, reading) + + +@dataclass(frozen=True) +class _MemoryEvaluation: + """The memory reading with the judgement applied.""" + + effective: MemoryBudget | None + raw: _cgroup.RawMemory + notices: tuple[Notice, ...] + + +_COVERS_MACHINE_MESSAGES = { + NoticeCode.CPU_QUOTA_COVERS_MACHINE: 'The CPU quota of {cores} cores is at least the cores of the machine ' + '({machine_cpus}), so it does not restrict this process.', + NoticeCode.CPU_SET_COVERS_MACHINE: 'The set of {cores} allowed cores covers every core of the machine ' + '({machine_cpus}), so it does not restrict this process.', +} +"""How each CPU reading explains itself away where it covers the machine. + +Kept apart from the reading so that the sentence is built only for a reading that is dropped. `CpuLoad.sample()` +evaluates the CPU limit on every call, and nothing there ever reads these. +""" + + +def _spell_cores(cores: float) -> str: + """Spell a number of cores for a message. + + A quota can allow half a core, so the number is a float throughout. A whole number of them is not written + as a fraction: a set of cores has no fractional size at all, and "64.0 allowed cores" reads as a bug. + """ + # Not `float.is_integer()`: an `int` satisfies this annotation, and only Python 3.12 gives `int` that + # method. The remainder answers for both, and for a value no arithmetic here can produce anyway. + return str(int(cores)) if cores % 1 == 0 else str(cores) + + +@dataclass(frozen=True) +class _CpuRestriction: + """One CPU restriction as read, with the notice that explains it away where it covers the machine.""" + + cores: float + limit_directory: Path + """The level this restriction was read at.""" + + usage_directory: Path | None + """Where the CPU time this restriction applies to is counted. `None` when nothing counts it.""" + + code: NoticeCode + + +@dataclass(frozen=True) +class _CpuEvaluation: + """The CPU readings with the judgement applied.""" + + effective: float | None + limit_directory: Path | None + """The level `effective` was read at. `None` when nothing restricts the CPU.""" + + usage_directory: Path | None + """Where the consumed CPU time has to be read to match `effective`. + + `None` when no level counts the time this limit applies to, and therefore no rate can be measured. It is + never a stand-in for the group of this process: that group is named like any other. + """ + + raw_quota: float | None + raw_set_size: int | None + notices: tuple[Notice, ...] + + +def _evaluate_memory(machine_memory: int | None) -> _MemoryEvaluation: + """Judge the raw memory reading.""" + raw = _cgroup.read_memory() + rejection = _memory_rejection(raw, machine_memory) + + if rejection is not None: + return _MemoryEvaluation(effective=None, raw=raw, notices=(rejection,)) + + if raw.limit is None or raw.working_set is None: + # Nothing to report and nothing to explain: the mechanism is there and no level limits anything. The + # second test cannot be true here - `_memory_rejection` has already answered for it - and it stays + # because it is what narrows the type below. + return _MemoryEvaluation(effective=None, raw=raw, notices=()) + + return _MemoryEvaluation(effective=MemoryBudget(limit=raw.limit, working_set=raw.working_set), raw=raw, notices=()) + + +def _memory_rejection(raw: _cgroup.RawMemory, machine_memory: int | None) -> Notice | None: + """Say why the raw memory reading cannot be reported. `None` where it can. + + Each rule here drops a limit that would mislead a consumer, and names what was dropped. + """ + if raw.unreadable_directory is not None: + # Every level that did answer is looser than the one that did not, so there is nothing safe to report. + return Notice( + code=NoticeCode.MEMORY_LIMIT_UNREADABLE, + message=f'The memory limit of {raw.unreadable_directory} cannot be read, so a tighter limit than ' + 'any this process can see may apply and nothing is reported.', + ) + + if raw.limit is None: + # No limit and no mechanism read the same way from the outside, and only one of them is a fact about + # this machine. A consumer that expected a limit needs to know which it is looking at. + return ( + Notice( + code=NoticeCode.MEMORY_METRICS_UNAVAILABLE, + message='No mechanism here carries a memory limit, so nothing was read. This is what a ' + 'machine without cgroups looks like.', + ) + if _cgroup.locate_controllers().memory is None + else None + ) + + if machine_memory is None: + # Without the machine memory, a real limit and a v1 "unlimited" sentinel look the same. + return Notice( + code=NoticeCode.MACHINE_MEMORY_UNKNOWN, + message=f'The memory of the machine cannot be read, so the limit of {raw.limit} bytes cannot be ' + 'told apart from an "unlimited" sentinel and is not reported.', + ) + + if raw.limit >= machine_memory: + # This is how an unrestricted group spells "no limit". The exact sentinel differs between runtimes. + return Notice( + code=NoticeCode.MEMORY_LIMIT_COVERS_MACHINE, + message=f'The memory limit of {raw.limit} bytes is at least the memory of the machine ' + f'({machine_memory} bytes), so it does not restrict this process.', + ) + + if raw.working_set is None: + return Notice( + code=NoticeCode.MEMORY_USAGE_UNAVAILABLE, + message=f'Found a memory limit of {raw.limit} bytes but no usage metric to pair it with, so the ' + 'limit is not reported.', + ) + + return None + + +def _evaluate_cpu_limit(machine_cpus: int | None) -> _CpuEvaluation: + """Judge the raw CPU readings. The tighter of the quota and the set wins. + + A reading is trusted when the machine core count is unknown. CPU has no sentinel: an absent quota or set is + an absent file, so whatever was read is a number a human configured. + """ + raw = _cgroup.read_cpu() + + if raw.unreadable_directory is not None: + # As for memory: what that level enforces is unknown, and every level that answered is looser. Both + # readings are already empty here, and this is before them so that nothing has to rely on that. + return _CpuEvaluation( + effective=None, + limit_directory=None, + usage_directory=None, + raw_quota=None, + raw_set_size=None, + notices=( + Notice( + code=NoticeCode.CPU_LIMIT_UNREADABLE, + message=f'The CPU limit of {raw.unreadable_directory} cannot be read, so a tighter limit ' + 'than any this process can see may apply and nothing is reported.', + ), + ), + ) + + quota, cpu_set = raw.quota, raw.cpu_set + + restrictions = [] + + if quota is not None: + restrictions.append( + _CpuRestriction( + cores=quota.cores, + limit_directory=quota.limit_directory, + usage_directory=quota.usage_directory, + code=NoticeCode.CPU_QUOTA_COVERS_MACHINE, + ) + ) + + if cpu_set is not None: + restrictions.append( + _CpuRestriction( + cores=float(cpu_set.cores), + limit_directory=cpu_set.limit_directory, + usage_directory=cpu_set.usage_directory, + code=NoticeCode.CPU_SET_COVERS_MACHINE, + ) + ) + + notices = [] + candidates = [] + + if not restrictions and _no_cpu_mechanism(): + # As for memory: "nothing limits the CPU here" and "nothing here can say" are different answers. + notices.append( + Notice( + code=NoticeCode.CPU_METRICS_UNAVAILABLE, + message='No mechanism here carries a CPU limit, so nothing was read. This is what a machine ' + 'without cgroups looks like.', + ) + ) + + for restriction in restrictions: + if machine_cpus is not None and restriction.cores >= machine_cpus: + message = _COVERS_MACHINE_MESSAGES[restriction.code] + notices.append( + Notice( + code=restriction.code, + message=message.format(cores=_spell_cores(restriction.cores), machine_cpus=machine_cpus), + ) + ) + else: + candidates.append(restriction) + + tightest = min(candidates, key=lambda restriction: restriction.cores) if candidates else None + + # A limit whose level counts no CPU time still limits, and is still reported. Only the rate is impossible, + # and saying so is the whole point of the notice: the alternative is a ratio taken from another scope. + if tightest is not None and tightest.usage_directory is None: + notices.append( + Notice( + code=NoticeCode.CPU_USAGE_SCOPE_MISMATCH, + message=f'The CPU limit of {_spell_cores(tightest.cores)} cores applies to a level whose ' + 'consumed CPU time cannot be read, so no rate can be measured against it.', + ) + ) + + return _CpuEvaluation( + effective=tightest.cores if tightest is not None else None, + limit_directory=tightest.limit_directory if tightest is not None else None, + usage_directory=tightest.usage_directory if tightest is not None else None, + raw_quota=quota.cores if quota is not None else None, + raw_set_size=cpu_set.cores if cpu_set is not None else None, + notices=tuple(notices), + ) + + +def _no_cpu_mechanism() -> bool: + """Whether nothing on this machine carries a CPU limit of either kind.""" + controllers = _cgroup.locate_controllers() + + return controllers.cpu_quota is None and controllers.cpu_set is None + + +def _source(controller: _cgroup.Controller | None) -> Source | None: + """Spell one located controller as a source.""" + if controller is None: + return None + + return Source( + interface=Interface.CGROUP_V2 if controller.is_v2 else Interface.CGROUP_V1, + levels=tuple(str(directory) for directory in controller.dirs), + ) + + +def get_machine_memory_bytes() -> int | None: + """Read the total memory of the machine, in bytes. + + This is the number the filters compare a limit against, and it is here so that a consumer that got `None` + from `get_memory_budget()` has the other half of the answer without reaching for a second library. + + Containers normally see `/proc/meminfo` unvirtualized, so this is the memory of the node rather than of the + container. A runtime that virtualizes it (lxcfs) makes the limit and the "machine" coincide, and the limit + is then reported as no restriction. + + Returns: + The total memory, or `None` when it cannot be read - which is what happens off Linux. + """ + try: + for line in _PROC_MEMINFO.read_text().splitlines(): + key, _separator, value = line.partition(':') + if key == 'MemTotal': + # The value carries a unit, e.g. `8054932 kB`. + return int(value.split()[0]) * 1024 + except (OSError, ValueError, IndexError): + return None + + return None + + +def get_machine_cpu_count() -> int | None: + """Read the number of online CPU cores of the machine. + + This is the number the filters compare a quota or a set against, and the reason it is public: the usual + answers describe the process instead. `os.cpu_count()` honors the `PYTHON_CPU_COUNT` override, and under + musl both it and `psutil.cpu_count()` report the affinity of the process. Either would make a real + restriction look like the whole machine. The kernel is asked directly here. + + Returns: + The online cores, or `None` when even the fallbacks say nothing. + """ + try: + online = _SYS_CPU_ONLINE.read_text().strip() + except (OSError, ValueError): + return _machine_cpu_count_fallback() + + return count_cpu_list(online) or _machine_cpu_count_fallback() + + +def _machine_cpu_count_fallback() -> int | None: + """Count the cores where the kernel does not list them.""" + try: + count = os.sysconf('SC_NPROCESSORS_ONLN') + except (AttributeError, OSError, ValueError): + return os.cpu_count() + + return count if count > 0 else os.cpu_count() diff --git a/src/cgroups_sensor/py.typed b/src/cgroups_sensor/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py new file mode 100644 index 0000000..905c71a --- /dev/null +++ b/tests/e2e/conftest.py @@ -0,0 +1,201 @@ +from __future__ import annotations + +import os +import subprocess +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest + +from .harness import ( + CAPABILITIES, + IMAGE, + MEMORY_LIMIT, + REQUIRED_CAPABILITIES, + TIMEOUT_SECONDS, + command_works, + have, + is_unified, + machine_cpu_count, + unavailable, +) + +if TYPE_CHECKING: + from collections.abc import Callable, Iterator + +TWO_CORES = 2 + + +def pytest_sessionstart() -> None: + """Reject a capability name this suite does not know, which would otherwise disarm the gate silently. + + The raise is what ends the run, with pytest's own usage-error code. Nothing reads `session.exitstatus` + this early, unlike in `pytest_sessionfinish` below, where setting it is the only way to fail a run. + """ + unknown = REQUIRED_CAPABILITIES - CAPABILITIES + if unknown: + raise pytest.UsageError(f'E2E_REQUIRE names {", ".join(sorted(unknown))}, not one of {sorted(CAPABILITIES)}') + + +@pytest.fixture(scope='session', autouse=True) +def _linux_only() -> None: + """Skip the whole suite where cgroups cannot exist.""" + if not Path('/proc/self/cgroup').exists(): + pytest.skip('cgroups exist on Linux only') + + +@pytest.fixture(scope='session') +def _docker() -> None: + """Skip unless a working Docker is available, and pull the probe image once.""" + if not have('docker') or not command_works(['docker', 'info']): + unavailable('docker', 'docker is not available') + + subprocess.run( + ['docker', 'pull', '--quiet', IMAGE], + capture_output=True, + timeout=TIMEOUT_SECONDS, + check=False, + ) + + +@pytest.fixture +def _sudo() -> None: + """Skip unless sudo runs without asking for a password.""" + if not have('sudo') or not command_works(['sudo', '-n', 'true']): + unavailable('sudo', 'passwordless sudo is not available') + + +@pytest.fixture +def _systemd_user() -> None: + """Skip unless this user has a systemd manager that can hold a scope.""" + if not have('systemd-run') or not command_works(['systemd-run', '--user', '--scope', '-q', 'true']): + unavailable('systemd', 'a systemd user manager is not available') + + +@pytest.fixture +def _systemd_system() -> None: + """Skip unless a system scope can be started, which needs both sudo and a running systemd. + + The command is tried rather than the binary looked for: a machine can carry `systemd-run` and boot with + another init, and then every scope fails instead of skipping. + """ + if not have('systemd-run') or not command_works(['sudo', '-n', 'systemd-run', '--scope', '-q', 'true']): + unavailable('systemd', 'a system systemd manager is not available') + + +@pytest.fixture +def _unified() -> None: + """Skip unless the controllers live on the cgroup v2 unified hierarchy. + + Under cgroup v1 systemd delegates only `name=systemd` to user sessions, so a user scope gets no cgroup in + the resource controllers and its properties do nothing. + """ + if not is_unified(): + pytest.skip('the controllers are not on the unified hierarchy') + + +@pytest.fixture +def _systemd_cgroup_driver() -> None: + """Skip unless docker puts its containers under systemd slices. + + `--cgroup-parent` takes a slice name only under that driver. + """ + result = subprocess.run( + ['docker', 'info', '--format', '{{.CgroupDriver}}'], + capture_output=True, + text=True, + timeout=60, + check=False, + ) + if result.stdout.strip() != 'systemd': + pytest.skip(f'docker uses the {result.stdout.strip() or "unknown"!r} cgroup driver, this needs systemd') + + +@pytest.fixture +def parent_slice() -> Iterator[str]: + """Create a slice carrying a memory limit, and remove it afterwards. + + A slice exists only while a unit lives in it, so a sleeping holder keeps it alive. + """ + name = f'cgroups-sensor-e2e-{os.getpid()}.slice' + holder = f'cgroups-sensor-e2e-holder-{os.getpid()}' + + subprocess.run( + ['sudo', 'systemd-run', '-q', '--unit', holder, '--slice', name, 'sleep', 'infinity'], + capture_output=True, + timeout=60, + check=True, + ) + subprocess.run( + ['sudo', 'systemctl', 'set-property', '--runtime', name, f'MemoryMax={MEMORY_LIMIT}'], + capture_output=True, + timeout=60, + check=True, + ) + + yield name + + subprocess.run( + ['sudo', 'systemctl', 'stop', holder, name], + capture_output=True, + timeout=60, + check=False, + ) + + +@pytest.fixture +def _two_cores() -> None: + """Skip where pinning one core would not restrict anything.""" + if machine_cpu_count() < TWO_CORES: + pytest.skip('a single-core machine cannot be restricted to one core') + + +@pytest.fixture +def systemd_scope() -> Iterator[Callable[..., list[str]]]: + """Build `systemd-run` wrappers, and stop whatever scopes the test started. + + A system scope needs sudo. Ask for it with `system=True`, together with the `_sudo` fixture. + """ + units: list[tuple[str, bool]] = [] + + def build(*properties: str, system: bool = False) -> list[str]: + unit = f'cgroups-sensor-e2e-{os.getpid()}-{len(units)}' + units.append((unit, system)) + + wrapper = ['sudo'] if system else [] + wrapper += ['systemd-run', '--scope', '-q', '--unit', unit] + wrapper += [] if system else ['--user'] + for prop in properties: + wrapper += ['-p', prop] + + return wrapper + + yield build + + for unit, system in units: + stop = ['systemctl', '--user'] if not system else ['sudo', 'systemctl'] + subprocess.run([*stop, 'stop', f'{unit}.scope'], capture_output=True, check=False, timeout=60) + + +_EXECUTED: list[str] = [] + + +def pytest_runtest_logreport(report: pytest.TestReport) -> None: + """Record the tests that ran, for the guard below.""" + if report.when == 'call' and report.passed: + _EXECUTED.append(report.nodeid) + + +def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: + """Fail a run in which nothing ran at all. + + Every test here skips when its environment is missing, so an all-skipped suite would exit 0 and prove + nothing. That is the one outcome this suite must never report as success. Collecting is not running, so a + plain `--collect-only` is left alone. + """ + if session.config.getoption('--collect-only'): + return + + if exitstatus == 0 and not _EXECUTED: + session.exitstatus = 1 + print('\nthe e2e suite proved nothing: every test skipped') diff --git a/tests/e2e/harness.py b/tests/e2e/harness.py new file mode 100644 index 0000000..f503e03 --- /dev/null +++ b/tests/e2e/harness.py @@ -0,0 +1,399 @@ +from __future__ import annotations + +import json +import os +import platform +import shutil +import subprocess +import sys +import tarfile +import tempfile +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path +from typing import Any + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +SRC_DIR = REPO_ROOT / 'src' +PROBE = Path(__file__).parent / 'probe.py' + +IMAGE = 'python:3.13-alpine' +"""The image the container tests use. uv brings the interpreter, so this one only has to be small.""" + +DISTRO_IMAGES = ( + 'python:3.13-alpine', + 'python:3.13-slim', + 'fedora:43', + 'rockylinux:9', +) +"""Images the readings are compared across: musl, Debian glibc, and two distributions of another family.""" + +PYTHON_VERSION = '3.13' +"""The interpreter every probe runs on. uv installs it, so the image does not decide the version.""" + +PYTHON_VERSIONS = ('3.10', '3.11', '3.12', '3.13', '3.14') +"""Every interpreter the package supports, as `requires-python` spells it.""" + +UV_DOWNLOADS = Path(tempfile.gettempdir()) / 'cgroups-sensor-e2e-uv' +"""Where uv and its interpreters are kept, so only the first run pays for the download.""" + +MIB = 1024 * 1024 +MEMORY_LIMIT = 512 * MIB +TIGHTER_MEMORY_LIMIT = 256 * MIB +QUOTA_CORES = 0.5 +"""Half a core. Below any machine, so it is always a real restriction.""" + +TIMEOUT_SECONDS = 300 + +CANNOT_SET_UP = 77 +"""What a wrapper exits with when the environment refuses to produce the shape a test needs. + +GNU's convention for "skipped". A test that arranges a cgroup layout before probing has to say which of the +two happened: the layout could not be built here, or it was built and the reading is wrong. Without this the +setup fails silently and the assertion afterwards blames the sensor for it. +""" + +CAPABILITIES = frozenset({'docker', 'sudo', 'systemd', 'kubernetes'}) +"""What a lane can be told to prove. `E2E_REQUIRE` is checked against this, so a typo cannot quietly disarm it.""" + +REQUIRED_CAPABILITIES = frozenset(name for name in os.environ.get('E2E_REQUIRE', '').split(',') if name) +"""What this run must actually exercise, e.g. `E2E_REQUIRE=docker,systemd`. + +A test skips where its environment is missing, which is what makes one suite run everywhere. In CI that turns +a broken runner into a green job, so each lane names what it is there for and a missing capability fails. +""" + + +def unavailable(capability: str, reason: str) -> None: + """Skip because the environment cannot do this, or fail when this run was supposed to prove it.""" + if capability in REQUIRED_CAPABILITIES: + pytest.fail(f'{reason}, and E2E_REQUIRE names {capability}') + + pytest.skip(reason) + + +@dataclass(frozen=True) +class Reading: + """What the probe saw inside the environment under test.""" + + version: str + memory_limit: int | None + working_set: int | None + cpu_limit: float | None + cpu_usage: float | None + cpu_used_ratio: float | None + raw_memory_limit: int | None + raw_memory_working_set: int | None + raw_cpu_quota: float | None + raw_cpu_set_size: int | None + memory_limit_level: str | None + cpu_limit_level: str | None + cpu_rate_level: str | None + machine_memory_bytes: int | None + machine_cpu_count: int | None + allocated: int + notices: tuple[str, ...] + sources: dict[str, Any] + + @property + def interfaces(self) -> set[str]: + """The mechanisms the readings came from, e.g. `{'cgroup-v1'}`.""" + return {source['interface'] for source in self.sources.values() if source is not None} + + @property + def limit_interfaces(self) -> set[str]: + """The mechanisms the limits came from. + + The consumed CPU time is left out on purpose. It comes from the hierarchy carrying the limits + wherever that hierarchy counts anything, but where nothing does, the base `cpu.stat` of a + controller-less cgroup2 is all there is - and that is not a reason to fail a lane. + """ + return { + source['interface'] for name, source in self.sources.items() if name != 'cpu_usage' and source is not None + } + + def __repr__(self) -> str: + """Spell the numbers a failing assertion needs, with the sources summarized rather than dumped.""" + fields = ', '.join( + f'{name}={getattr(self, name)!r}' + for name in ( + 'memory_limit', + 'working_set', + 'cpu_limit', + 'cpu_usage', + 'cpu_used_ratio', + 'raw_memory_limit', + 'raw_memory_working_set', + 'raw_cpu_quota', + 'raw_cpu_set_size', + 'memory_limit_level', + 'cpu_limit_level', + 'cpu_rate_level', + 'machine_memory_bytes', + 'machine_cpu_count', + 'notices', + ) + ) + levels = {name: (source or {}).get('levels') for name, source in self.sources.items()} + + return f'Reading({fields}, interfaces={sorted(self.interfaces)}, levels={levels})' + + +def parse(output: str) -> Reading: + """Read the probe's JSON out of its output.""" + lines = [line for line in output.splitlines() if line.startswith('{')] + if not lines: + pytest.fail(f'the probe printed no JSON:\n{output}') + + payload = json.loads(lines[-1]) + payload['notices'] = tuple(payload['notices']) + + return Reading(**payload) + + +def run(command: list[str], *, env: dict[str, str] | None = None) -> Reading: + """Run one probe command and parse what it printed.""" + result = subprocess.run( + command, + capture_output=True, + text=True, + timeout=TIMEOUT_SECONDS, + env=env, + check=False, + ) + + if result.returncode == CANNOT_SET_UP: + pytest.skip(f'the environment cannot set this up: {result.stderr.strip()[-300:]}') + + if result.returncode != 0: + pytest.fail( + f'{" ".join(command)} exited {result.returncode}\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}' + ) + + return parse(result.stdout) + + +def probe_here(wrapper: list[str] | None = None, python_version: str | None = None) -> Reading: + """Run the probe on this machine, optionally under a wrapper command such as `systemd-run`. + + The interpreter is the one running the suite, unless a version is named. uv then installs that one. + """ + env = {**os.environ, 'PYTHONPATH': str(SRC_DIR)} + + if python_version is None: + interpreter = [sys.executable, str(PROBE)] + else: + interpreter = ['uv', 'run', '--no-project', '--python', python_version, 'python', str(PROBE)] + + if wrapper is None: + return run(interpreter, env=env) + + # A wrapper can strip the environment, so the variables travel as an explicit `env` call. + passthrough = ['env', f'PYTHONPATH={SRC_DIR}', f'HOME={os.environ.get("HOME", "/root")}'] + + return run([*wrapper, *passthrough, *interpreter], env=env) + + +def probe_command(python_version: str = PYTHON_VERSION) -> str: + """Spell how the probe is started inside a container. + + uv brings the interpreter, so the image decides neither the version nor whether one exists at all. + """ + return f'/uv/uv run --no-project --python {python_version} python /sensor/probe.py' + + +@lru_cache(maxsize=1) +def portable_uv() -> Path: + """Download the uv build that runs in any image, and hand back its path. + + The musl build is statically linked, so one binary covers musl and glibc images alike. The version follows + the uv of this machine, so the tests use the uv the project is developed with. + """ + target = UV_DOWNLOADS / 'uv' + if target.exists(): + return target + + # `uv --version`, not `uv version`: the latter reports the version of the project it is run in. + reported = subprocess.run( + ['uv', '--version'], + capture_output=True, + text=True, + timeout=60, + check=False, + ).stdout.split() + if len(reported) < 2: + unavailable('docker', 'the uv version of this machine cannot be read') + + version = reported[1] + + machine = 'aarch64' if platform.machine() in {'aarch64', 'arm64'} else 'x86_64' + url = f'https://github.com/astral-sh/uv/releases/download/{version}/uv-{machine}-unknown-linux-musl.tar.gz' + + UV_DOWNLOADS.mkdir(parents=True, exist_ok=True) + archive = UV_DOWNLOADS / 'uv.tar.gz' + if not command_works(['curl', '-fsSL', '-o', str(archive), url]): + unavailable('docker', f'{url} cannot be downloaded') + + with tarfile.open(archive) as tar: + member = next(entry for entry in tar.getmembers() if entry.name.endswith('/uv')) + member.name = 'uv' + tar.extract(member, UV_DOWNLOADS, filter='data') + + target.chmod(0o755) + + return target + + +def probe_in_container( + *docker_args: str, + image: str = IMAGE, + python_version: str = PYTHON_VERSION, + command: list[str] | None = None, +) -> Reading: + """Run the probe in a fresh container. The arguments go to `docker run`.""" + cache = UV_DOWNLOADS / 'cache' + interpreters = UV_DOWNLOADS / 'python' + cache.mkdir(parents=True, exist_ok=True) + interpreters.mkdir(parents=True, exist_ok=True) + + return run( + [ + 'docker', + 'run', + '--rm', + '--volume', + f'{SRC_DIR}:/sensor/src:ro', + '--volume', + f'{PROBE}:/sensor/probe.py:ro', + '--volume', + f'{portable_uv()}:/uv/uv:ro', + '--volume', + f'{cache}:/uv/cache', + '--volume', + f'{interpreters}:/uv/python', + '--env', + 'PYTHONPATH=/sensor/src', + '--env', + 'UV_CACHE_DIR=/uv/cache', + '--env', + 'UV_PYTHON_INSTALL_DIR=/uv/python', + *docker_args, + image, + *(command or ['sh', '-c', probe_command(python_version)]), + ] + ) + + +def pull_image(image: str) -> bool: + """Pull one image, and report whether it arrived. + + Pulling gets its own generous timeout: inside a virtual machine the network is slow enough that the + default one would report a working registry as a missing image. + """ + return command_works(['docker', 'pull', '--quiet', image], timeout=TIMEOUT_SECONDS) + + +def have(tool: str) -> bool: + """Whether a command exists on this machine.""" + return shutil.which(tool) is not None + + +def command_works(command: list[str], *, timeout: int = 60) -> bool: + """Whether a command runs and succeeds.""" + try: + return ( + subprocess.run( + command, + capture_output=True, + timeout=timeout, + check=False, + ).returncode + == 0 + ) + except (OSError, subprocess.SubprocessError): + return False + + +def machine_cpu_count() -> int: + """The cores of this machine, counted independently of the package. + + The package answers this with `get_machine_cpu_count()`. This one deliberately does not use it: a test + that asks the subject for the expected value proves nothing. + """ + return os.sysconf('SC_NPROCESSORS_ONLN') + + +def is_unified() -> bool: + """Whether the controllers live on the cgroup v2 unified hierarchy.""" + controllers = Path('/sys/fs/cgroup/cgroup.controllers') + + # Every file in cgroupfs reports zero bytes, so the content has to be read rather than sized. + return controllers.exists() and bool(controllers.read_text().strip()) + + +def machine_memory_bytes() -> int: + """The memory of this machine, in bytes. Read here rather than asked of the package, as above.""" + for line in Path('/proc/meminfo').read_text().splitlines(): + if line.startswith('MemTotal:'): + return int(line.split()[1]) * 1024 + + pytest.skip('/proc/meminfo carries no MemTotal') + + +def notices_about(reading: Reading, metric: str) -> list[str]: + """The notices that explain a dropped reading of one metric. + + A notice of the other metric is no explanation, so the two are told apart here rather than at each call + site. The machine facts belong to the metric they were compared against. + """ + prefixes = {'memory': ('memory', 'machine-memory'), 'cpu': ('cpu',)}[metric] + + return [code for code in reading.notices if code.startswith(prefixes)] + + +def check_invariants(reading: Reading) -> None: + """Check what must hold of every reading, whatever the environment. + + Called by every test on top of its own assertions. A reported limit has to be real, and a missing one has + to be explained. + """ + # The probe reads `__version__`, which the package resolves lazily on first access. Nothing else in the + # suite touches that path, and a source tree nothing installed reports `unknown` rather than nothing. + assert reading.version + + if reading.memory_limit is not None: + assert reading.working_set is not None + # The probe charged this much anonymous memory to its own cgroup before reading, so a working set + # below it is not the memory of this process - a raw counter, a stale file, or another cgroup. + assert reading.allocated <= reading.working_set <= reading.memory_limit + assert reading.machine_memory_bytes is not None + assert reading.memory_limit < reading.machine_memory_bytes + else: + # Nothing was reported, so either no limit was found or a notice says why it was dropped. + assert reading.raw_memory_limit is None or notices_about(reading, 'memory') + + if reading.raw_memory_limit is not None: + # Whatever became of the limit, the level it was read at is one of the levels that were searched. + assert reading.memory_limit_level in reading.sources['memory']['levels'] + if reading.raw_memory_working_set is not None: + assert 0 <= reading.raw_memory_working_set <= reading.raw_memory_limit + + if reading.cpu_limit is not None: + assert reading.cpu_limit > 0 + if reading.machine_cpu_count is not None: + assert reading.cpu_limit < reading.machine_cpu_count + # A limit whose level counts no CPU time is still reported, and then says so instead of a rate. + if reading.cpu_used_ratio is None: + assert 'cpu-usage-scope-mismatch' in reading.notices + else: + # The probe keeps a core busy while it measures, so a rate of nothing means the time was counted + # somewhere this process is not. The level it was counted at is named next to the failure. + assert 0.0 < reading.cpu_used_ratio <= 1.0 + assert reading.cpu_rate_level is not None + assert reading.cpu_limit_level is not None + else: + assert reading.cpu_used_ratio is None + raw_cpu = (reading.raw_cpu_quota, reading.raw_cpu_set_size) + assert raw_cpu == (None, None) or notices_about(reading, 'cpu') diff --git a/tests/e2e/probe.py b/tests/e2e/probe.py new file mode 100644 index 0000000..4557abd --- /dev/null +++ b/tests/e2e/probe.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import json +import threading +import time +from typing import Any + +import cgroups_sensor + +MEASUREMENT_SECONDS = 1.0 +"""How long the CPU rate is measured for, and therefore how long the burner below runs.""" + +ALLOCATION_BYTES = 32 * 1024 * 1024 +"""How much memory the probe charges to its own cgroup before reading, so that the working set has a floor. + +Anonymous memory, touched: nothing about it is reclaimable file cache, so a working set below this would mean +the reading is not the memory of this process. +""" + + +def burn(seconds: float) -> None: + """Keep one core busy for a while. + + The rate is the one reading that says nothing when it is measured in the wrong cgroup: an idle process + reads as zero everywhere, correct or not. So the probe makes itself busy while it measures. + """ + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + pass + + +def source(value: cgroups_sensor.Source | None) -> dict[str, Any] | None: + """Spell one source as JSON.""" + return None if value is None else {'interface': value.interface, 'levels': list(value.levels)} + + +def main() -> None: + # Kept alive until everything has been read. `bytearray` zero-fills, so every page is really charged. + ballast = bytearray(ALLOCATION_BYTES) + + reading = cgroups_sensor.snapshot() + + burner = threading.Thread(target=burn, args=(MEASUREMENT_SECONDS,), daemon=True) + burner.start() + cpu_used_ratio = cgroups_sensor.get_cpu_used_ratio(MEASUREMENT_SECONDS) + burner.join() + + description = cgroups_sensor.describe() + + print( + json.dumps( + { + 'version': cgroups_sensor.__version__, + 'memory_limit': reading.memory_budget.limit if reading.memory_budget is not None else None, + 'working_set': reading.memory_budget.working_set if reading.memory_budget is not None else None, + 'cpu_limit': reading.cpu_limit, + 'cpu_usage': reading.cpu_usage, + 'cpu_used_ratio': cpu_used_ratio, + 'raw_memory_limit': description.raw_memory_limit, + 'raw_memory_working_set': description.raw_memory_working_set, + 'raw_cpu_quota': description.raw_cpu_quota, + 'raw_cpu_set_size': description.raw_cpu_set_size, + 'memory_limit_level': description.memory_limit_level, + 'cpu_limit_level': description.cpu_limit_level, + 'cpu_rate_level': description.cpu_rate_level, + 'machine_memory_bytes': description.machine_memory_bytes, + 'machine_cpu_count': description.machine_cpu_count, + 'allocated': len(ballast), + 'notices': [notice.code for notice in description.notices], + 'sources': { + 'memory': source(description.memory_source), + 'cpu_quota': source(description.cpu_quota_source), + 'cpu_set': source(description.cpu_set_source), + 'cpu_usage': source(description.cpu_usage_source), + }, + } + ) + ) + + +if __name__ == '__main__': + main() diff --git a/tests/e2e/scripts/guest.sh b/tests/e2e/scripts/guest.sh new file mode 100755 index 0000000..78e8aa2 --- /dev/null +++ b/tests/e2e/scripts/guest.sh @@ -0,0 +1,253 @@ +#!/usr/bin/env bash +set -eu + +REPO_ROOT=$(cd "$(dirname "$0")/../../.." && pwd) +PROFILE=${GUEST_PROFILE:-ubuntu-v1-hybrid} +WORK=${GUEST_WORK:-/tmp/cgroups-sensor-guest/$PROFILE} +# The images are the same bytes on every run, so they live apart from the keys and overlays a run creates - +# and outside /tmp, which a WSL or machine restart wipes. Re-downloading a gigabyte is the slowest thing here. +DOWNLOADS=${GUEST_DOWNLOADS:-${XDG_CACHE_HOME:-$HOME/.cache}/cgroups-sensor-guest} +SSH_PORT=${GUEST_SSH_PORT:-2222} +GUEST_USER=bench + +UBUNTU=https://cloud-images.ubuntu.com/releases/22.04/release +FEDORA=https://download.fedoraproject.org/pub/fedora/linux/releases/43/Cloud/x86_64/images + +# A profile is one machine to run the suite on. The suite skips whatever a machine cannot do, so each profile +# covers what it can: the Ubuntu ones exist for cgroup v1, the Fedora one for a second distribution. +case $PROFILE in + ubuntu-v1-hybrid | ubuntu-v1-legacy) + # cgroup v1 cannot be produced on a modern host, and systemd honours the flags below only up to v255. + # Ubuntu 22.04 ships systemd 249. Its kernel and initrd are published next to the image, which is what + # lets the command line be set from here instead of editing the image. + # + # hybrid mounts the v1 controllers next to a controller-less cgroup2, so the sensor has to notice that the + # unified hierarchy carries nothing and fall back per controller. legacy is plain v1. + IMAGE_URL=${GUEST_IMAGE_URL:-$UBUNTU/ubuntu-22.04-server-cloudimg-amd64.img} + KERNEL_URL=${GUEST_KERNEL_URL:-$UBUNTU/unpacked/ubuntu-22.04-server-cloudimg-amd64-vmlinuz-generic} + INITRD_URL=${GUEST_INITRD_URL:-$UBUNTU/unpacked/ubuntu-22.04-server-cloudimg-amd64-initrd-generic} + PACKAGES=docker.io + PREPARE='' + # The guest exists for cgroup v1. One that came up on v2 would run a smaller suite and still report + # success, so the suite is told what it has to find. + REQUIRE='docker,sudo' + INTERFACE=cgroup-v1 + CGROUP_ARGS='systemd.unified_cgroup_hierarchy=0' + [ "$PROFILE" = ubuntu-v1-legacy ] && CGROUP_ARGS="$CGROUP_ARGS systemd.legacy_systemd_cgroup_controller=1" + CMDLINE="root=/dev/vda1 console=ttyS0 $CGROUP_ARGS" + ;; + fedora-v2) + # A second distribution, with a much newer systemd, on the cgroup v2 layout it defaults to. Nothing has to + # be passed to the kernel, so the image boots through its own bootloader. + # Fedora publishes no "latest" name, so the build is pinned. Bump it when the release is retired. + IMAGE_URL=${GUEST_IMAGE_URL:-$FEDORA/Fedora-Cloud-Base-Generic-43-1.6.x86_64.qcow2} + KERNEL_URL='' + INITRD_URL='' + PACKAGES=moby-engine + REQUIRE='docker,sudo,systemd' + INTERFACE=cgroup-v2 + # Only the test plumbing needs this: the suite bind-mounts the repository into containers, which SELinux + # denies without relabelling. The sensor itself reads `/proc` and `/sys` and is not affected. + PREPARE='setenforce 0' + CMDLINE='' + ;; + *) + echo "guest: GUEST_PROFILE must be ubuntu-v1-hybrid, ubuntu-v1-legacy or fedora-v2" >&2 + exit 1 + ;; +esac + +# scp spells the port -P and reads -p as "preserve timestamps", so the two need separate option strings. +SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR" +KEY=$WORK/id_ed25519 + +# shellcheck disable=SC2086 +in_guest() { ssh -i "$KEY" $SSH_OPTS -p "$SSH_PORT" "$GUEST_USER@127.0.0.1" "$@"; } +# shellcheck disable=SC2086 +copy_in() { scp -q -i "$KEY" $SSH_OPTS -P "$SSH_PORT" "$1" "$GUEST_USER@127.0.0.1:$2"; } + +require_tools() { + local missing=0 + + command -v qemu-system-x86_64 >/dev/null || { echo "guest: qemu-system-x86 is required" >&2; missing=1; } + command -v cloud-localds >/dev/null || { echo "guest: cloud-image-utils is required" >&2; missing=1; } + command -v ssh >/dev/null || { echo "guest: openssh-client is required" >&2; missing=1; } + # The guest gets this uv: the images ship a Python this package no longer supports. + command -v uv >/dev/null || { echo "guest: uv is required, see https://docs.astral.sh/uv/" >&2; missing=1; } + + [ "$missing" -eq 0 ] || exit 1 + + # Emulation needs no privileges but is several times slower, so it has to be asked for by name. + if [ "${GUEST_ACCEL:-kvm}" = kvm ] && ! { [ -r /dev/kvm ] && [ -w /dev/kvm ]; }; then + echo "guest: /dev/kvm is not usable. Run with GUEST_ACCEL=tcg to emulate instead, or grant access:" >&2 + echo " on a runner: echo 'KERNEL==\"kvm\", GROUP=\"kvm\", MODE=\"0666\"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules" >&2 + echo " locally: sudo usermod -aG kvm \$USER, then log in again" >&2 + exit 1 + fi +} + +# Download once per machine, then hand back the cached path. +fetch() { + local url=$1 dest="$DOWNLOADS/$(basename "$1")" + + if [ ! -s "$dest" ]; then + echo "guest: downloading $(basename "$url")" >&2 + # These are hundreds of megabytes over a network this script does not control, and a run that fails here + # says nothing about the sensor. + curl -fsSL --retry 3 --retry-delay 5 --retry-connrefused -o "$dest" "$url" + fi + + echo "$dest" +} + +# Write the cloud-init seed that creates the user and installs a container engine. +write_seed() { + [ -s "$KEY" ] || ssh-keygen -q -t ed25519 -N '' -f "$KEY" + + cat > "$WORK/user-data" </dev/null && break + echo -n . + sleep 2 + done + echo + + in_guest true 2>/dev/null || { + echo 'guest: the guest never came up, the last of its console output:' >&2 + tail -30 "$WORK/console.log" >&2 + exit 1 + } + + # One mount per v1 controller, so counting both kinds says which layout the guest came up in: only cgroup + # mounts is legacy, both kinds is hybrid, only cgroup2 is the unified hierarchy. + echo -n 'guest: up, ' + in_guest 'printf "%s, cgroup mounts: %s v1, %s v2\n" \ + "$(. /etc/os-release && echo "$PRETTY_NAME")" \ + "$(grep -c " - cgroup " /proc/self/mountinfo || true)" "$(grep -c " - cgroup2 " /proc/self/mountinfo || true)"' + + # ssh answers as soon as sshd is up, while cloud-init is still installing packages behind it. + echo 'guest: waiting for cloud-init to finish' + in_guest 'cloud-init status --wait >/dev/null 2>&1 || true' +} + +install_suite() { + echo 'guest: copying the repository in' + in_guest 'rm -rf sensor && mkdir -p sensor' + tar -C "$REPO_ROOT" --exclude=.git --exclude=.venv --exclude=analysis --exclude=__pycache__ \ + -cf - src tests pyproject.toml uv.lock README.md LICENSE | in_guest 'tar -C sensor -xf -' + + copy_in "$(command -v uv)" uv + in_guest 'sudo install -m 0755 uv /usr/local/bin/uv' + in_guest "getent group docker >/dev/null && sudo usermod -aG docker $GUEST_USER || true" +} + +# Run the suite inside and hand back its exit code. +run_suite() { + # The arguments travel through ssh as one string, so each has to carry its own quoting. Without it an + # argument with spaces, such as `-k 'a or b'`, arrives as several. + local args='' + [ "$#" -eq 0 ] || args=" ${*@Q}" + + # The floors travel with the command. Without them the suite would skip whatever the guest cannot do and + # report success, which is exactly what these profiles exist to catch. + local floors="E2E_REQUIRE='$REQUIRE' E2E_INTERFACE='$INTERFACE'" + local suite="cd sensor && uv run poe install-dev && $floors uv run poe e2e-tests$args" + local status=0 + + echo 'guest: running the e2e suite inside' + # The freshly granted docker group only applies to new logins, so borrow it for this command. + if in_guest 'getent group docker >/dev/null'; then + in_guest "sg docker -c \"$suite\"" || status=$? + else + in_guest "$suite" || status=$? + fi + + return "$status" +} + +shutdown_guest() { + [ -s "$WORK/qemu.pid" ] && kill "$(cat "$WORK/qemu.pid")" 2>/dev/null + rm -f "$WORK/qemu.pid" +} + +require_tools +mkdir -p "$WORK" "$DOWNLOADS" + +image=$(fetch "$IMAGE_URL") +kernel='' +initrd='' +if [ -n "$KERNEL_URL" ]; then + kernel=$(fetch "$KERNEL_URL") + initrd=$(fetch "$INITRD_URL") +fi + +write_seed +boot_guest "$image" "$kernel" "$initrd" +trap shutdown_guest EXIT + +wait_for_guest +install_suite + +status=0 +run_suite "$@" || status=$? +in_guest 'sudo poweroff' 2>/dev/null || true + +exit "$status" diff --git a/tests/e2e/test_docker.py b/tests/e2e/test_docker.py new file mode 100644 index 0000000..18bd6b8 --- /dev/null +++ b/tests/e2e/test_docker.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +import pytest + +from .harness import ( + DISTRO_IMAGES, + MEMORY_LIMIT, + PYTHON_VERSIONS, + QUOTA_CORES, + TIGHTER_MEMORY_LIMIT, + check_invariants, + probe_command, + probe_in_container, + pull_image, +) + +pytestmark = pytest.mark.usefixtures('_docker') + + +def test_no_limits() -> None: + """Reports no restriction for a container that sets none. + + This is where the spellings differ: cgroup v2 writes `max`, cgroup v1 a sentinel near 2**63. + """ + reading = probe_in_container() + + check_invariants(reading) + assert reading.memory_limit is None + assert reading.cpu_limit is None + if reading.raw_memory_limit is not None: + assert 'memory-limit-covers-machine' in reading.notices + + +def test_memory_only() -> None: + """Reads a memory limit. Nothing restricts the CPU, so no CPU limit is reported.""" + reading = probe_in_container('--memory', str(MEMORY_LIMIT)) + + check_invariants(reading) + assert reading.memory_limit == MEMORY_LIMIT + # The floor `check_invariants` applies is the memory the probe charged to this cgroup on purpose. + assert reading.working_set is not None + assert reading.working_set < reading.memory_limit + assert reading.cpu_limit is None + + +def test_cpu_quota() -> None: + """Reads a fractional CPU quota.""" + reading = probe_in_container('--cpus', str(QUOTA_CORES)) + + check_invariants(reading) + assert reading.cpu_limit == QUOTA_CORES + assert reading.memory_limit is None + + +@pytest.mark.usefixtures('_two_cores') +def test_cpu_set_only() -> None: + """Reads a set of allowed cores, which restricts the CPU without any quota.""" + reading = probe_in_container('--cpuset-cpus', '0') + + check_invariants(reading) + assert reading.cpu_limit == 1.0 + assert reading.raw_cpu_set_size == 1 + assert reading.raw_cpu_quota is None + + +@pytest.mark.usefixtures('_two_cores') +def test_every_axis_at_once() -> None: + """Takes the tighter of the two CPU axes. Here the quota is tighter than the set.""" + reading = probe_in_container( + '--memory', + str(MEMORY_LIMIT), + '--cpus', + str(QUOTA_CORES), + '--cpuset-cpus', + '0', + ) + + check_invariants(reading) + assert reading.memory_limit == MEMORY_LIMIT + assert reading.cpu_limit == QUOTA_CORES + assert reading.raw_cpu_set_size == 1 + + +def test_host_cgroup_namespace() -> None: + """Reads the limit with the cgroup namespace of the machine, not one of its own.""" + reading = probe_in_container('--memory', str(MEMORY_LIMIT), '--cgroupns=host') + + check_invariants(reading) + assert reading.memory_limit == MEMORY_LIMIT + + # Under cgroup v2 the whole hierarchy is mounted, so the container sees its ancestry and the walk has + # several levels. Under cgroup v1 each controller is mounted at the container's own cgroup, which hides + # the ancestry whatever the namespace. + if reading.sources['memory']['interface'] == 'cgroup-v2': + assert len(reading.sources['memory']['levels']) > 1 + + +def test_tighter_limit_inside_the_container() -> None: + """Takes the tightest limit when two levels carry different ones. + + The container gets one limit and the probe puts itself under a tighter one. That is the shape kubelet + produces, and the only one here where two levels disagree. + """ + # The kernel forbids a cgroup that both holds processes and delegates controllers, so the shell vacates + # the root first, then enables the controller, then moves into the tighter cgroup. `0` is how a process + # names itself to cgroupfs; an explicit PID takes the path meant for moving somebody else. + setup = f""" + set -e + if [ -e /sys/fs/cgroup/cgroup.controllers ]; then + mkdir -p /sys/fs/cgroup/init /sys/fs/cgroup/inner + echo 0 > /sys/fs/cgroup/init/cgroup.procs + echo +memory > /sys/fs/cgroup/cgroup.subtree_control + echo {TIGHTER_MEMORY_LIMIT} > /sys/fs/cgroup/inner/memory.max + echo 0 > /sys/fs/cgroup/inner/cgroup.procs + else + mkdir -p /sys/fs/cgroup/memory/inner + echo {TIGHTER_MEMORY_LIMIT} > /sys/fs/cgroup/memory/inner/memory.limit_in_bytes + echo 0 > /sys/fs/cgroup/memory/inner/cgroup.procs + fi + exec {probe_command()} + """ + + reading = probe_in_container( + '--privileged', + '--memory', + str(MEMORY_LIMIT), + command=['sh', '-c', setup], + ) + + check_invariants(reading) + assert reading.memory_limit == TIGHTER_MEMORY_LIMIT + + +@pytest.mark.parametrize('image', DISTRO_IMAGES) +def test_distributions(image: str) -> None: + """Reads the same limits whatever distribution the process runs on. + + The sensor only reads `/proc` and `/sys`, so the C library and the package layout should not matter. This + pins that: musl and glibc, and a distribution outside the Debian family. + """ + if not pull_image(image): + pytest.skip(f'{image} cannot be pulled') + + reading = probe_in_container( + '--memory', + str(MEMORY_LIMIT), + '--cpus', + str(QUOTA_CORES), + image=image, + ) + + check_invariants(reading) + assert reading.memory_limit == MEMORY_LIMIT + assert reading.cpu_limit == QUOTA_CORES + assert reading.cpu_usage is not None + + +@pytest.mark.parametrize('python_version', PYTHON_VERSIONS) +def test_python_versions(python_version: str) -> None: + """Reads the same limits on every supported interpreter, in a container this time.""" + reading = probe_in_container( + '--memory', + str(MEMORY_LIMIT), + '--cpus', + str(QUOTA_CORES), + python_version=python_version, + ) + + check_invariants(reading) + assert reading.memory_limit == MEMORY_LIMIT + assert reading.cpu_limit == QUOTA_CORES + + +@pytest.mark.usefixtures('_sudo', '_systemd_cgroup_driver') +def test_limit_on_a_parent_cgroup(parent_slice: str) -> None: + """Reads a limit set outside the container, on the slice the container was put into. + + The container carries no limit of its own here, so the sensor has to walk up to find one. + """ + reading = probe_in_container('--cgroupns=host', f'--cgroup-parent={parent_slice}') + + check_invariants(reading) + assert reading.memory_limit == MEMORY_LIMIT + # The limit sits above the container's own cgroup. + assert len(reading.sources['memory']['levels']) > 1 diff --git a/tests/e2e/test_kubernetes.py b/tests/e2e/test_kubernetes.py new file mode 100644 index 0000000..a39722a --- /dev/null +++ b/tests/e2e/test_kubernetes.py @@ -0,0 +1,209 @@ +from __future__ import annotations + +import json +import subprocess +import time +from typing import TYPE_CHECKING, Any + +import pytest + +from .harness import ( + IMAGE, + MEMORY_LIMIT, + PROBE, + SRC_DIR, + TIMEOUT_SECONDS, + check_invariants, + have, + parse, + unavailable, +) + +if TYPE_CHECKING: + from collections.abc import Iterator + + from .harness import Reading + +CLUSTER = 'cgroups-sensor-e2e' +POD = 'sensor-probe' +SRC_MAP = 'sensor-src' +PROBE_MAP = 'sensor-probe-script' +POD_WAIT_SECONDS = 300 + + +def kubectl(*args: str, check: bool = True) -> str: + """Run one kubectl command against the test cluster.""" + result = subprocess.run( + ['kubectl', '--context', f'kind-{CLUSTER}', *args], + capture_output=True, + text=True, + timeout=TIMEOUT_SECONDS, + check=False, + ) + if check and result.returncode != 0: + pytest.fail(f'kubectl {" ".join(args)} exited {result.returncode}\n{result.stderr}') + + return result.stdout + + +@pytest.fixture(scope='session') +def _cluster() -> Iterator[None]: + """Bring up a kind cluster carrying the probe image, and take it down afterwards.""" + if not have('kind') or not have('kubectl') or not have('docker'): + unavailable('kubernetes', 'kind, kubectl and docker are needed to run a cluster') + + existing = subprocess.run( + ['kind', 'get', 'clusters'], + capture_output=True, + text=True, + timeout=TIMEOUT_SECONDS, + check=False, + ) + pre_existing = CLUSTER in existing.stdout.split() + + if not pre_existing: + created = subprocess.run( + ['kind', 'create', 'cluster', '--name', CLUSTER, '--wait', '120s'], + capture_output=True, + text=True, + timeout=900, + check=False, + ) + if created.returncode != 0: + unavailable('kubernetes', f'the cluster could not be created: {created.stderr.strip()[-300:]}') + + subprocess.run( + ['docker', 'pull', '--quiet', IMAGE], + capture_output=True, + timeout=TIMEOUT_SECONDS, + check=False, + ) + subprocess.run( + ['kind', 'load', 'docker-image', IMAGE, '--name', CLUSTER], + capture_output=True, + timeout=900, + check=False, + ) + + # The package and the probe travel as config maps, so nothing has to be built into an image. The files go + # in one by one, or a stray `__pycache__` would travel with them. + sources = sorted((SRC_DIR / 'cgroups_sensor').glob('*.py')) + kubectl('delete', 'configmap', SRC_MAP, PROBE_MAP, '--ignore-not-found') + kubectl('create', 'configmap', SRC_MAP, *[f'--from-file={path}' for path in sources]) + kubectl('create', 'configmap', PROBE_MAP, f'--from-file={PROBE}') + + yield + + kubectl('delete', 'configmap', SRC_MAP, PROBE_MAP, '--ignore-not-found', check=False) + if not pre_existing: + subprocess.run( + ['kind', 'delete', 'cluster', '--name', CLUSTER], + capture_output=True, + timeout=300, + check=False, + ) + + +def manifest(resources: dict[str, Any]) -> str: + """Spell the probe pod, with the container resources it should be given.""" + return json.dumps( + { + 'apiVersion': 'v1', + 'kind': 'Pod', + 'metadata': {'name': POD}, + 'spec': { + 'restartPolicy': 'Never', + 'containers': [ + { + 'name': 'probe', + 'image': IMAGE, + 'imagePullPolicy': 'Never', + 'command': ['python3', '/probe/probe.py'], + 'env': [{'name': 'PYTHONPATH', 'value': '/sensor'}], + 'resources': resources, + 'volumeMounts': [ + {'name': 'src', 'mountPath': '/sensor/cgroups_sensor'}, + {'name': 'probe', 'mountPath': '/probe'}, + ], + } + ], + 'volumes': [ + {'name': 'src', 'configMap': {'name': SRC_MAP}}, + {'name': 'probe', 'configMap': {'name': PROBE_MAP}}, + ], + }, + } + ) + + +def probe_in_pod(resources: dict[str, Any]) -> Reading: + """Run the probe in a one-shot pod and read what it printed.""" + kubectl('delete', 'pod', POD, '--ignore-not-found', '--now') + + apply = subprocess.run( + ['kubectl', '--context', f'kind-{CLUSTER}', 'apply', '-f', '-'], + input=manifest(resources), + capture_output=True, + text=True, + timeout=TIMEOUT_SECONDS, + check=False, + ) + if apply.returncode != 0: + pytest.fail(f'the pod could not be created:\n{apply.stderr}') + + phase = '' + for _ in range(POD_WAIT_SECONDS): + phase = kubectl('get', f'pod/{POD}', '-o', 'jsonpath={.status.phase}', check=False).strip() + if phase in {'Succeeded', 'Failed'}: + break + + reason = kubectl( + 'get', + f'pod/{POD}', + '-o', + 'jsonpath={.status.conditions[?(@.type=="PodScheduled")].reason}', + check=False, + ).strip() + if reason == 'Unschedulable': + pytest.skip('the pod does not fit this node') + + time.sleep(1) + + logs = kubectl('logs', f'pod/{POD}', check=False) + if phase != 'Succeeded': + pytest.fail(f'the pod ended as {phase or "unknown"}\n{logs}\n{kubectl("describe", f"pod/{POD}", check=False)}') + + reading = parse(logs) + kubectl('delete', 'pod', POD, '--ignore-not-found', '--now', check=False) + + return reading + + +@pytest.mark.usefixtures('_cluster') +def test_pod_with_limits() -> None: + """Reads container limits from inside the pod's own cgroup namespace. + + kubelet nests a pod several levels under kubepods. From inside, the container sees itself at the root. + """ + reading = probe_in_pod( + { + # Small requests on purpose: kubernetes copies limits into requests when none are given, and a pod + # requesting the whole node can never be scheduled. + 'requests': {'cpu': '50m', 'memory': '64Mi'}, + 'limits': {'cpu': '500m', 'memory': str(MEMORY_LIMIT)}, + } + ) + + check_invariants(reading) + assert reading.memory_limit == MEMORY_LIMIT + assert reading.cpu_limit == 0.5 + + +@pytest.mark.usefixtures('_cluster') +def test_pod_without_limits() -> None: + """Reports no restriction for a pod that sets none, inside the same kubepods hierarchy.""" + reading = probe_in_pod({}) + + check_invariants(reading) + assert reading.memory_limit is None + assert reading.cpu_limit is None diff --git a/tests/e2e/test_machine.py b/tests/e2e/test_machine.py new file mode 100644 index 0000000..699aa7b --- /dev/null +++ b/tests/e2e/test_machine.py @@ -0,0 +1,186 @@ +from __future__ import annotations + +import os +from typing import TYPE_CHECKING + +import pytest + +from .harness import ( + CANNOT_SET_UP, + MEMORY_LIMIT, + PYTHON_VERSIONS, + QUOTA_CORES, + check_invariants, + is_unified, + machine_cpu_count, + machine_memory_bytes, + notices_about, + probe_here, +) + +if TYPE_CHECKING: + from collections.abc import Callable + + +def test_unrestricted() -> None: + """Reads this machine as it is. Whatever it reports has to be coherent.""" + reading = probe_here() + + check_invariants(reading) + # A Linux machine always has a cgroup, even when nothing is limited there. + assert reading.sources['memory'] is not None + assert reading.machine_cpu_count == machine_cpu_count() + assert reading.machine_memory_bytes == machine_memory_bytes() + + +@pytest.mark.parametrize('python_version', PYTHON_VERSIONS) +def test_python_versions(python_version: str) -> None: + """Reads the same machine facts on every supported interpreter. + + The version is a real axis here. `os.cpu_count()` started honoring `PYTHON_CPU_COUNT` in 3.13, and the + filters compare against these numbers. + """ + reading = probe_here(python_version=python_version) + + check_invariants(reading) + assert reading.machine_cpu_count == machine_cpu_count() + assert reading.machine_memory_bytes == machine_memory_bytes() + + +@pytest.mark.usefixtures('_systemd_user', '_unified') +def test_memory_limit(systemd_scope: Callable[..., list[str]]) -> None: + """Reads a memory limit set on the scope this process runs in.""" + reading = probe_here(systemd_scope(f'MemoryMax={MEMORY_LIMIT}')) + + check_invariants(reading) + assert reading.memory_limit == MEMORY_LIMIT + # The floor `check_invariants` applies is the memory the probe charged to this cgroup on purpose. + assert reading.working_set is not None + assert reading.working_set < reading.memory_limit + # The machine can carry unrelated notices, e.g. about a CPU set covering every core. + assert not notices_about(reading, 'memory') + + +@pytest.mark.usefixtures('_systemd_user', '_unified') +def test_cpu_quota(systemd_scope: Callable[..., list[str]]) -> None: + """Reads a CPU quota set on the scope this process runs in.""" + reading = probe_here(systemd_scope(f'CPUQuota={int(QUOTA_CORES * 100)}%')) + + check_invariants(reading) + assert reading.cpu_limit == QUOTA_CORES + assert reading.raw_cpu_quota == QUOTA_CORES + + +@pytest.mark.usefixtures('_systemd_user', '_unified') +def test_memory_limit_above_the_machine(systemd_scope: Callable[..., list[str]]) -> None: + """Drops a memory limit larger than the machine, and says so. + + Nothing rejects such a limit. A container given more memory than the node it landed on gets one. + """ + limit = machine_memory_bytes() + 1024**3 + reading = probe_here(systemd_scope(f'MemoryMax={limit}')) + + check_invariants(reading) + assert reading.memory_limit is None + assert reading.raw_memory_limit == limit + assert 'memory-limit-covers-machine' in reading.notices + + +@pytest.mark.usefixtures('_systemd_user', '_unified') +def test_cpu_quota_above_the_machine(systemd_scope: Callable[..., list[str]]) -> None: + """Drops a CPU quota larger than the machine, and says so. + + systemd writes the quota as asked. A Kubernetes limit above node capacity does the same. + """ + quota_cores = machine_cpu_count() + 1 + reading = probe_here(systemd_scope(f'CPUQuota={quota_cores * 100}%')) + + check_invariants(reading) + assert reading.cpu_limit is None + assert reading.raw_cpu_quota == quota_cores + assert 'cpu-quota-covers-machine' in reading.notices + + +@pytest.mark.usefixtures('_sudo', '_systemd_system', '_two_cores') +def test_every_axis_at_once(systemd_scope: Callable[..., list[str]]) -> None: + """Reads all three limits at once. The tighter of the two CPU axes wins. + + A system scope, not a user one: under cgroup v1 a user scope gets no cgroup in the resource controllers, + and `AllowedCPUs=` needs a delegated cpuset that only the unified hierarchy has. + """ + properties = [f'MemoryMax={MEMORY_LIMIT}', f'CPUQuota={int(QUOTA_CORES * 100)}%'] + if is_unified(): + properties.append('AllowedCPUs=0') + + reading = probe_here(systemd_scope(*properties, system=True)) + + check_invariants(reading) + assert reading.memory_limit == MEMORY_LIMIT + # The quota is half a core, the set is a whole one. + assert reading.cpu_limit == QUOTA_CORES + if is_unified(): + assert reading.raw_cpu_set_size == 1 + + +@pytest.mark.usefixtures('_systemd_user', '_unified') +def test_limit_on_an_ancestor(systemd_scope: Callable[..., list[str]]) -> None: + """Reads a limit from an ancestor when the own cgroup carries no memory files at all. + + `Delegate=yes` lets the probe move into a child cgroup. That child inherits no memory controller, so the + sensor has to walk up past a level with nothing to read. + """ + wrapper = systemd_scope(f'MemoryMax={MEMORY_LIMIT}', 'Delegate=yes') + move_into_leaf = [ + 'bash', + '-c', + ( + 'set -e; ' + 'own=$(grep "^0::" /proc/self/cgroup | cut -d: -f3); ' + 'leaf="/sys/fs/cgroup$own/leaf"; ' + # What a delegated scope is for. Where systemd delegates it differently - the Fedora guest, with a + # much newer one - the shape cannot be built, and this says so instead of probing the scope itself + # and blaming the reading. The errno of whichever step failed travels out on stderr. + # `0` is the kernel's word for "the writing process". An explicit PID takes the path meant for + # moving somebody else, which the kernel of Fedora 43 refuses with EINVAL where the older one of + # Ubuntu 22.04 allowed it. + f'{{ mkdir -p "$leaf" && echo 0 > "$leaf/cgroup.procs"; }} || {{ ' + # Where it still fails, say what the kernel refused and who owns what a delegated scope hands + # over - the two things that tell a missing delegation apart from a refused move. + 'echo "own=$own type=$(cat "/sys/fs/cgroup$own/cgroup.type" 2>&1)"' + ' "subtree=[$(cat "/sys/fs/cgroup$own/cgroup.subtree_control" 2>&1)]"' + ' "as=$(id -u):$(id -g)" "owner=$(stat -c %U:%G:%a "$leaf/cgroup.procs" 2>&1)" >&2; ' + f'exit {CANNOT_SET_UP}; }}; ' + 'exec "$@"' + ), + '--', + ] + reading = probe_here([*wrapper, *move_into_leaf]) + + check_invariants(reading) + assert reading.memory_limit == MEMORY_LIMIT + + # The walk starts at the leaf, which carries no memory files at all, and finds the limit above it. + levels = reading.sources['memory']['levels'] + assert levels[0].endswith('/leaf') + assert len(levels) > 1 + # The chain says where it looked, this says where the limit came from. + assert reading.memory_limit_level in levels[1:] + + +def test_expected_interface() -> None: + """Check that this run reached the cgroup interface it was started for. + + A guest booted for cgroup v1 that comes up on v2 runs a smaller suite and reports success, which is the + one way this lane can lie. `E2E_INTERFACE=cgroup-v1` makes it say so instead. + + The limits are what has to come from that interface. The CPU counter is allowed to differ, for the reason + `Reading.limit_interfaces` gives. + """ + expected = os.environ.get('E2E_INTERFACE') + if not expected: + pytest.skip('E2E_INTERFACE names no interface to check') + + reading = probe_here() + + assert reading.limit_interfaces, 'no mechanism carries any limit here' + assert reading.limit_interfaces == {expected} diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py new file mode 100644 index 0000000..bef19e0 --- /dev/null +++ b/tests/unit/conftest.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from cgroups_sensor import _cgroup + +if TYPE_CHECKING: + from collections.abc import Callable, Iterator + from pathlib import Path + +V2_MOUNTINFO = '25 30 0:22 / {root} rw,nosuid,nodev,noexec,relatime shared:4 - cgroup2 cgroup2 rw,nsdelegate' +"""A single unified hierarchy exposed from its top, which is what a container runtime sets up.""" + +V1_MOUNTINFO = ( + '29 25 0:25 / {root}/systemd rw,nosuid shared:9 - cgroup cgroup rw,name=systemd\n' + '30 25 0:26 / {root}/memory rw,nosuid shared:14 - cgroup cgroup rw,memory\n' + '31 25 0:27 / {root}/cpu,cpuacct rw,nosuid shared:15 - cgroup cgroup rw,cpu,cpuacct\n' + '32 25 0:28 / {root}/cpuset rw,nosuid shared:16 - cgroup cgroup rw,cpuset' +) +"""One hierarchy per controller, with the CPU accounting sharing a mount with the CPU bandwidth controller.""" + +V1_CPU_SPLIT_MOUNTINFO = ( + '30 25 0:26 / {root}/cpu rw,nosuid shared:14 - cgroup cgroup rw,cpu\n' + '31 25 0:27 / {root}/cpuacct rw,nosuid shared:15 - cgroup cgroup rw,cpuacct' +) +"""cgroup v1 with the quota and the accounting in hierarchies of their own, each with its own mount point.""" + +V1_CPUSET_MOUNTINFO = '30 25 0:26 / {root}/cpuset rw,nosuid shared:14 - cgroup cgroup rw,cpuset' +"""A cpuset hierarchy alone, which is a machine where nothing counts CPU time at all.""" + +V1_CPUSET_SPLIT_MOUNTINFO = ( + V1_CPUSET_MOUNTINFO + '\n31 25 0:27 / {root}/cpuacct rw,nosuid shared:15 - cgroup cgroup rw,cpuacct' +) +"""The set and the accounting in hierarchies of their own, which `cgexec -g cpuset:...` moves a process into.""" + +HYBRID_MOUNTINFO = ( + '25 30 0:22 / {root}/unified rw shared:4 - cgroup2 cgroup2 rw,nsdelegate\n' + '30 25 0:26 / {root}/memory rw,nosuid shared:14 - cgroup cgroup rw,memory\n' + '31 25 0:27 / {root}/cpu,cpuacct rw,nosuid shared:15 - cgroup cgroup rw,cpu,cpuacct' +) +"""What systemd mounts with `systemd.unified_cgroup_hierarchy=0`: the controllers on cgroup v1, and a cgroup2 +that carries none of them next to those.""" + +V2_SELF_CGROUP = '0::{path}\n' +"""The unified hierarchy is the entry with no controllers listed.""" + +V1_SELF_CGROUP = '4:cpuset:{path}\n3:cpu,cpuacct:{path}\n2:memory:{path}\n1:name=systemd:{path}\n' +"""One entry per cgroup v1 hierarchy, including the named one that carries no controller.""" + +HYBRID_SELF_CGROUP = '0::/system.slice/docker.service\n3:cpu,cpuacct:/docker/abc\n2:memory:/docker/abc\n' +"""The two interfaces name different cgroups for the same process, which is what makes the layout dangerous.""" + + +@pytest.fixture(autouse=True) +def _isolated_module_state() -> Iterator[None]: + """Reset the process-wide discovery cache, so that nothing leaks between tests.""" + _cgroup.clear_cache() + yield + _cgroup.clear_cache() + + +@pytest.fixture +def fake_cgroup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Callable[..., Path]: + """Return a builder that lays out a fake cgroup filesystem, points the backend at it and returns its root.""" + + def build(*, mountinfo: str, self_cgroup: str, files: dict[str, str]) -> Path: + root = tmp_path / 'cgroup' + root.mkdir(parents=True, exist_ok=True) + + for relative_path, content in files.items(): + path = root / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + + mountinfo_path = tmp_path / 'mountinfo' + mountinfo_path.write_text(mountinfo.format(root=root)) + self_cgroup_path = tmp_path / 'self_cgroup' + self_cgroup_path.write_text(self_cgroup) + + monkeypatch.setattr(_cgroup, '_PROC_SELF_MOUNTINFO', mountinfo_path) + monkeypatch.setattr(_cgroup, '_PROC_SELF_CGROUP', self_cgroup_path) + + return root + + return build + + +@pytest.fixture +def _no_cgroup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Point the backend at `/proc` files that do not exist, as on a system without cgroups.""" + monkeypatch.setattr(_cgroup, '_PROC_SELF_MOUNTINFO', tmp_path / 'missing') + monkeypatch.setattr(_cgroup, '_PROC_SELF_CGROUP', tmp_path / 'missing') diff --git a/tests/unit/test_cgroup.py b/tests/unit/test_cgroup.py new file mode 100644 index 0000000..9edc598 --- /dev/null +++ b/tests/unit/test_cgroup.py @@ -0,0 +1,1543 @@ +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest + +from cgroups_sensor import _cgroup as cgroup + +from .conftest import ( + HYBRID_MOUNTINFO, + HYBRID_SELF_CGROUP, + V1_CPU_SPLIT_MOUNTINFO, + V1_CPUSET_MOUNTINFO, + V1_CPUSET_SPLIT_MOUNTINFO, + V1_MOUNTINFO, + V1_SELF_CGROUP, + V2_MOUNTINFO, + V2_SELF_CGROUP, +) + +if TYPE_CHECKING: + from collections.abc import Callable + + +def test_read_hierarchies_v2(fake_cgroup: Callable[..., Path]) -> None: + """Finds the unified hierarchy and the cgroup this process belongs to in it.""" + # Every cgroup carries `cgroup.procs`, and the mount is only taken as ours where our cgroup is under it. + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/init.scope'), + files={'init.scope/cgroup.procs': ''}, + ) + + unified, controllers = cgroup._read_hierarchies() + + assert unified is not None + assert unified.mount_point == root + assert unified.mount_root == '/' + assert unified.own_path == '/init.scope' + assert controllers == {} + + +def test_read_hierarchies_v1(fake_cgroup: Callable[..., Path]) -> None: + """Finds every controller of a cgroup v1 mount that carries more than one.""" + root = fake_cgroup( + mountinfo=V1_MOUNTINFO, + self_cgroup=V1_SELF_CGROUP.format(path='/docker/abc'), + # A mount counts as ours only where our cgroup exists under it, and every cgroup carries this file. + files={ + 'memory/docker/abc/cgroup.procs': '', + 'cpu,cpuacct/docker/abc/cgroup.procs': '', + 'cpuset/docker/abc/cgroup.procs': '', + }, + ) + + unified, controllers = cgroup._read_hierarchies() + + assert unified is None + assert controllers['memory'].mount_point == root / 'memory' + assert controllers['memory'].own_path == '/docker/abc' + assert controllers['cpuset'].mount_point == root / 'cpuset' + # Otherwise the CPU metrics get split across versions. + assert controllers['cpu'].mount_point == root / 'cpu,cpuacct' + assert controllers['cpuacct'].mount_point == root / 'cpu,cpuacct' + + +def test_read_hierarchies_bad_lines(fake_cgroup: Callable[..., Path]) -> None: + """Skips the lines it cannot parse, which a table of every filesystem on the machine is full of.""" + mountinfo = ( + 'not a mount line\n' + '24 30 0:21 / /sys rw - sysfs sysfs rw\n' + # The ` - ` separator is present but the fields around it are too few to be a mount entry. + '25 30 0:22 - cgroup2 cgroup2 rw\n' + '26 30 0:23 / /x rw - cgroup2\n' + f'{V2_MOUNTINFO}\n' + '25 30 0:22 /' + ) + root = fake_cgroup(mountinfo=mountinfo, self_cgroup=V2_SELF_CGROUP.format(path='/'), files={}) + + unified, _controllers = cgroup._read_hierarchies() + + assert unified is not None + assert unified.mount_point == root + + +def test_read_hierarchies_first_cgroup2_mount_wins(fake_cgroup: Callable[..., Path]) -> None: + """Keeps the first of two cgroup2 mounts that both expose this cgroup, as the mount order suggests.""" + mountinfo = ( + '25 30 0:22 / {root}/first rw shared:4 - cgroup2 cgroup2 rw,nsdelegate\n' + '26 30 0:22 / {root}/second rw shared:5 - cgroup2 cgroup2 rw,nsdelegate' + ) + root = fake_cgroup( + mountinfo=mountinfo, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'first/cgroup.controllers': 'cpu memory\n', 'second/cgroup.controllers': 'cpu memory\n'}, + ) + + unified, _controllers = cgroup._read_hierarchies() + + assert unified is not None + assert unified.mount_point == root / 'first' + + +def test_read_hierarchies_conventional_mount_point_wins( + fake_cgroup: Callable[..., Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Prefers the conventional mount point over the mount order when both expose this cgroup. + + A second mount of the same hierarchy is somebody's tool. `/sys/fs/cgroup` is where the system put it. + """ + mountinfo = ( + '25 30 0:22 / {root}/elsewhere rw shared:4 - cgroup2 cgroup2 rw,nsdelegate\n' + '26 30 0:22 / {root}/conventional rw shared:5 - cgroup2 cgroup2 rw,nsdelegate' + ) + root = fake_cgroup( + mountinfo=mountinfo, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'elsewhere/cgroup.procs': '', 'conventional/cgroup.procs': ''}, + ) + monkeypatch.setattr(cgroup, '_CONVENTIONAL_MOUNT_POINT', root / 'conventional') + + unified, _controllers = cgroup._read_hierarchies() + + assert unified is not None + assert unified.mount_point == root / 'conventional' + + +def test_read_hierarchies_skips_a_foreign_cgroup2_mount(fake_cgroup: Callable[..., Path]) -> None: + """Skips a cgroup2 mount that exposes another subtree, however early it is listed. + + A runtime can bind-mount one container's cgroup into another, and an agent watching the machine mounts a + subtree of its own. Reading the first line of the mount table would report those numbers as this process's. + """ + mountinfo = ( + '25 30 0:22 /other {root}/foreign rw shared:4 - cgroup2 cgroup2 rw,nsdelegate\n' + '26 30 0:22 / {root}/ours rw shared:5 - cgroup2 cgroup2 rw,nsdelegate' + ) + root = fake_cgroup( + mountinfo=mountinfo, + self_cgroup=V2_SELF_CGROUP.format(path='/mine'), + files={'foreign/memory.max': '1000\n', 'ours/mine/memory.max': '2000\n'}, + ) + + unified, _controllers = cgroup._read_hierarchies() + + assert unified is not None + assert unified.mount_point == root / 'ours' + + +def test_read_hierarchies_skips_a_mount_without_this_cgroup(fake_cgroup: Callable[..., Path]) -> None: + """Skips a mount of the right subtree that does not actually hold this cgroup.""" + mountinfo = ( + '25 30 0:22 / {root}/stale rw shared:4 - cgroup2 cgroup2 rw,nsdelegate\n' + '26 30 0:22 / {root}/live rw shared:5 - cgroup2 cgroup2 rw,nsdelegate' + ) + root = fake_cgroup( + mountinfo=mountinfo, + self_cgroup=V2_SELF_CGROUP.format(path='/mine'), + files={'stale/other/memory.max': '1000\n', 'live/mine/memory.max': '2000\n'}, + ) + + unified, _controllers = cgroup._read_hierarchies() + + assert unified is not None + assert unified.mount_point == root / 'live' + + +def test_read_hierarchies_no_unified_when_no_mount_covers(fake_cgroup: Callable[..., Path]) -> None: + """Reports no unified hierarchy when none of the mounts exposes this cgroup. + + Reading the mount anyway would answer with the numbers of whatever cgroup sits at the top of it. + """ + mountinfo = '25 30 0:22 /elsewhere {root}/only rw shared:4 - cgroup2 cgroup2 rw,nsdelegate' + fake_cgroup( + mountinfo=mountinfo, + self_cgroup=V2_SELF_CGROUP.format(path='/mine'), + files={'only/memory.max': '1000\n', 'only/memory.current': '900\n', 'only/memory.stat': 'inactive_file 0\n'}, + ) + + unified, _controllers = cgroup._read_hierarchies() + + assert unified is None + assert cgroup.read_memory() == cgroup.RawMemory( + limit=None, + working_set=None, + limit_directory=None, + unreadable_directory=None, + ) + + +def test_read_hierarchies_v1_answers_when_no_cgroup2_mount_covers(fake_cgroup: Callable[..., Path]) -> None: + """Leaves the cgroup v1 hierarchies to answer when the cgroup2 mount is somebody else's. + + A hybrid machine carries both, and a foreign cgroup2 mount claimed as ours would mask the v1 controllers + that do describe this process. + """ + foreign = '25 30 0:22 /elsewhere {root}/foreign rw shared:4 - cgroup2 cgroup2 rw,nsdelegate' + root = fake_cgroup( + mountinfo=f'{foreign}\n{V1_MOUNTINFO}', + self_cgroup=f'{V2_SELF_CGROUP.format(path="/mine")}2:memory:/\n', + files={ + 'foreign/memory.max': '1000\n', + 'foreign/memory.current': '900\n', + 'foreign/memory.stat': 'inactive_file 0\n', + 'memory/memory.limit_in_bytes': '536870912\n', + 'memory/memory.usage_in_bytes': '1000\n', + 'memory/memory.stat': 'total_inactive_file 400\n', + }, + ) + + assert cgroup.read_memory() == cgroup.RawMemory( + limit=536870912, + working_set=600, + limit_directory=root / 'memory', + unreadable_directory=None, + ) + + +@pytest.mark.parametrize( + ('escaped', 'expected'), + [ + pytest.param('/plain/path', '/plain/path', id='nothing to decode'), + pytest.param('/mnt\\040point', '/mnt point', id='space'), + pytest.param('/tab\\011here', '/tab\there', id='tab'), + pytest.param('/back\\134slash', '/back\\slash', id='backslash'), + pytest.param('/literal\\134040', '/literal\\040', id='escaped backslash in front of an octal sequence'), + ], +) +def test_unescape(escaped: str, expected: str) -> None: + """Decodes the octal sequences a path field of the mount table escapes special characters as.""" + assert cgroup._unescape(escaped) == expected + + +def test_read_hierarchies_escaped_paths(fake_cgroup: Callable[..., Path]) -> None: + """Decodes both path fields, which `/proc/self/cgroup` spells unescaped and so cannot be compared against.""" + mountinfo = '25 30 0:22 /docker\\040abc {root}/mnt\\040point rw shared:4 - cgroup2 cgroup2 rw' + root = fake_cgroup( + mountinfo=mountinfo, + self_cgroup=V2_SELF_CGROUP.format(path='/docker abc'), + files={'mnt point/cgroup.procs': ''}, + ) + + unified, _controllers = cgroup._read_hierarchies() + + assert unified is not None + assert unified.mount_point == root / 'mnt point' + assert unified.mount_root == '/docker abc' + + +@pytest.mark.parametrize( + ('self_cgroup', 'expected_unified', 'expected_controllers'), + [ + pytest.param('0::/init.scope\n', '/init.scope', {}, id='unified only'), + pytest.param( + '2:memory:/docker/abc\n1:name=systemd:/docker/abc\n', + None, + {'memory': '/docker/abc', 'systemd': '/docker/abc'}, + id='v1 only, with a named hierarchy', + ), + pytest.param( + '2:memory:/system.slice\n0::/user.slice\n', + '/user.slice', + {'memory': '/system.slice'}, + id='both interfaces at once', + ), + pytest.param('nonsense\n0::/\n', '/', {}, id='unparsable line skipped'), + ], +) +def test_read_own_paths( + fake_cgroup: Callable[..., Path], + self_cgroup: str, + expected_unified: str | None, + expected_controllers: dict[str, str], +) -> None: + """Reads the cgroup this process belongs to in each hierarchy.""" + fake_cgroup(mountinfo=V2_MOUNTINFO, self_cgroup=self_cgroup, files={}) + + unified, controllers = cgroup._read_own_paths() + + assert unified == expected_unified + assert controllers == expected_controllers + + +@pytest.mark.parametrize( + ('mount_root', 'cgroup_path', 'expected'), + [ + pytest.param('/', '/', [''], id='own cgroup at the top of the mount'), + pytest.param( + '/', + '/kubepods/pod/container', + ['kubepods/pod/container', 'kubepods/pod', 'kubepods', ''], + id='nested', + ), + pytest.param('/docker/abc', '/docker/abc/nested', ['nested', ''], id='mount root stripped'), + pytest.param('/docker/abc', '/system.slice', [''], id='mount does not cover the own cgroup'), + # `unshare -C` without a remount leaves the mount pointing above the new namespace's root. + pytest.param('/..', '/foo', [''], id='mount root above the cgroup namespace'), + ], +) +def test_candidate_dirs( + tmp_path: Path, + mount_root: str, + cgroup_path: str, + expected: list[str], +) -> None: + """Walks from the cgroup of this process up to the top of the mount.""" + hierarchy = cgroup._Hierarchy(mount_point=tmp_path, mount_root=mount_root, own_path=cgroup_path) + + dirs = cgroup._candidate_dirs(hierarchy) + + assert list(dirs) == [tmp_path / relative if relative else tmp_path for relative in expected] + + +@pytest.mark.parametrize( + ('mountinfo', 'self_cgroup', 'files', 'expected'), + [ + pytest.param( + V2_MOUNTINFO, + V2_SELF_CGROUP.format(path='/'), + {'memory.max': '536870912\n', 'memory.current': '1000\n', 'memory.stat': 'inactive_file 400\n'}, + 536870912, + id='v2 limit', + ), + pytest.param( + V2_MOUNTINFO, + V2_SELF_CGROUP.format(path='/'), + {'memory.max': 'max\n', 'memory.current': '1000\n', 'memory.stat': 'inactive_file 400\n'}, + None, + id='v2 unlimited', + ), + pytest.param( + V2_MOUNTINFO, + V2_SELF_CGROUP.format(path='/'), + # `memory.high` throttles reclaim instead of triggering an out-of-memory kill, so it is not a limit. + {'memory.high': '536870912\n', 'memory.current': '1000\n', 'memory.stat': 'inactive_file 400\n'}, + None, + id='memory.high without memory.max is not a limit', + ), + pytest.param( + V2_MOUNTINFO, + V2_SELF_CGROUP.format(path='/pod/container'), + { + 'pod/container/memory.max': 'max\n', + 'pod/container/memory.current': '1000\n', + 'pod/container/memory.stat': 'inactive_file 400\n', + 'pod/memory.max': '268435456\n', + 'pod/memory.current': '9000\n', + 'pod/memory.stat': 'inactive_file 1000\n', + }, + 268435456, + id='limit inherited from an ancestor', + ), + pytest.param( + V1_MOUNTINFO, + V1_SELF_CGROUP.format(path='/'), + { + 'memory/memory.limit_in_bytes': '536870912\n', + 'memory/memory.usage_in_bytes': '1000\n', + 'memory/memory.stat': 'total_inactive_file 400\n', + }, + 536870912, + id='v1 limit', + ), + pytest.param( + V1_MOUNTINFO, + V1_SELF_CGROUP.format(path='/'), + # The sentinel is passed through as read - only the sensor layer knows the memory of the machine. + { + 'memory/memory.limit_in_bytes': '9223372036854771712\n', + 'memory/memory.usage_in_bytes': '1000\n', + 'memory/memory.stat': 'total_inactive_file 400\n', + }, + 9223372036854771712, + id='v1 unlimited sentinel', + ), + ], +) +def test_read_memory( + fake_cgroup: Callable[..., Path], + mountinfo: str, + self_cgroup: str, + files: dict[str, str], + expected: int | None, +) -> None: + """Reads the memory limit under both cgroup interfaces.""" + fake_cgroup(mountinfo=mountinfo, self_cgroup=self_cgroup, files=files) + + assert cgroup.read_memory().limit == expected + + +def test_read_memory_tightest_level(fake_cgroup: Callable[..., Path]) -> None: + """Reports the tightest limit of the chain, so a budget cannot exceed what the kernel enforces.""" + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/kubepods/pod/container'), + files={ + # The node-level cgroup is generous and busy, because every pod on the node is charged against it. + 'kubepods/memory.max': '8000\n', + 'kubepods/memory.current': '6000\n', + 'kubepods/memory.stat': 'inactive_file 0\n', + # The container this process runs in is limited far more tightly, and barely uses its share. + 'kubepods/pod/container/memory.max': '1000\n', + 'kubepods/pod/container/memory.current': '100\n', + 'kubepods/pod/container/memory.stat': 'inactive_file 0\n', + }, + ) + + # 900 bytes can still be allocated before the container's own limit, and 2000 before the node's. + assert cgroup.read_memory() == cgroup.RawMemory( + limit=1000, + working_set=100, + limit_directory=root / 'kubepods' / 'pod' / 'container', + unreadable_directory=None, + ) + + +def test_read_memory_smallest_distance(fake_cgroup: Callable[..., Path]) -> None: + """Keeps the smallest distance to a limit, which is not always at the tightest one. + + A kill follows from the memory that can still be allocated, so that distance is what has to survive the + move onto the tightest limit. + """ + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/qos/pod'), + files={ + # The pod is nearly full of its own generous limit. + 'qos/pod/memory.max': '1000\n', + 'qos/pod/memory.current': '960\n', + 'qos/pod/memory.stat': 'inactive_file 10\n', + # The QoS class above is tighter, but half of what it holds belongs to the sibling pods. + 'qos/memory.max': '500\n', + 'qos/memory.current': '260\n', + 'qos/memory.stat': 'inactive_file 10\n', + }, + ) + + # The pod is 50 bytes from its own limit, the QoS class 250 from its. The tighter distance wins, and the + # limit it is expressed against is the QoS class's. + assert cgroup.read_memory() == cgroup.RawMemory( + limit=500, + working_set=450, + limit_directory=root / 'qos', + unreadable_directory=None, + ) + + +def test_read_memory_partial_level(fake_cgroup: Callable[..., Path]) -> None: + """Reports the ceiling without a usage when the tightest level has none to pair with it. + + The looser levels know only their own distance, which would overstate what can still be allocated here. + """ + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/kubepods/pod/container'), + files={ + 'kubepods/memory.max': '8000\n', + 'kubepods/memory.current': '6000\n', + 'kubepods/memory.stat': 'inactive_file 0\n', + # The tightest limit, with no page cache metric to derive a working set from. + 'kubepods/pod/container/memory.max': '1000\n', + 'kubepods/pod/container/memory.current': '100\n', + 'kubepods/pod/container/memory.stat': 'anon 100\n', + }, + ) + + assert cgroup.read_memory() == cgroup.RawMemory( + limit=1000, + working_set=None, + limit_directory=root / 'kubepods' / 'pod' / 'container', + unreadable_directory=None, + ) + + +@pytest.mark.parametrize( + ('mountinfo', 'self_cgroup', 'files'), + [ + pytest.param( + V2_MOUNTINFO, + V2_SELF_CGROUP.format(path='/'), + { + 'memory.max': '536870912\n', + 'memory.current': '1000\n', + 'memory.stat': 'anon 600\ninactive_file 400\nfile 400\n', + }, + id='v2', + ), + pytest.param( + V1_MOUNTINFO, + V1_SELF_CGROUP.format(path='/'), + { + 'memory/memory.limit_in_bytes': '536870912\n', + 'memory/memory.usage_in_bytes': '1000\n', + 'memory/memory.stat': 'rss 600\ntotal_inactive_file 400\n', + }, + id='v1', + ), + ], +) +def test_read_memory_working_set( + fake_cgroup: Callable[..., Path], + mountinfo: str, + self_cgroup: str, + files: dict[str, str], +) -> None: + """Subtracts the page cache the kernel drops on demand, which would not predict a kill.""" + fake_cgroup(mountinfo=mountinfo, self_cgroup=self_cgroup, files=files) + + assert cgroup.read_memory().working_set == 600 + + +def test_read_memory_usage_above_the_limit(fake_cgroup: Callable[..., Path]) -> None: + """Clamps the working set to the limit, because sitting above it while the kernel reclaims is not actionable.""" + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'memory.max': '1000\n', 'memory.current': '1200\n', 'memory.stat': 'inactive_file 0\n'}, + ) + + assert cgroup.read_memory() == cgroup.RawMemory( + limit=1000, + working_set=1000, + limit_directory=root, + unreadable_directory=None, + ) + + +def test_read_memory_ancestor_without_usage_files(fake_cgroup: Callable[..., Path]) -> None: + """Counts an ancestor's limit towards the ceiling even when its own usage cannot be read at all.""" + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/pod/container'), + files={ + 'pod/container/memory.max': '1000\n', + 'pod/container/memory.current': '100\n', + 'pod/container/memory.stat': 'inactive_file 0\n', + # The tightest limit sits on the ancestor, which carries no `memory.current` of its own. + 'pod/memory.max': '500\n', + }, + ) + + # The tightest limit is the ancestor's, and nothing says how close it is to it. + assert cgroup.read_memory() == cgroup.RawMemory( + limit=500, + working_set=None, + limit_directory=root / 'pod', + unreadable_directory=None, + ) + + +def test_read_memory_no_working_set(fake_cgroup: Callable[..., Path]) -> None: + """Reports a limit that no usage can be paired with as-is, leaving the judgement to the sensor layer.""" + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'memory.max': '536870912\n', 'memory.current': '1000\n', 'memory.stat': 'anon 600\n'}, + ) + + assert cgroup.read_memory() == cgroup.RawMemory( + limit=536870912, + working_set=None, + limit_directory=root, + unreadable_directory=None, + ) + + +def test_read_memory_missing_files(fake_cgroup: Callable[..., Path]) -> None: + """Reads the closest level that carries the controller, which the own cgroup need not.""" + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/pod/container'), + files={ + 'pod/memory.max': '268435456\n', + 'pod/memory.current': '1000\n', + 'pod/memory.stat': 'inactive_file 400\n', + }, + ) + # The cgroup of the process exists, it just carries no memory files of its own. + (root / 'pod' / 'container').mkdir() + + assert cgroup.read_memory() == cgroup.RawMemory( + limit=268435456, + working_set=600, + limit_directory=root / 'pod', + unreadable_directory=None, + ) + + +@pytest.mark.parametrize( + ('mountinfo', 'self_cgroup', 'files', 'expected'), + [ + pytest.param( + V2_MOUNTINFO, + V2_SELF_CGROUP.format(path='/'), + {'cpu.max': '200000 100000\n', 'cpu.stat': 'usage_usec 0\n'}, + 2.0, + id='v2 quota', + ), + pytest.param( + V2_MOUNTINFO, + V2_SELF_CGROUP.format(path='/'), + {'cpu.max': '50000 100000\n', 'cpu.stat': 'usage_usec 0\n'}, + 0.5, + id='v2 fractional quota', + ), + pytest.param( + V2_MOUNTINFO, + V2_SELF_CGROUP.format(path='/'), + {'cpu.max': 'max 100000\n', 'cpu.stat': 'usage_usec 0\n'}, + None, + id='v2 unlimited', + ), + pytest.param( + V2_MOUNTINFO, + V2_SELF_CGROUP.format(path='/'), + # A weight shapes how siblings share contended CPU; it grants no quota and is not a limit. + {'cpu.weight': '100\n', 'cpu.stat': 'usage_usec 0\n'}, + None, + id='cpu.weight without a quota is not a limit', + ), + pytest.param( + V2_MOUNTINFO, + V2_SELF_CGROUP.format(path='/pod/container'), + { + 'pod/container/cpu.max': 'max 100000\n', + 'pod/container/cpu.stat': 'usage_usec 0\n', + 'pod/cpu.max': '150000 100000\n', + }, + 1.5, + id='v2 quota inherited from an ancestor', + ), + pytest.param( + V1_MOUNTINFO, + V1_SELF_CGROUP.format(path='/'), + { + 'cpu,cpuacct/cpu.cfs_quota_us': '150000\n', + 'cpu,cpuacct/cpu.cfs_period_us': '100000\n', + 'cpu,cpuacct/cpuacct.usage': '0\n', + }, + 1.5, + id='v1 quota', + ), + pytest.param( + V1_MOUNTINFO, + V1_SELF_CGROUP.format(path='/'), + { + 'cpu,cpuacct/cpu.cfs_quota_us': '-1\n', + 'cpu,cpuacct/cpu.cfs_period_us': '100000\n', + 'cpu,cpuacct/cpuacct.usage': '0\n', + }, + None, + id='v1 unlimited', + ), + ], +) +def test_read_cpu_quota( + fake_cgroup: Callable[..., Path], + mountinfo: str, + self_cgroup: str, + files: dict[str, str], + expected: float | None, +) -> None: + """Reads the CPU bandwidth quota under both cgroup interfaces.""" + fake_cgroup(mountinfo=mountinfo, self_cgroup=self_cgroup, files=files) + + quota = cgroup.read_cpu_quota() + + assert (quota.cores if quota is not None else None) == expected + + +@pytest.mark.parametrize( + ('mountinfo', 'self_cgroup', 'files', 'expected'), + [ + pytest.param( + V2_MOUNTINFO, + V2_SELF_CGROUP.format(path='/'), + {'cpuset.cpus.effective': '0-1\n'}, + 2, + id='v2 range', + ), + pytest.param( + V2_MOUNTINFO, + V2_SELF_CGROUP.format(path='/'), + {'cpuset.cpus.effective': '0-1,4,6-7\n'}, + 5, + id='v2 ranges mixed with single cores', + ), + pytest.param( + V2_MOUNTINFO, + V2_SELF_CGROUP.format(path='/'), + {'cpuset.cpus.effective': '\n'}, + None, + id='inherited from the parent', + ), + pytest.param( + V1_MOUNTINFO, + V1_SELF_CGROUP.format(path='/'), + {'cpuset/cpuset.cpus': '0-3\n'}, + 4, + id='v1', + ), + ], +) +def test_read_cpu_set_size( + fake_cgroup: Callable[..., Path], + mountinfo: str, + self_cgroup: str, + files: dict[str, str], + expected: int | None, +) -> None: + """Counts the cores of a CPU set spelled as a mix of ranges and single numbers.""" + fake_cgroup(mountinfo=mountinfo, self_cgroup=self_cgroup, files=files) + + cpu_set = cgroup.read_cpu_set_size() + + assert (cpu_set.cores if cpu_set is not None else None) == expected + + +def test_read_cpu_set_size_unparsable_entry(fake_cgroup: Callable[..., Path]) -> None: + """Names the level when one entry of the list does not parse, rather than counting the rest. + + Every entry has to parse: counting `0-1` out of `0-1,nonsense` would report a set narrower than the one + the kernel enforces, which is a restriction a consumer would then size a pool from. + """ + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpuset.cpus.effective': '0-1,nonsense\n'}, + ) + + assert cgroup.read_cpu().unreadable_directory == root + + +@pytest.mark.parametrize( + ('mountinfo', 'self_cgroup', 'files'), + [ + pytest.param( + V2_MOUNTINFO, + V2_SELF_CGROUP.format(path='/'), + # cgroup v2 reports the consumed CPU time in microseconds, among other keys. + {'cpu.stat': 'usage_usec 2500000\nuser_usec 2000000\n'}, + id='v2', + ), + pytest.param( + V1_MOUNTINFO, + V1_SELF_CGROUP.format(path='/'), + # cgroup v1 reports it in nanoseconds, in a file of its own. + {'cpu,cpuacct/cpuacct.usage': '2500000000\n'}, + id='v1', + ), + ], +) +def test_read_cpu_usage( + fake_cgroup: Callable[..., Path], + mountinfo: str, + self_cgroup: str, + files: dict[str, str], +) -> None: + """Reports the consumed CPU time in seconds under both cgroup interfaces.""" + fake_cgroup(mountinfo=mountinfo, self_cgroup=self_cgroup, files=files) + + assert cgroup.read_cpu_usage() == 2.5 + + +def test_read_cpu_set_size_file_disappears(fake_cgroup: Callable[..., Path]) -> None: + """Reports nothing when the control file is gone by read time, because discovery outlives the files.""" + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpuset.cpus.effective': '0-1\n'}, + ) + cpu_set = cgroup.read_cpu_set_size() + + assert cpu_set is not None + assert cpu_set.cores == 2 + + (root / 'cpuset.cpus.effective').unlink() + + assert cgroup.read_cpu_set_size() is None + + +def test_read_cpu_set_size_unreadable_file(fake_cgroup: Callable[..., Path]) -> None: + """Names the level whose set cannot be read, instead of reporting no restriction. + + The file is located and read in two steps, and a cgroup can be removed between them. A directory in its + place reproduces that failure without a race. + """ + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpuset.cpus.effective': '0-1\n'}, + ) + (root / 'cpuset.cpus.effective').unlink() + (root / 'cpuset.cpus.effective').mkdir() + + assert cgroup.read_cpu() == cgroup.RawCpu(quota=None, cpu_set=None, unreadable_directory=root) + + +def test_locate_cpu_usage_skips_a_controller_less_unified_hierarchy(fake_cgroup: Callable[..., Path]) -> None: + """Counts the time where the limits are, not in a cgroup2 that carries no controllers. + + `cpu.stat` exists in every cgroup v2 group whether or not the controller is enabled, so a hybrid machine + offers a counter for a cgroup this process only nominally belongs to - here the whole docker service + rather than one container. + """ + root = fake_cgroup( + mountinfo=HYBRID_MOUNTINFO, + self_cgroup=HYBRID_SELF_CGROUP, + files={ + 'unified/cgroup.controllers': '\n', + 'unified/system.slice/docker.service/cpu.stat': 'usage_usec 777000000\n', + 'cpu,cpuacct/docker/abc/cpuacct.usage': '1000000000\n', + }, + ) + + controller = cgroup.locate_controllers().cpu_usage + + assert controller is not None + assert controller.is_v2 is False + assert controller.dirs[0] == root / 'cpu,cpuacct' / 'docker' / 'abc' + assert cgroup.read_cpu_usage() == 1.0 + + +def test_locate_cpu_usage_uses_the_unified_hierarchy_when_nothing_else_counts( + fake_cgroup: Callable[..., Path], +) -> None: + """Falls back to the base file when no accounting hierarchy exists, because it is then all there is.""" + fake_cgroup( + mountinfo=HYBRID_MOUNTINFO.replace(',cpuacct', ''), + self_cgroup=HYBRID_SELF_CGROUP.replace('3:cpu,cpuacct:', '3:cpu:'), + files={ + 'unified/cgroup.controllers': '\n', + 'unified/system.slice/docker.service/cpu.stat': 'usage_usec 777000000\n', + }, + ) + + controller = cgroup.locate_controllers().cpu_usage + + assert controller is not None + assert controller.is_v2 is True + assert cgroup.read_cpu_usage() == 777.0 + + +def test_locate_cpu_usage_keeps_a_unified_hierarchy_that_carries_controllers( + fake_cgroup: Callable[..., Path], +) -> None: + """Reads the unified hierarchy where it is the real one, which the controllers it lists say.""" + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cgroup.controllers': 'cpuset cpu memory pids\n', 'cpu.stat': 'usage_usec 2500000\n'}, + ) + + controller = cgroup.locate_controllers().cpu_usage + + assert controller is not None + assert controller.is_v2 is True + assert cgroup.read_cpu_usage() == 2.5 + + +def test_cpu_usage_dir_rejects_a_group_of_the_same_name(fake_cgroup: Callable[..., Path]) -> None: + """Refuses a counter that only shares a name with the level the quota binds at. + + The two hierarchies name different cgroups for this process here, which `cgclassify` and `cgrules.conf` + produce. `/limited` exists under the accounting hierarchy too, and belongs to somebody else - reading it + would report their CPU time as this process's, and a rate of a busy stranger reads as saturation. + """ + root = fake_cgroup( + mountinfo=V1_CPU_SPLIT_MOUNTINFO, + self_cgroup='3:cpu:/limited\n4:cpuacct:/ours\n', + files={ + 'cpu/limited/cpu.cfs_quota_us': '200000\n', + 'cpu/limited/cpu.cfs_period_us': '100000\n', + # A group of the same name, in the hierarchy this process is not in. + 'cpuacct/limited/cpuacct.usage': '999999999999\n', + 'cpuacct/ours/cpuacct.usage': '111\n', + }, + ) + + quota = cgroup.read_cpu_quota() + + assert quota is not None + assert quota.cores == 2.0 + assert quota.limit_directory == root / 'cpu' / 'limited' + assert quota.usage_directory is None + + +def test_read_memory_ancestor_without_usage_drops_the_distance(fake_cgroup: Callable[..., Path]) -> None: + """Drops the working set when any level's usage is missing, not only the tightest level's. + + Memory is charged up the chain, so the level closest to its own limit can be any of them. An ancestor + whose usage cannot be read leaves one distance unknown, and an unknown may be the smallest one. + """ + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/pod/container'), + files={ + 'pod/container/memory.max': '1000\n', + 'pod/container/memory.current': '100\n', + 'pod/container/memory.stat': 'inactive_file 0\n', + # The tighter limit, with nothing saying how close the pod is to it. + 'pod/memory.max': '500\n', + }, + ) + + assert cgroup.read_memory() == cgroup.RawMemory( + limit=500, + working_set=None, + limit_directory=root / 'pod', + unreadable_directory=None, + ) + + +def test_read_memory_unreadable_limit(fake_cgroup: Callable[..., Path]) -> None: + """Names the level whose limit cannot be read, instead of answering with a looser ancestor. + + An emulated cgroupfs, a denied delegation or a truncated file all produce this. The ancestor's ten + gigabytes would be handed to a consumer whose real ceiling is half a gigabyte. + """ + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/pod/container'), + files={ + 'pod/memory.max': '10737418240\n', + 'pod/memory.current': '100\n', + 'pod/memory.stat': 'inactive_file 0\n', + 'pod/container/memory.max': 'not a number\n', + 'pod/container/memory.current': '100\n', + 'pod/container/memory.stat': 'inactive_file 0\n', + }, + ) + + assert cgroup.read_memory() == cgroup.RawMemory( + limit=None, + working_set=None, + limit_directory=None, + unreadable_directory=root / 'pod' / 'container', + ) + + +def test_read_memory_looser_level_without_usage(fake_cgroup: Callable[..., Path]) -> None: + """Drops the working set for a missing usage on a level that is not the tightest one. + + That level is looser, so it holds more memory before its own limit - but it also counts what its other + children use, and it can still be the closest of all of them to its ceiling. + """ + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/pod/container'), + files={ + # The tightest limit, and it says how close it is. + 'pod/container/memory.max': '500\n', + 'pod/container/memory.current': '100\n', + 'pod/container/memory.stat': 'inactive_file 0\n', + # Looser, and silent about its own usage. + 'pod/memory.max': '1000\n', + }, + ) + + assert cgroup.read_memory() == cgroup.RawMemory( + limit=500, + working_set=None, + limit_directory=root / 'pod' / 'container', + unreadable_directory=None, + ) + + +def test_read_memory_limit_file_cannot_be_opened(fake_cgroup: Callable[..., Path]) -> None: + """Names a level whose limit file is there but cannot be read at all, as for one that holds nonsense. + + A denied delegation and a racing runtime produce this rather than a bad number. A directory in place of + the file reproduces it without needing either. + """ + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/pod/container'), + files={ + 'pod/memory.max': '10737418240\n', + 'pod/memory.current': '100\n', + 'pod/memory.stat': 'inactive_file 0\n', + 'pod/container/memory.current': '100\n', + }, + ) + (root / 'pod' / 'container' / 'memory.max').mkdir() + + assert cgroup.read_memory().unreadable_directory == root / 'pod' / 'container' + + +def test_read_memory_negative_limit(fake_cgroup: Callable[..., Path]) -> None: + """Names a level whose limit is a negative number of bytes, which is no limit and no absence either.""" + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/pod/container'), + files={ + 'pod/memory.max': '10737418240\n', + 'pod/memory.current': '100\n', + 'pod/memory.stat': 'inactive_file 0\n', + 'pod/container/memory.max': '-5\n', + }, + ) + + assert cgroup.read_memory().unreadable_directory == root / 'pod' / 'container' + + +def test_read_cpu_quota_unreadable_level(fake_cgroup: Callable[..., Path]) -> None: + """Names the level whose quota cannot be read, instead of answering with a looser ancestor.""" + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/pod/container'), + files={ + 'pod/cpu.max': '800000 100000\n', + 'pod/cpu.stat': 'usage_usec 0\n', + 'pod/container/cpu.max': 'nonsense\n', + 'pod/container/cpu.stat': 'usage_usec 0\n', + }, + ) + + assert cgroup.read_cpu() == cgroup.RawCpu( + quota=None, + cpu_set=None, + unreadable_directory=root / 'pod' / 'container', + ) + + +def test_read_cpu_set_size_inherited_under_v1(fake_cgroup: Callable[..., Path]) -> None: + """Reads the set a cgroup v1 level inherits, which its own file spells as empty. + + `cpuset.cpus` holds what was configured on that level, and an empty file means "whatever the parent + allows". Reading only that would lose the restriction entirely. + """ + root = fake_cgroup( + mountinfo=V1_CPUSET_MOUNTINFO, + self_cgroup='4:cpuset:/child\n', + files={ + 'cpuset/cpuset.cpus': '0-1\n', + 'cpuset/child/cpuset.cpus': '\n', + 'cpuset/child/cpuset.effective_cpus': '0-1\n', + }, + ) + + cpu_set = cgroup.read_cpu_set_size() + + assert cpu_set is not None + assert cpu_set.cores == 2 + assert cpu_set.limit_directory == root / 'cpuset' / 'child' + + +def test_read_cpu_set_size_without_a_counter_of_its_own(fake_cgroup: Callable[..., Path]) -> None: + """Reports no counter when the accounting hierarchy does not carry the level the set applies to. + + `cgexec -g cpuset:limited` moves the process in the cpuset hierarchy only, so its cgroup in the accounting + hierarchy is the root - whose counter is the CPU time of the whole machine. + """ + fake_cgroup( + mountinfo=V1_CPUSET_SPLIT_MOUNTINFO, + self_cgroup='4:cpuset:/limited\n5:cpuacct:/\n', + files={'cpuset/limited/cpuset.cpus': '0-1\n', 'cpuacct/cpuacct.usage': '5000000000\n'}, + ) + + cpu_set = cgroup.read_cpu_set_size() + + assert cpu_set is not None + assert cpu_set.cores == 2 + assert cpu_set.usage_directory is None + + +def test_read_cpu_set_size_from_an_ancestor(fake_cgroup: Callable[..., Path]) -> None: + """Reads the set of an ancestor when the own cgroup carries none, because it applies there too. + + A cgroup gets `cpuset.cpus.effective` only once its parent enables the controller for it, and a set on the + level above restricts everything below it either way. + """ + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/pod/container'), + files={ + 'pod/container/cpu.stat': 'usage_usec 0\n', + 'pod/cpuset.cpus.effective': '0-1\n', + 'pod/cpu.stat': 'usage_usec 0\n', + }, + ) + + cpu_set = cgroup.read_cpu_set_size() + + assert cpu_set is not None + assert cpu_set.cores == 2 + # The set restricts the pod, so that is the level it is read at and the one whose time it is measured + # against. + assert cpu_set.limit_directory == root / 'pod' + assert cpu_set.usage_directory == root / 'pod' + + +def test_read_cpu_set_size_without_any_accounting_hierarchy(fake_cgroup: Callable[..., Path]) -> None: + """Reports no counter where nothing on the machine counts CPU time at all.""" + fake_cgroup( + mountinfo=V1_CPUSET_MOUNTINFO, + self_cgroup='4:cpuset:/limited\n', + files={'cpuset/limited/cpuset.cpus': '0-1\n'}, + ) + + cpu_set = cgroup.read_cpu_set_size() + + assert cpu_set is not None + assert cpu_set.cores == 2 + assert cpu_set.usage_directory is None + + +def test_read_cpu_set_size_across_split_hierarchies(fake_cgroup: Callable[..., Path]) -> None: + """Finds the counter of the level the set applies to, even in another hierarchy.""" + root = fake_cgroup( + mountinfo=V1_CPUSET_SPLIT_MOUNTINFO, + self_cgroup='4:cpuset:/limited\n5:cpuacct:/limited\n', + files={ + 'cpuset/limited/cpuset.cpus': '0-1\n', + 'cpuacct/limited/cpuacct.usage': '5000000000\n', + }, + ) + + cpu_set = cgroup.read_cpu_set_size() + + assert cpu_set is not None + assert cpu_set.limit_directory == root / 'cpuset' / 'limited' + assert cpu_set.usage_directory == root / 'cpuacct' / 'limited' + + +def test_locate_controllers_hybrid(fake_cgroup: Callable[..., Path]) -> None: + """Falls back to cgroup v1 for a controller the unified hierarchy does not carry.""" + root = fake_cgroup( + mountinfo=f'{V2_MOUNTINFO}\n{V1_MOUNTINFO}', + self_cgroup=f'{V2_SELF_CGROUP.format(path="/")}2:memory:/\n', + files={ + 'memory/memory.limit_in_bytes': '536870912\n', + 'memory/memory.usage_in_bytes': '1000\n', + 'memory/memory.stat': 'total_inactive_file 400\n', + }, + ) + + memory = cgroup.locate_controllers().memory + + assert memory is not None + assert memory.is_v2 is False + assert cgroup.read_memory() == cgroup.RawMemory( + limit=536870912, + working_set=600, + limit_directory=root / 'memory', + unreadable_directory=None, + ) + + +@pytest.mark.usefixtures('_no_cgroup') +def test_no_cgroups() -> None: + """Reports nothing on a system that has no cgroups.""" + assert cgroup.read_memory() == cgroup.RawMemory( + limit=None, + working_set=None, + limit_directory=None, + unreadable_directory=None, + ) + assert cgroup.read_cpu_quota() is None + assert cgroup.read_cpu_set_size() is None + assert cgroup.read_cpu_usage() is None + + +def raise_permission_error(_self: Path) -> bool: + """Stand in for `Path.exists` on a directory the kernel refuses to traverse.""" + raise PermissionError(13, 'Permission denied') + + +def test_exists_on_an_unreadable_directory(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Reports a path as missing when the lookup is denied. Only Python 3.14 does that on its own.""" + monkeypatch.setattr(Path, 'exists', raise_permission_error) + + assert cgroup._exists(tmp_path) is False + + +def test_locate_controllers_with_unreadable_directories( + fake_cgroup: Callable[..., Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Finds nothing rather than raising when the cgroup chain holds a directory it may not traverse.""" + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'memory.max': '1000\n', 'memory.current': '100\n', 'memory.stat': 'inactive_file 0\n'}, + ) + monkeypatch.setattr(Path, 'exists', raise_permission_error) + + assert cgroup.locate_controllers().memory is None + assert cgroup.read_memory() == cgroup.RawMemory( + limit=None, + working_set=None, + limit_directory=None, + unreadable_directory=None, + ) + + +def test_read_hierarchies_undecodable_path(fake_cgroup: Callable[..., Path], tmp_path: Path) -> None: + """Keeps reading when another mount point is not valid UTF-8, instead of losing every hierarchy.""" + root = fake_cgroup(mountinfo=V2_MOUNTINFO, self_cgroup=V2_SELF_CGROUP.format(path='/'), files={}) + mountinfo = tmp_path / 'mountinfo' + mountinfo.write_bytes(b'24 30 0:21 / /mnt/\xff\xfe rw - ext4 /dev/sda1 rw\n' + mountinfo.read_bytes()) + + unified, _controllers = cgroup._read_hierarchies() + + assert unified is not None + assert unified.mount_point == root + + +def test_read_cpu_quota_tightest_level(fake_cgroup: Callable[..., Path]) -> None: + """Takes the tightest quota when two levels hold different ones, and says where it binds.""" + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/pod/container'), + files={ + 'pod/container/cpu.max': '400000 100000\n', + 'pod/container/cpu.stat': 'usage_usec 0\n', + 'pod/cpu.max': '150000 100000\n', + 'pod/cpu.stat': 'usage_usec 0\n', + }, + ) + + # The quota binds on the pod, so a rate has to be measured there rather than in the container. + assert cgroup.read_cpu_quota() == cgroup.RawCpuQuota( + cores=1.5, + limit_directory=root / 'pod', + usage_directory=root / 'pod', + ) + + +def test_read_cpu_quota_on_the_own_group(fake_cgroup: Callable[..., Path]) -> None: + """Names the group of this process when the quota binds there.""" + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/pod/container'), + files={ + 'pod/container/cpu.max': '150000 100000\n', + 'pod/container/cpu.stat': 'usage_usec 0\n', + 'pod/cpu.max': '400000 100000\n', + 'pod/cpu.stat': 'usage_usec 0\n', + }, + ) + + assert cgroup.read_cpu_quota() == cgroup.RawCpuQuota( + cores=1.5, + limit_directory=root / 'pod' / 'container', + usage_directory=root / 'pod' / 'container', + ) + + +def test_read_cpu_usage_at_a_named_level(fake_cgroup: Callable[..., Path]) -> None: + """Reads the counter of the level it is asked for, which counts the siblings of this process too.""" + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/pod/container'), + files={ + 'pod/container/cpu.stat': 'usage_usec 2500000\n', + 'pod/cpu.stat': 'usage_usec 90000000\n', + }, + ) + + assert cgroup.read_cpu_usage(root / 'pod') == 90.0 + + +def test_read_cpu_set_size_reads_the_closest_level(fake_cgroup: Callable[..., Path]) -> None: + """Reads the set of the own cgroup, not of an ancestor that allows more. + + A set restricts the group of this process, so its time is the time of that group, counted at the closest + level that counts any. + """ + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/pod/container'), + files={ + 'pod/container/cpuset.cpus.effective': '0-1\n', + 'pod/container/cpu.stat': 'usage_usec 0\n', + 'pod/cpuset.cpus.effective': '0-7\n', + 'pod/cpu.stat': 'usage_usec 0\n', + 'cpuset.cpus.effective': '0-15\n', + }, + ) + + cpu_set = cgroup.read_cpu_set_size() + + assert cpu_set is not None + assert cpu_set.cores == 2 + assert cpu_set.usage_directory == root / 'pod' / 'container' + + +def test_read_cpu_usage_reads_the_closest_level(fake_cgroup: Callable[..., Path]) -> None: + """Reads the counter of the own cgroup, not the one of an ancestor that counts the siblings too.""" + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/pod/container'), + files={ + 'pod/container/cpu.stat': 'usage_usec 2500000\n', + 'pod/cpu.stat': 'usage_usec 90000000\n', + 'cpu.stat': 'usage_usec 900000000\n', + }, + ) + + assert cgroup.read_cpu_usage() == 2.5 + + +def test_read_memory_limit_of_zero(fake_cgroup: Callable[..., Path]) -> None: + """Takes a limit of zero as the tightest limit there is, not as an absent one. + + A cgroup can be given no memory at all. Reading that as "unrestricted" would hand a consumer the machine. + """ + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/pod/container'), + files={ + 'pod/container/memory.max': '0\n', + 'pod/container/memory.current': '100\n', + 'pod/container/memory.stat': 'inactive_file 0\n', + 'pod/memory.max': '1000\n', + 'pod/memory.current': '200\n', + 'pod/memory.stat': 'inactive_file 0\n', + }, + ) + + assert cgroup.read_memory() == cgroup.RawMemory( + limit=0, + working_set=0, + limit_directory=root / 'pod' / 'container', + unreadable_directory=None, + ) + + +def test_read_memory_usage_below_the_cache(fake_cgroup: Callable[..., Path]) -> None: + """Floors the charged memory at zero when the reclaimable cache reads larger than the usage. + + The two are separate readings and can disagree, and a negative amount of charged memory is meaningless. + The floor sits in the level reading, because that is where the claim "memory charged" is made. + """ + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'memory.max': '1000\n', 'memory.current': '100\n', 'memory.stat': 'inactive_file 400\n'}, + ) + controller = cgroup.locate_controllers().memory + + assert controller is not None + assert cgroup._read_working_set(controller, root) == 0 + assert cgroup.read_memory() == cgroup.RawMemory( + limit=1000, + working_set=0, + limit_directory=root, + unreadable_directory=None, + ) + + +def test_read_cpu_quota_v1_zero_period(fake_cgroup: Callable[..., Path]) -> None: + """Names the level rather than dividing by a period of zero, which no kernel writes.""" + root = fake_cgroup( + mountinfo=V1_MOUNTINFO, + self_cgroup=V1_SELF_CGROUP.format(path='/'), + files={ + 'cpu,cpuacct/cpu.cfs_quota_us': '100000\n', + 'cpu,cpuacct/cpu.cfs_period_us': '0\n', + 'cpu,cpuacct/cpuacct.usage': '0\n', + }, + ) + + assert cgroup.read_cpu().unreadable_directory == root / 'cpu,cpuacct' + + +def test_read_hierarchies_first_v1_mount_wins(fake_cgroup: Callable[..., Path]) -> None: + """Keeps the first mount of a cgroup v1 controller, because mounts are listed in creation order.""" + mountinfo = ( + '30 25 0:26 / {root}/first rw,nosuid shared:14 - cgroup cgroup rw,memory\n' + '31 25 0:27 / {root}/second rw,nosuid shared:15 - cgroup cgroup rw,memory' + ) + root = fake_cgroup( + mountinfo=mountinfo, + self_cgroup=V1_SELF_CGROUP.format(path='/'), + files={'first/cgroup.procs': '', 'second/cgroup.procs': ''}, + ) + + _unified, controllers = cgroup._read_hierarchies() + + assert controllers['memory'].mount_point == root / 'first' + + +def test_read_hierarchies_skips_a_foreign_v1_mount(fake_cgroup: Callable[..., Path]) -> None: + """Skips a cgroup v1 mount of another subtree, as the unified hierarchy does. + + Nothing stops a controller from being mounted twice, and the first line of the mount table is not + necessarily the mount this process lives in. + """ + mountinfo = ( + '30 25 0:26 /other {root}/foreign rw,nosuid shared:14 - cgroup cgroup rw,memory\n' + '31 25 0:27 / {root}/ours rw,nosuid shared:15 - cgroup cgroup rw,memory' + ) + root = fake_cgroup( + mountinfo=mountinfo, + self_cgroup=V1_SELF_CGROUP.format(path='/mine'), + files={ + 'foreign/memory.limit_in_bytes': '1000\n', + 'foreign/memory.usage_in_bytes': '900\n', + 'foreign/memory.stat': 'total_inactive_file 0\n', + 'ours/mine/memory.limit_in_bytes': '2000\n', + 'ours/mine/memory.usage_in_bytes': '100\n', + 'ours/mine/memory.stat': 'total_inactive_file 0\n', + }, + ) + + _unified, controllers = cgroup._read_hierarchies() + + assert controllers['memory'].mount_point == root / 'ours' + assert cgroup.read_memory() == cgroup.RawMemory( + limit=2000, + working_set=100, + limit_directory=root / 'ours' / 'mine', + unreadable_directory=None, + ) + + +def test_read_memory_v1_uses_the_hierarchical_key(fake_cgroup: Callable[..., Path]) -> None: + """Reads `total_inactive_file` under cgroup v1, which counts the children as the limit does. + + A real v1 kernel reports both keys, and the per-cgroup one alone would understate the cache. + """ + root = fake_cgroup( + mountinfo=V1_MOUNTINFO, + self_cgroup=V1_SELF_CGROUP.format(path='/'), + files={ + 'memory/memory.limit_in_bytes': '1000\n', + 'memory/memory.usage_in_bytes': '900\n', + 'memory/memory.stat': 'rss 500\ninactive_file 100\ntotal_inactive_file 400\n', + }, + ) + + assert cgroup.read_memory() == cgroup.RawMemory( + limit=1000, + working_set=500, + limit_directory=root / 'memory', + unreadable_directory=None, + ) + + +def test_read_cpu_quota_across_split_hierarchies(fake_cgroup: Callable[..., Path]) -> None: + """Finds the counter of the level the quota binds at, even in another hierarchy. + + Under cgroup v1 `cpu` and `cpuacct` are separate controllers, and a system can mount them apart. The level + is then the same cgroup under two mount points. + """ + root = fake_cgroup( + mountinfo=V1_CPU_SPLIT_MOUNTINFO, + self_cgroup='3:cpu:/slice/own\n4:cpuacct:/slice/own\n', + files={ + 'cpu/slice/cpu.cfs_quota_us': '50000\n', + 'cpu/slice/cpu.cfs_period_us': '100000\n', + 'cpu/slice/own/cpu.cfs_quota_us': '-1\n', + 'cpu/slice/own/cpu.cfs_period_us': '100000\n', + 'cpuacct/slice/cpuacct.usage': '7000000000\n', + 'cpuacct/slice/own/cpuacct.usage': '1000000\n', + }, + ) + + quota = cgroup.read_cpu_quota() + + assert quota is not None + assert quota.cores == 0.5 + assert quota.limit_directory == root / 'cpu' / 'slice' + # The counter of that level lives under the other mount point. + assert quota.usage_directory == root / 'cpuacct' / 'slice' + assert cgroup.read_cpu_usage(quota.usage_directory) == 7.0 + + +def test_read_cpu_quota_without_accounting(fake_cgroup: Callable[..., Path]) -> None: + """Reports no counter when the level the quota binds at counts no CPU time anywhere.""" + fake_cgroup( + mountinfo=V1_CPU_SPLIT_MOUNTINFO, + self_cgroup='3:cpu:/slice/own\n4:cpuacct:/other\n', + files={ + 'cpu/slice/own/cgroup.procs': '', + 'cpu/slice/cpu.cfs_quota_us': '50000\n', + 'cpu/slice/cpu.cfs_period_us': '100000\n', + 'cpuacct/other/cpuacct.usage': '1000000\n', + }, + ) + + quota = cgroup.read_cpu_quota() + + assert quota is not None + assert quota.cores == 0.5 + assert quota.usage_directory is None + + +@pytest.mark.parametrize( + 'cpu_max', + [ + pytest.param('0 100000\n', id='no quota at all'), + pytest.param('-1 100000\n', id='a negative quota'), + pytest.param('100000 0\n', id='a period of zero'), + ], +) +def test_read_cpu_quota_v2_degenerate(fake_cgroup: Callable[..., Path], cpu_max: str) -> None: + """Names the level for a cgroup v2 quota that describes no bandwidth. + + This interface spells "no quota" as `max`, so anything here that is not bandwidth is a file this module + cannot use - not an absent limit. cgroup v1, which has no such word, reads a negative quota as unlimited. + """ + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpu.max': cpu_max, 'cpu.stat': 'usage_usec 0\n'}, + ) + + assert cgroup.read_cpu().unreadable_directory == root + + +def test_read_cpu_quota_appearing_after_discovery(fake_cgroup: Callable[..., Path]) -> None: + """Sees a quota written to a level that carried none when the chain was discovered. + + `systemctl set-property` creates the control files of a level at runtime. Discovery happens once per + process, so a chain cut down to the levels that carried the controller at startup would hide such a limit + for as long as the process lives. + """ + # A systemd scope carries no `cpu.max` until it is given a quota, while the slice above it has one. + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/slice/own'), + files={ + 'slice/own/cpu.stat': 'usage_usec 0\n', + 'slice/cpu.max': 'max 100000\n', + 'slice/cpu.stat': 'usage_usec 0\n', + }, + ) + + assert cgroup.read_cpu_quota() is None + + # The scope of this process is given a quota of its own, below where the chain would have been cut. + (root / 'slice' / 'own' / 'cpu.max').write_text('50000 100000\n') + + quota = cgroup.read_cpu_quota() + + assert quota is not None + assert quota.cores == 0.5 + assert quota.limit_directory == root / 'slice' / 'own' diff --git a/tests/unit/test_cpu_list.py b/tests/unit/test_cpu_list.py new file mode 100644 index 0000000..0038ff4 --- /dev/null +++ b/tests/unit/test_cpu_list.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +import pytest + +from cgroups_sensor._cpu_list import count_cpu_list + + +@pytest.mark.parametrize( + ('cpu_list', 'expected'), + [ + pytest.param('0-3', 4, id='a range'), + pytest.param('0-1,4,6-7', 5, id='ranges mixed with single cores'), + pytest.param('7', 1, id='a single core'), + pytest.param('5-4', None, id='a reversed range counts nothing, so it is not a count'), + pytest.param('3-1', None, id='a range reversed by more than one'), + pytest.param('0-,2', None, id='an unfinished range'), + pytest.param('nonsense', None, id='not a number'), + ], +) +def test_count_cpu_list(cpu_list: str, expected: int | None) -> None: + """Counts a CPU list, and refuses one that does not describe a set of cores. + + A real kernel writes neither of the refused shapes. An emulated cgroupfs can, and a count of zero would + become a limit of no cores that every consumer then divides by. + """ + assert count_cpu_list(cpu_list) == expected diff --git a/tests/unit/test_sensor.py b/tests/unit/test_sensor.py new file mode 100644 index 0000000..6eecca0 --- /dev/null +++ b/tests/unit/test_sensor.py @@ -0,0 +1,1217 @@ +from __future__ import annotations + +import asyncio +import json +import os +import threading +from itertools import count +from pathlib import Path +from types import SimpleNamespace +from typing import TYPE_CHECKING + +import pytest + +import cgroups_sensor +from cgroups_sensor import _cgroup, _sensor + +from .conftest import V1_CPU_SPLIT_MOUNTINFO, V1_MOUNTINFO, V1_SELF_CGROUP, V2_MOUNTINFO, V2_SELF_CGROUP + +if TYPE_CHECKING: + from collections.abc import Callable + +MACHINE_TOTAL_BYTES = 8 * 1024**3 +MACHINE_CORES = 8 + +# The autouse fixture below replaces these module attributes, so the tests of the real implementations go +# through references captured before any fixture runs. +real_machine_memory_bytes = _sensor.get_machine_memory_bytes +real_machine_cpu_count = _sensor.get_machine_cpu_count + + +@pytest.fixture(autouse=True) +def _fixed_machine(monkeypatch: pytest.MonkeyPatch) -> None: + """Pin the machine facts the filters compare against, so the expected values do not move with the machine.""" + monkeypatch.setattr(_sensor, 'get_machine_memory_bytes', lambda: MACHINE_TOTAL_BYTES) + monkeypatch.setattr(_sensor, 'get_machine_cpu_count', lambda: MACHINE_CORES) + + +def fake_time(monkeypatch: pytest.MonkeyPatch, *, sleep: Callable[[float], object]) -> None: + """Advance the clock the sensor reads by one second per reading, running `sleep` in place of waiting.""" + clock = count(start=100.0, step=1.0) + monkeypatch.setattr(_sensor, 'time', SimpleNamespace(monotonic=lambda: next(clock), sleep=sleep)) + + +NOTICE_PREFIXES = {'memory': ('memory', 'machine-memory'), 'cpu': ('cpu',)} +"""How a notice code names the metric it is about. `machine-memory-unknown` is a memory notice.""" + + +def notice_codes(metric: str | None = None) -> tuple[str, ...]: + """The notices of the current description, or only those about one metric. + + A fixture that lays out one metric leaves the other without a mechanism, which is itself a notice. Tests + of one metric therefore ask for that metric. + """ + codes = tuple(str(notice.code) for notice in cgroups_sensor.describe().notices) + if metric is None: + return codes + + return tuple(code for code in codes if code.startswith(NOTICE_PREFIXES[metric])) + + +def test_get_memory_budget_restricted(fake_cgroup: Callable[..., Path]) -> None: + """Reports a limit below the memory of the machine, with the working set measured against it.""" + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={ + 'memory.max': '536870912\n', + 'memory.current': '150000000\n', + 'memory.stat': 'inactive_file 50000000\n', + }, + ) + + assert cgroups_sensor.get_memory_budget() == cgroups_sensor.MemoryBudget(limit=536870912, working_set=100000000) + assert notice_codes('memory') == () + + +def test_get_memory_budget_covers_machine(fake_cgroup: Callable[..., Path]) -> None: + """Drops a limit at or above the memory of the machine, which is how an unrestricted group spells it.""" + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={ + 'memory.max': f'{MACHINE_TOTAL_BYTES * 2}\n', + 'memory.current': '150000000\n', + 'memory.stat': 'inactive_file 50000000\n', + }, + ) + + assert cgroups_sensor.get_memory_budget() is None + assert notice_codes('memory') == ('memory-limit-covers-machine',) + + +def test_get_memory_budget_v1_sentinel(fake_cgroup: Callable[..., Path]) -> None: + """Drops the sentinel a cgroup v1 hierarchy spells an absent limit as.""" + fake_cgroup( + mountinfo=V1_MOUNTINFO, + self_cgroup=V1_SELF_CGROUP.format(path='/'), + files={ + 'memory/memory.limit_in_bytes': '9223372036854771712\n', + 'memory/memory.usage_in_bytes': '1000\n', + 'memory/memory.stat': 'total_inactive_file 400\n', + }, + ) + + assert cgroups_sensor.get_memory_budget() is None + assert notice_codes('memory') == ('memory-limit-covers-machine',) + # The raw sentinel stays visible next to the machine memory it lost to. + description = cgroups_sensor.describe() + assert description.raw_memory_limit == 9223372036854771712 + assert description.raw_memory_working_set == 600 + + +def test_get_memory_budget_no_working_set(fake_cgroup: Callable[..., Path]) -> None: + """Drops a limit that no usage can be paired with, and says why instead of pairing it with the raw usage.""" + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'memory.max': '536870912\n', 'memory.current': '1000\n', 'memory.stat': 'anon 600\n'}, + ) + + assert cgroups_sensor.get_memory_budget() is None + assert notice_codes('memory') == ('memory-usage-unavailable',) + # The pair that explains the rejection stays visible: a limit next to no usable usage. + description = cgroups_sensor.describe() + assert description.raw_memory_limit == 536870912 + assert description.raw_memory_working_set is None + + +def test_get_memory_budget_fake_limit_without_usage(fake_cgroup: Callable[..., Path]) -> None: + """Does not complain about a missing usage next to a limit that restricts nothing anyway.""" + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'memory.max': f'{MACHINE_TOTAL_BYTES * 2}\n', 'memory.current': '1000\n', 'memory.stat': 'anon 600\n'}, + ) + + assert cgroups_sensor.get_memory_budget() is None + assert notice_codes('memory') == ('memory-limit-covers-machine',) + + +@pytest.mark.usefixtures('_no_cgroup') +def test_get_memory_budget_no_mechanism() -> None: + """Reports nothing when no mechanism carries a limit, and says that is what happened. + + "Nothing limits you here" and "nothing here can tell" are different answers, and only one of them is a + fact about the machine. + """ + assert cgroups_sensor.get_memory_budget() is None + assert notice_codes() == ('memory-metrics-unavailable', 'cpu-metrics-unavailable') + + +def test_get_memory_budget_exactly_the_machine(fake_cgroup: Callable[..., Path]) -> None: + """Drops a limit exactly equal to the memory of the machine, which restricts nothing either.""" + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={ + 'memory.max': f'{MACHINE_TOTAL_BYTES}\n', + 'memory.current': '150000000\n', + 'memory.stat': 'inactive_file 50000000\n', + }, + ) + + assert cgroups_sensor.get_memory_budget() is None + assert notice_codes('memory') == ('memory-limit-covers-machine',) + + +@pytest.mark.parametrize( + 'limit', + [ + pytest.param(536870912, id='a plausible limit'), + pytest.param(9223372036854771712, id='the v1 unlimited sentinel'), + ], +) +def test_get_memory_budget_unknown_machine_memory( + fake_cgroup: Callable[..., Path], + monkeypatch: pytest.MonkeyPatch, + limit: int, +) -> None: + """Drops any limit when the memory of the machine is unknown, because the sentinel cannot be told apart.""" + fake_cgroup( + mountinfo=V1_MOUNTINFO, + self_cgroup=V1_SELF_CGROUP.format(path='/'), + files={ + 'memory/memory.limit_in_bytes': f'{limit}\n', + 'memory/memory.usage_in_bytes': '1000\n', + 'memory/memory.stat': 'total_inactive_file 400\n', + }, + ) + monkeypatch.setattr(_sensor, 'get_machine_memory_bytes', lambda: None) + + assert cgroups_sensor.get_memory_budget() is None + assert notice_codes('memory') == ('machine-memory-unknown',) + # The raw reading stays visible, so the rejection is attributable. + assert cgroups_sensor.describe().raw_memory_limit == limit + + +@pytest.mark.parametrize( + ('quota', 'cpu_set_cores', 'machine_cores', 'expected'), + [ + pytest.param(None, None, 8, None, id='nothing restricts the cpu'), + pytest.param(2.0, None, 8, 2.0, id='bandwidth quota only'), + pytest.param(None, 2, 8, 2.0, id='cpu set only'), + pytest.param(None, 8, 8, None, id='a cpu set covering every core is not a restriction'), + pytest.param(8.0, None, 8, None, id='a quota covering every core is not a restriction'), + pytest.param(10.0, None, 8, None, id='a quota above the cores of the machine is not a restriction'), + pytest.param(10.0, 2, 8, 2.0, id='a quota above the machine leaves the cpu set to bind'), + pytest.param(4.0, 2, 8, 2.0, id='cpu set is tighter than the quota'), + pytest.param(1.0, 2, 8, 1.0, id='quota is tighter than the cpu set'), + pytest.param(2.0, None, None, 2.0, id='a quota counts when the cores of the machine are unknown'), + pytest.param(None, 2, None, 2.0, id='a cpu set counts when the cores of the machine are unknown'), + ], +) +def test_get_cpu_limit( + monkeypatch: pytest.MonkeyPatch, + quota: float | None, + cpu_set_cores: int | None, + machine_cores: int | None, + expected: float | None, +) -> None: + """Takes the tighter of the bandwidth quota and the CPU set, which restrict the CPU independently.""" + counter = Path('/sys/fs/cgroup') + raw_quota = ( + None if quota is None else _cgroup.RawCpuQuota(cores=quota, limit_directory=counter, usage_directory=counter) + ) + raw_set = ( + None + if cpu_set_cores is None + else _cgroup.RawCpuSet(cores=cpu_set_cores, limit_directory=counter, usage_directory=counter) + ) + monkeypatch.setattr(_cgroup, 'read_cpu_quota', lambda: raw_quota) + monkeypatch.setattr(_cgroup, 'read_cpu_set_size', lambda: raw_set) + monkeypatch.setattr(_sensor, 'get_machine_cpu_count', lambda: machine_cores) + + assert cgroups_sensor.get_cpu_limit() == expected + + +@pytest.mark.usefixtures('_no_cgroup') +def test_get_cpu_limit_notices(monkeypatch: pytest.MonkeyPatch) -> None: + """Explains each reading that covers the machine, so a missing limit is attributable.""" + counter = Path('/sys/fs/cgroup') + quota = _cgroup.RawCpuQuota(cores=10.0, limit_directory=counter, usage_directory=counter) + monkeypatch.setattr(_cgroup, 'read_cpu_quota', lambda: quota) + cpu_set = _cgroup.RawCpuSet(cores=8, limit_directory=counter, usage_directory=counter) + monkeypatch.setattr(_cgroup, 'read_cpu_set_size', lambda: cpu_set) + + assert cgroups_sensor.get_cpu_limit() is None + assert notice_codes('cpu') == ('cpu-quota-covers-machine', 'cpu-set-covers-machine') + + +def test_get_cpu_used_ratio(fake_cgroup: Callable[..., Path], monkeypatch: pytest.MonkeyPatch) -> None: + """Measures the consumed CPU time across the interval, against the cores the process may use.""" + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpu.max': '200000 100000\n', 'cpu.stat': 'usage_usec 0\n'}, + ) + # The counter advances by one core-second during the measurement interval. + fake_time(monkeypatch, sleep=lambda _seconds: (root / 'cpu.stat').write_text('usage_usec 1000000\n')) + + # One core-second over one second of wall time, out of the two cores the quota allows. + assert cgroups_sensor.get_cpu_used_ratio() == pytest.approx(0.5) + + +def test_get_cpu_used_ratio_counter_restart( + fake_cgroup: Callable[..., Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Clamps the counter that restarts when the process is moved to another group.""" + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpu.max': '200000 100000\n', 'cpu.stat': 'usage_usec 5000000\n'}, + ) + fake_time(monkeypatch, sleep=lambda _seconds: (root / 'cpu.stat').write_text('usage_usec 0\n')) + + assert cgroups_sensor.get_cpu_used_ratio() == 0.0 + + +def test_get_cpu_used_ratio_no_limit(fake_cgroup: Callable[..., Path]) -> None: + """Reports nothing when the CPU of this process is unrestricted - measuring the machine is not its job.""" + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpu.max': 'max 100000\n', 'cpu.stat': 'usage_usec 0\n'}, + ) + + assert cgroups_sensor.get_cpu_used_ratio() is None + + +def test_get_cpu_used_ratio_no_counter(fake_cgroup: Callable[..., Path]) -> None: + """Reports nothing when a limit is found but no consumed CPU time can be measured against it.""" + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpu.max': '200000 100000\n'}, + ) + + assert cgroups_sensor.get_cpu_used_ratio() is None + + +def test_get_cpu_used_ratio_second_reading_fails( + fake_cgroup: Callable[..., Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Reports nothing when the counter disappears mid-measurement instead of comparing across the gap.""" + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpu.max': '200000 100000\n', 'cpu.stat': 'usage_usec 0\n'}, + ) + fake_time(monkeypatch, sleep=lambda _seconds: (root / 'cpu.stat').unlink()) + + assert cgroups_sensor.get_cpu_used_ratio() is None + + +def test_get_cpu_used_ratio_clamped_to_one( + fake_cgroup: Callable[..., Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Clamps a burst above the quota to 1, so the promised range holds despite clock skew.""" + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpu.max': '200000 100000\n', 'cpu.stat': 'usage_usec 0\n'}, + ) + # Three core-seconds within a one-second window exceed the two cores the quota allows. + fake_time(monkeypatch, sleep=lambda _seconds: (root / 'cpu.stat').write_text('usage_usec 3000000\n')) + + assert cgroups_sensor.get_cpu_used_ratio() == 1.0 + + +def test_get_cpu_used_ratio_empty_window( + fake_cgroup: Callable[..., Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Reports nothing when the clock does not advance across the interval, instead of dividing by zero.""" + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpu.max': '200000 100000\n', 'cpu.stat': 'usage_usec 0\n'}, + ) + monkeypatch.setattr(_sensor, 'time', SimpleNamespace(monotonic=lambda: 100.0, sleep=lambda _seconds: None)) + + assert cgroups_sensor.get_cpu_used_ratio() is None + + +def test_get_cpu_used_ratio_measures_the_given_interval( + fake_cgroup: Callable[..., Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Waits for the interval it was given, not for a fixed one.""" + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpu.max': '200000 100000\n', 'cpu.stat': 'usage_usec 0\n'}, + ) + slept: list[float] = [] + fake_time(monkeypatch, sleep=slept.append) + + cgroups_sensor.get_cpu_used_ratio(interval=0.25) + + assert slept == [0.25] + + +def test_get_cpu_used_ratio_async_measures_the_given_interval(fake_cgroup: Callable[..., Path]) -> None: + """Waits for the interval it was given, as the blocking variant does.""" + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpu.max': '200000 100000\n', 'cpu.stat': 'usage_usec 0\n'}, + ) + slept: list[float] = [] + + async def sleep(seconds: float) -> None: + slept.append(seconds) + + with pytest.MonkeyPatch.context() as patch: + patch.setattr(asyncio, 'sleep', sleep) + asyncio.run(cgroups_sensor.get_cpu_used_ratio_async(interval=0.25)) + + assert slept == [0.25] + + +@pytest.mark.usefixtures('_no_cgroup') +@pytest.mark.parametrize('interval', [0, -1.0, 0.005]) +def test_get_cpu_used_ratio_invalid_interval(interval: float) -> None: + """Rejects a window shorter than the counter can resolve, instead of sleeping and reporting nothing.""" + with pytest.raises(ValueError, match='interval must be at least'): + cgroups_sensor.get_cpu_used_ratio(interval=interval) + + +def test_get_cpu_used_ratio_async(fake_cgroup: Callable[..., Path], monkeypatch: pytest.MonkeyPatch) -> None: + """Measures the same ratio as the blocking variant, waiting with `asyncio.sleep` instead.""" + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpu.max': '200000 100000\n', 'cpu.stat': 'usage_usec 0\n'}, + ) + + async def sleep(_seconds: float) -> None: + (root / 'cpu.stat').write_text('usage_usec 1000000\n') + + clock = count(start=100.0, step=1.0) + monkeypatch.setattr(_sensor, 'time', SimpleNamespace(monotonic=lambda: next(clock))) + monkeypatch.setattr(asyncio, 'sleep', sleep) + + assert asyncio.run(cgroups_sensor.get_cpu_used_ratio_async()) == pytest.approx(0.5) + + +def test_get_cpu_used_ratio_async_no_limit(fake_cgroup: Callable[..., Path]) -> None: + """Reports nothing when the CPU is unrestricted, same as the blocking variant.""" + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpu.max': 'max 100000\n', 'cpu.stat': 'usage_usec 0\n'}, + ) + + assert asyncio.run(cgroups_sensor.get_cpu_used_ratio_async()) is None + + +@pytest.mark.usefixtures('_no_cgroup') +def test_get_cpu_used_ratio_async_invalid_interval() -> None: + """Rejects an unmeasurable window, same as the blocking variant.""" + with pytest.raises(ValueError, match='interval must be at least'): + asyncio.run(cgroups_sensor.get_cpu_used_ratio_async(interval=0.005)) + + +def test_snapshot(fake_cgroup: Callable[..., Path]) -> None: + """Takes every reading at once.""" + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={ + 'memory.max': '536870912\n', + 'memory.current': '150000000\n', + 'memory.stat': 'inactive_file 50000000\n', + 'cpu.max': '200000 100000\n', + 'cpu.stat': 'usage_usec 2500000\n', + 'cpuset.cpus.effective': '0-3\n', + }, + ) + + assert cgroups_sensor.snapshot() == cgroups_sensor.Snapshot( + memory_budget=cgroups_sensor.MemoryBudget(limit=536870912, working_set=100000000), + cpu_limit=2.0, + cpu_usage=2.5, + ) + + +@pytest.mark.usefixtures('_no_cgroup') +def test_snapshot_no_mechanism() -> None: + """Reports an all-empty snapshot, without raising, on a system that has no cgroups.""" + assert cgroups_sensor.snapshot() == cgroups_sensor.Snapshot(memory_budget=None, cpu_limit=None, cpu_usage=None) + + +def test_snapshot_v1(fake_cgroup: Callable[..., Path]) -> None: + """Reads every metric through the cgroup v1 controllers, each mounted as a hierarchy of its own.""" + fake_cgroup( + mountinfo=V1_MOUNTINFO, + self_cgroup=V1_SELF_CGROUP.format(path='/'), + files={ + 'memory/memory.limit_in_bytes': '536870912\n', + 'memory/memory.usage_in_bytes': '1000\n', + 'memory/memory.stat': 'total_inactive_file 400\n', + 'cpu,cpuacct/cpu.cfs_quota_us': '150000\n', + 'cpu,cpuacct/cpu.cfs_period_us': '100000\n', + 'cpu,cpuacct/cpuacct.usage': '2500000000\n', + 'cpuset/cpuset.cpus': '0-1\n', + }, + ) + + assert cgroups_sensor.snapshot() == cgroups_sensor.Snapshot( + memory_budget=cgroups_sensor.MemoryBudget(limit=536870912, working_set=600), + cpu_limit=1.5, + cpu_usage=2.5, + ) + + description = cgroups_sensor.describe() + for source in ( + description.memory_source, + description.cpu_quota_source, + description.cpu_set_source, + description.cpu_usage_source, + ): + assert source is not None + assert source.interface is cgroups_sensor.Interface.CGROUP_V1 + + +def test_describe(fake_cgroup: Callable[..., Path]) -> None: + """Reports the source of every metric, the raw values before filtering, and the machine facts.""" + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={ + 'memory.max': '536870912\n', + 'memory.current': '150000000\n', + 'memory.stat': 'inactive_file 50000000\n', + 'cpu.max': '1000000 100000\n', + 'cpu.stat': 'usage_usec 2500000\n', + 'cpuset.cpus.effective': '0-3\n', + }, + ) + + description = cgroups_sensor.describe() + + v2 = cgroups_sensor.Interface.CGROUP_V2 + + # The readings themselves, so that one dump of this is a complete answer. + assert description.memory_budget == cgroups_sensor.MemoryBudget(limit=536870912, working_set=100000000) + assert description.cpu_limit == 4.0 + assert description.memory_source == cgroups_sensor.Source(interface=v2, levels=(str(root),)) + assert description.cpu_quota_source == cgroups_sensor.Source(interface=v2, levels=(str(root),)) + assert description.cpu_set_source == cgroups_sensor.Source(interface=v2, levels=(str(root),)) + assert description.cpu_usage_source == cgroups_sensor.Source(interface=v2, levels=(str(root),)) + assert description.raw_memory_limit == 536870912 + assert description.raw_memory_working_set == 100000000 + assert description.memory_limit_level == str(root) + # The set of four cores is the effective limit here, and its time is counted in the own cgroup. + assert description.cpu_limit_level == str(root) + # The quota of ten cores exceeds the machine, so it is visible here and filtered from `get_cpu_limit`. + assert description.raw_cpu_quota == 10.0 + assert description.raw_cpu_set_size == 4 + assert description.machine_memory_bytes == MACHINE_TOTAL_BYTES + assert description.machine_cpu_count == MACHINE_CORES + assert [notice.code for notice in description.notices] == ['cpu-quota-covers-machine'] + + +def test_describe_hybrid_interfaces(fake_cgroup: Callable[..., Path]) -> None: + """Names the interface per metric, because a hybrid layout serves different metrics from different versions.""" + fake_cgroup( + mountinfo=f'{V2_MOUNTINFO}\n{V1_MOUNTINFO}', + self_cgroup=f'{V2_SELF_CGROUP.format(path="/")}2:memory:/\n', + files={ + 'memory/memory.limit_in_bytes': '536870912\n', + 'memory/memory.usage_in_bytes': '1000\n', + 'memory/memory.stat': 'total_inactive_file 400\n', + }, + ) + + description = cgroups_sensor.describe() + + assert description.memory_source is not None + assert description.memory_source.interface is cgroups_sensor.Interface.CGROUP_V1 + assert description.cpu_quota_source is None + + +def test_describe_names_the_level_the_memory_limit_came_from(fake_cgroup: Callable[..., Path]) -> None: + """Names the level holding the limit, which the chain of searched levels does not say. + + Every level is kept in `levels`, carrying the metric or not, so `levels[0]` is the cgroup of this process + rather than the source of the limit. + """ + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/pod/container'), + files={ + 'pod/container/memory.max': '536870912\n', + 'pod/container/memory.current': '1000\n', + 'pod/container/memory.stat': 'inactive_file 400\n', + # The tighter limit sits on the pod above. + 'pod/memory.max': '268435456\n', + 'pod/memory.current': '2000\n', + 'pod/memory.stat': 'inactive_file 400\n', + }, + ) + + description = cgroups_sensor.describe() + + assert description.memory_source is not None + assert description.memory_source.levels == ( + str(root / 'pod' / 'container'), + str(root / 'pod'), + str(root), + ) + assert description.memory_limit_level == str(root / 'pod') + + +def test_describe_names_the_level_the_cpu_limit_came_from(fake_cgroup: Callable[..., Path]) -> None: + """Names the level the quota binds on, which is also where a rate has to be measured. + + The quota sits on the slice above this process, so the counter of the own cgroup would answer a different + question - see `test_get_cpu_used_ratio_measures_where_the_quota_binds`. + """ + root = loaded_slice(fake_cgroup, own_usec=0, slice_usec=0) + + description = cgroups_sensor.describe() + + assert description.cpu_limit_level == str(root / 'bench.slice') + # One hierarchy carries both, so the two coincide here. The test below is where they do not. + assert description.cpu_rate_level == str(root / 'bench.slice') + + +def test_describe_names_both_cpu_levels_across_split_hierarchies(fake_cgroup: Callable[..., Path]) -> None: + """Tells the level holding the limit from the level counting its time, which cgroup v1 can keep apart. + + `cpu` and `cpuacct` are separate controllers and can be mounted separately. The quota is then readable in + one hierarchy and the time it applies to only in the other. + """ + root = fake_cgroup( + mountinfo=V1_CPU_SPLIT_MOUNTINFO, + self_cgroup='3:cpu:/slice\n4:cpuacct:/slice\n', + files={ + 'cpu/slice/cpu.cfs_quota_us': '50000\n', + 'cpu/slice/cpu.cfs_period_us': '100000\n', + 'cpuacct/slice/cpuacct.usage': '1000000000\n', + }, + ) + + description = cgroups_sensor.describe() + + assert description.cpu_limit == 0.5 + assert description.cpu_limit_level == str(root / 'cpu' / 'slice') + assert description.cpu_rate_level == str(root / 'cpuacct' / 'slice') + + +@pytest.mark.parametrize( + ('member', 'expected'), + [ + pytest.param(cgroups_sensor.NoticeCode.MEMORY_LIMIT_COVERS_MACHINE, 'memory-limit-covers-machine', id='notice'), + pytest.param(cgroups_sensor.Interface.CGROUP_V2, 'cgroup-v2', id='interface'), + ], +) +def test_enum_members_print_as_their_strings(member: str, expected: str) -> None: + """Prints as the string it carries, which is what a log line shows and what a payload carries. + + A `str` enum does not do this on its own: a member prints as `NoticeCode.MEMORY_LIMIT_COVERS_MACHINE`, and + an f-string of it differs between the supported Python versions. The JSON form is what the end-to-end + probe reports its readings through, and it holds only as long as these stay `str` members. + """ + assert str(member) == expected + assert f'{member}' == expected + assert member == expected + assert json.dumps(member) == f'"{expected}"' + + +@pytest.mark.parametrize( + ('cores', 'expected'), + [ + pytest.param(64.0, '64', id='a whole number of cores'), + pytest.param(0.5, '0.5', id='half a core'), + pytest.param(1.5, '1.5', id='one and a half'), + ], +) +def test_spell_cores(cores: float, expected: str) -> None: + """Writes a whole number of cores without a fraction, and a fractional quota as it is.""" + assert _sensor._spell_cores(cores) == expected + + +@pytest.mark.usefixtures('_no_cgroup') +def test_covers_machine_notice_spells_the_cores(monkeypatch: pytest.MonkeyPatch) -> None: + """Says "64 allowed cores" in the message a consumer reads, not "64.0" - a set has no fractional size.""" + counter = Path('/sys/fs/cgroup') + cpu_set = _cgroup.RawCpuSet(cores=64, limit_directory=counter, usage_directory=counter) + monkeypatch.setattr(_cgroup, 'read_cpu_quota', lambda: None) + monkeypatch.setattr(_cgroup, 'read_cpu_set_size', lambda: cpu_set) + + (notice,) = [n for n in cgroups_sensor.describe().notices if str(n.code) == 'cpu-set-covers-machine'] + + assert 'The set of 64 allowed cores' in notice.message + + +def test_every_notice_code_names_its_metric() -> None: + """Spells every code as `-...`, which is the only thing telling a memory notice from a CPU one. + + Nothing in the package enforces that, and both test suites sort notices by it. A code matching no metric + would silently drop out of every such filter, and a test asserting "no memory notices" would then pass + while one was raised. + """ + metrics = { + str(code): [metric for metric, prefixes in NOTICE_PREFIXES.items() if str(code).startswith(prefixes)] + for code in cgroups_sensor.NoticeCode + } + + assert all(len(found) == 1 for found in metrics.values()), metrics + + +@pytest.mark.usefixtures('_no_cgroup') +def test_describe_no_mechanism() -> None: + """Reports no sources when no mechanism exists at all, and one notice per metric saying so.""" + description = cgroups_sensor.describe() + + assert description.memory_source is None + assert description.cpu_quota_source is None + assert description.cpu_set_source is None + assert description.cpu_usage_source is None + assert description.raw_memory_limit is None + assert notice_codes() == ('memory-metrics-unavailable', 'cpu-metrics-unavailable') + + +def test_clear_cache(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fake_cgroup: Callable[..., Path]) -> None: + """Forgets the discovered sources, so a process moved to another group stops reading the old directories.""" + monkeypatch.setattr(_cgroup, '_PROC_SELF_MOUNTINFO', tmp_path / 'missing') + monkeypatch.setattr(_cgroup, '_PROC_SELF_CGROUP', tmp_path / 'missing') + assert cgroups_sensor.get_memory_budget() is None + + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={ + 'memory.max': '536870912\n', + 'memory.current': '150000000\n', + 'memory.stat': 'inactive_file 50000000\n', + }, + ) + + # The stale discovery still reports nothing until the cache is dropped. + assert cgroups_sensor.get_memory_budget() is None + + cgroups_sensor.clear_cache() + + assert cgroups_sensor.get_memory_budget() == cgroups_sensor.MemoryBudget(limit=536870912, working_set=100000000) + + +@pytest.mark.parametrize( + ('content', 'expected'), + [ + pytest.param('MemFree: 100 kB\nMemTotal: 8054932 kB\n', 8054932 * 1024, id='the usual spelling'), + pytest.param('MemFree: 100 kB\n', None, id='no MemTotal line'), + pytest.param('MemTotal: garbage kB\n', None, id='unparsable value'), + pytest.param('MemTotal:\n', None, id='empty value'), + ], +) +def test_machine_memory_bytes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + content: str, + expected: int | None, +) -> None: + """Parses the machine memory out of `/proc/meminfo`, and reports nothing rather than raising on any other.""" + meminfo = tmp_path / 'meminfo' + meminfo.write_text(content) + monkeypatch.setattr(_sensor, '_PROC_MEMINFO', meminfo) + + assert real_machine_memory_bytes() == expected + + +def test_machine_memory_bytes_missing_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Reports nothing on a system without `/proc/meminfo`.""" + monkeypatch.setattr(_sensor, '_PROC_MEMINFO', tmp_path / 'missing') + + assert real_machine_memory_bytes() is None + + +@pytest.mark.parametrize( + ('online', 'expected'), + [ + pytest.param('0-15\n', 16, id='a range'), + pytest.param('0-3,8-11\n', 8, id='ranges with a gap'), + pytest.param('0\n', 1, id='a single core'), + ], +) +def test_machine_cpu_count(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, online: str, expected: int) -> None: + """Counts the cores the kernel lists as online, whatever the process is allowed to run on. + + Under musl `sysconf` reports the affinity of the process, so a cpuset would look like the whole machine. + """ + listing = tmp_path / 'online' + listing.write_text(online) + monkeypatch.setattr(_sensor, '_SYS_CPU_ONLINE', listing) + # Numbers no case expects, so that a count taken from a fallback instead of the listing fails here. + # `raising=False`, because Windows has no `os.sysconf` to replace. + monkeypatch.setattr(os, 'sysconf', lambda _name: 99, raising=False) + monkeypatch.setattr(os, 'cpu_count', lambda: 98) + + assert real_machine_cpu_count() == expected + + +def test_machine_cpu_count_ignores_process_override(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Falls back to `sysconf` where the kernel lists nothing, and not to the `PYTHON_CPU_COUNT` override.""" + monkeypatch.setattr(_sensor, '_SYS_CPU_ONLINE', tmp_path / 'missing') + monkeypatch.setattr(os, 'sysconf', lambda _name: 16, raising=False) + monkeypatch.setattr(os, 'cpu_count', lambda: 2) + + assert real_machine_cpu_count() == 16 + + +@pytest.mark.parametrize( + 'sysconf_result', + [ + pytest.param(-1, id='the machine count is unavailable'), + pytest.param(ValueError('unknown name'), id='the name is not configured'), + ], +) +def test_machine_cpu_count_fallback( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + sysconf_result: int | Exception, +) -> None: + """Falls back to `os.cpu_count()` where neither the kernel nor `sysconf` can answer.""" + + def sysconf(_name: str) -> int: + if isinstance(sysconf_result, Exception): + raise sysconf_result + return sysconf_result + + monkeypatch.setattr(_sensor, '_SYS_CPU_ONLINE', tmp_path / 'missing') + monkeypatch.setattr(os, 'sysconf', sysconf, raising=False) + monkeypatch.setattr(os, 'cpu_count', lambda: 2) + + assert real_machine_cpu_count() == 2 + + +def loaded_slice(fake_cgroup: Callable[..., Path], *, own_usec: int, slice_usec: int) -> Path: + """Lay out a slice whose quota binds above this process, with the load sitting at that level. + + This is the shape a `CPUQuota=` slice with several busy units in it produces: the process of interest is + nearly idle, while the level the quota throttles is saturated. + """ + return fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/bench.slice/own.service'), + files={ + 'bench.slice/own.service/cpu.max': 'max 100000\n', + 'bench.slice/own.service/cpu.stat': f'usage_usec {own_usec}\n', + 'bench.slice/cpu.max': '50000 100000\n', + 'bench.slice/cpu.stat': f'usage_usec {slice_usec}\n', + }, + ) + + +def test_get_cpu_used_ratio_measures_where_the_quota_binds( + fake_cgroup: Callable[..., Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Measures the level the quota throttles, not the idle process inside it. + + The quota of half a core sits on the slice, and the slice consumes half a core-second per second. The + process itself barely runs, and reading only its own counter would report an idle machine while the kernel + is throttling. + """ + root = loaded_slice(fake_cgroup, own_usec=7100, slice_usec=0) + + def sleep(_seconds: float) -> None: + (root / 'bench.slice' / 'own.service' / 'cpu.stat').write_text('usage_usec 14200\n') + (root / 'bench.slice' / 'cpu.stat').write_text('usage_usec 500000\n') + + fake_time(monkeypatch, sleep=sleep) + + assert cgroups_sensor.get_cpu_used_ratio() == pytest.approx(1.0) + + +def test_cpu_load_first_sample(fake_cgroup: Callable[..., Path]) -> None: + """Reports nothing on the first call, because a rate needs an earlier reading.""" + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpu.max': '200000 100000\n', 'cpu.stat': 'usage_usec 0\n'}, + ) + + assert cgroups_sensor.CpuLoad().sample() is None + + +def test_cpu_load_measures_between_calls( + fake_cgroup: Callable[..., Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Measures across the time between two calls, and blocks for nothing.""" + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpu.max': '200000 100000\n', 'cpu.stat': 'usage_usec 0\n'}, + ) + fake_time(monkeypatch, sleep=lambda _seconds: None) + load = cgroups_sensor.CpuLoad() + + assert load.sample() is None + + # One core-second over the one second the fake clock advances, out of the two the quota allows. + (root / 'cpu.stat').write_text('usage_usec 1000000\n') + + assert load.sample() == pytest.approx(0.5) + + +def test_cpu_load_counter_restart(fake_cgroup: Callable[..., Path], monkeypatch: pytest.MonkeyPatch) -> None: + """Clamps the counter that restarts when the process is moved to another group.""" + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpu.max': '200000 100000\n', 'cpu.stat': 'usage_usec 5000000\n'}, + ) + fake_time(monkeypatch, sleep=lambda _seconds: None) + load = cgroups_sensor.CpuLoad() + load.sample() + + (root / 'cpu.stat').write_text('usage_usec 0\n') + + assert load.sample() == 0.0 + + +def test_cpu_load_keeps_the_previous_reading( + fake_cgroup: Callable[..., Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Keeps the earlier reading when one fails, because a counter that only grows stays comparable.""" + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpu.max': '200000 100000\n', 'cpu.stat': 'usage_usec 0\n'}, + ) + fake_time(monkeypatch, sleep=lambda _seconds: None) + load = cgroups_sensor.CpuLoad() + + assert load.sample() is None + + (root / 'cpu.stat').unlink() + assert load.sample() is None + + # Two core-seconds against the two the quota allows, over the two seconds the clock advanced meanwhile. + (root / 'cpu.stat').write_text('usage_usec 4000000\n') + assert load.sample() == pytest.approx(1.0) + + +def test_cpu_load_when_nothing_restricts_the_cpu(fake_cgroup: Callable[..., Path]) -> None: + """Reports nothing where no limit applies, as the one-shot call does.""" + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpu.max': 'max 100000\n', 'cpu.stat': 'usage_usec 0\n'}, + ) + load = cgroups_sensor.CpuLoad() + + assert load.sample() is None + assert load.sample() is None + + +def test_no_rate_without_a_counter_for_the_limit( + fake_cgroup: Callable[..., Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Reports no rate at all when the level the limit applies to counts no CPU time. + + Measuring the group of this process instead would divide the time of one scope by the limit of another, + which is the mistake the notice exists to prevent. + """ + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpu.max': '200000 100000\n', 'cpu.stat': 'usage_usec 0\n'}, + ) + quota = _cgroup.RawCpuQuota(cores=2.0, limit_directory=Path('/sys/fs/cgroup'), usage_directory=None) + monkeypatch.setattr(_cgroup, 'read_cpu_quota', lambda: quota) + monkeypatch.setattr(_cgroup, 'read_cpu_set_size', lambda: None) + + # The limit itself still applies and is still reported. + assert cgroups_sensor.get_cpu_limit() == 2.0 + assert notice_codes('cpu') == ('cpu-usage-scope-mismatch',) + + assert cgroups_sensor.get_cpu_used_ratio() is None + assert cgroups_sensor.CpuLoad().sample() is None + + +def test_cpu_load_when_the_limit_moves_to_another_level( + fake_cgroup: Callable[..., Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Reports nothing for the sample across a limit that moved to another level. + + Arbitrary time passes between two calls of the sampler, and a quota can appear on an ancestor meanwhile. + The counter then belongs to another scope than the previous one, and their difference means nothing. + """ + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/pod/container'), + files={ + 'pod/container/cpu.max': '200000 100000\n', + 'pod/container/cpu.stat': 'usage_usec 0\n', + 'pod/cpu.max': 'max 100000\n', + 'pod/cpu.stat': 'usage_usec 0\n', + }, + ) + fake_time(monkeypatch, sleep=lambda _seconds: None) + load = cgroups_sensor.CpuLoad() + + assert load.sample() is None + + # A tighter quota appears on the pod, so the limit now binds there and the counter to read moves with it. + (root / 'pod' / 'cpu.max').write_text('50000 100000\n') + + assert load.sample() is None + + # The reading taken at the pod becomes the baseline, and the next sample measures against it. + (root / 'pod' / 'cpu.stat').write_text('usage_usec 500000\n') + + assert load.sample() == pytest.approx(1.0) + + +class _CountingLock: + """Stands in for the sampler's lock and records that it was taken.""" + + def __init__(self, lock: threading.Lock) -> None: + self.lock = lock + self.taken = 0 + + def __enter__(self) -> None: + self.taken += 1 + self.lock.acquire() + + def __exit__(self, *_exception: object) -> None: + self.lock.release() + + +def test_cpu_load_swaps_the_previous_reading_under_its_lock( + fake_cgroup: Callable[..., Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Replaces the kept reading inside the lock, which is what the promise of thread safety rests on. + + A race cannot be reproduced on demand, so what is pinned here is the discipline that prevents it. Reading + the previous value and writing the new one is one step, and two threads must not interleave inside it. + """ + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpu.max': '200000 100000\n', 'cpu.stat': 'usage_usec 0\n'}, + ) + load = cgroups_sensor.CpuLoad() + counting = _CountingLock(load._lock) + monkeypatch.setattr(load, '_lock', counting) + + load.sample() + load.sample() + + assert counting.taken == 2 + + +def test_cpu_load_sampled_from_several_threads(fake_cgroup: Callable[..., Path]) -> None: + """Answers every caller without deadlocking or raising when several threads share one sampler. + + Sharing one is not what the class recommends - each caller measures the others' windows - but it must not + break, because a caller cannot always tell that it happened. + """ + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpu.max': '200000 100000\n', 'cpu.stat': 'usage_usec 0\n'}, + ) + load = cgroups_sensor.CpuLoad() + samples: list[float | None] = [] + + def sample_repeatedly() -> None: + # One `extend` per thread, so that the list is touched as rarely as the sampler allows. + samples.extend(load.sample() for _ in range(50)) + + threads = [threading.Thread(target=sample_repeatedly) for _ in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=30) + + assert not [thread for thread in threads if thread.is_alive()] + assert len(samples) == 8 * 50 + assert all(sample is None or 0.0 <= sample <= 1.0 for sample in samples) + + +def test_cpu_load_when_the_limit_moves_without_changing_value( + fake_cgroup: Callable[..., Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Reports nothing for a limit that moved level while keeping its value. + + kubelet writes onto the pod the number the container carried. The limits then compare equal while the + counters belong to two scopes, and only the level says so. + """ + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/pod/container'), + files={ + 'pod/container/cpu.max': '50000 100000\n', + 'pod/container/cpu.stat': 'usage_usec 0\n', + 'pod/cpu.max': 'max 100000\n', + # The pod has been running its other containers all along. + 'pod/cpu.stat': 'usage_usec 900000000\n', + }, + ) + fake_time(monkeypatch, sleep=lambda _seconds: None) + load = cgroups_sensor.CpuLoad() + + assert load.sample() is None + + # The same half core, one level up. Comparing the pod's counter against the container's baseline would + # report a saturated process out of nothing. + (root / 'pod' / 'container' / 'cpu.max').write_text('max 100000\n') + (root / 'pod' / 'cpu.max').write_text('50000 100000\n') + + assert load.sample() is None + + +def test_cpu_load_when_the_limit_changes_in_place( + fake_cgroup: Callable[..., Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Reports nothing for the sample across a limit that changed value. + + The same consumption divided by two different numbers of cores is not a rate of anything. A container + resized between two calls is the ordinary way this happens. + """ + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpu.max': '400000 100000\n', 'cpu.stat': 'usage_usec 0\n'}, + ) + fake_time(monkeypatch, sleep=lambda _seconds: None) + load = cgroups_sensor.CpuLoad() + + assert load.sample() is None + + # Four cores become one, and the counter keeps growing across the change. + (root / 'cpu.max').write_text('100000 100000\n') + (root / 'cpu.stat').write_text('usage_usec 1000000\n') + + assert load.sample() is None + + # The reading taken under the new limit becomes the baseline. + (root / 'cpu.stat').write_text('usage_usec 2000000\n') + + assert load.sample() == pytest.approx(1.0) + + +def test_get_cpu_used_ratio_returns_at_once_without_a_limit( + fake_cgroup: Callable[..., Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Waits for nothing when there is nothing to measure, which a loop around it has to expect.""" + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpu.max': 'max 100000\n', 'cpu.stat': 'usage_usec 0\n'}, + ) + slept: list[float] = [] + fake_time(monkeypatch, sleep=slept.append) + + assert cgroups_sensor.get_cpu_used_ratio(interval=5.0) is None + assert slept == [] + + +def test_cpu_load_two_samples_too_close(fake_cgroup: Callable[..., Path], monkeypatch: pytest.MonkeyPatch) -> None: + """Reports nothing for a window shorter than the counter's own resolution. + + Two callers sharing one sampler take each other's windows, and the second of them would otherwise read a + saturated cgroup as idle: the counter has not moved yet. + """ + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpu.max': '200000 100000\n', 'cpu.stat': 'usage_usec 5000000\n'}, + ) + clock = count(start=100.0, step=0.001) + monkeypatch.setattr(_sensor, 'time', SimpleNamespace(monotonic=lambda: next(clock), sleep=lambda _s: None)) + load = cgroups_sensor.CpuLoad() + + assert load.sample() is None + assert load.sample() is None + + +def test_nothing_raises_on_unreadable_files(fake_cgroup: Callable[..., Path]) -> None: + """Reports nothing rather than raising when every control file holds something unreadable, and says so. + + The contract of the whole module is that a reading fails into `None`. An emulated cgroupfs, a truncated + file or a racing runtime can produce any of these, and a limit that cannot be read is not a limit that is + not there: what such a level enforces is unknown, so nothing is reported and a notice names it. + """ + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={ + 'memory.max': 'not a number\n', + 'memory.current': '\n', + 'memory.stat': 'inactive_file\n', + 'cpu.max': 'one two three\n', + 'cpu.stat': 'usage_usec not-a-number\n', + 'cpuset.cpus.effective': '9-1\n', + }, + ) + + assert cgroups_sensor.snapshot() == cgroups_sensor.Snapshot( + memory_budget=None, + cpu_limit=None, + cpu_usage=None, + ) + assert cgroups_sensor.get_cpu_used_ratio(interval=0.01) is None + assert cgroups_sensor.CpuLoad().sample() is None + assert notice_codes() == ('memory-limit-unreadable', 'cpu-limit-unreadable') + + +def test_nothing_raises_on_directories_instead_of_files( + fake_cgroup: Callable[..., Path], + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Reports nothing rather than raising when a `/proc` file is not a file at all.""" + fake_cgroup(mountinfo=V2_MOUNTINFO, self_cgroup=V2_SELF_CGROUP.format(path='/'), files={}) + for name in ('_PROC_SELF_MOUNTINFO', '_PROC_SELF_CGROUP'): + monkeypatch.setattr(_cgroup, name, tmp_path) + + assert cgroups_sensor.snapshot() == cgroups_sensor.Snapshot( + memory_budget=None, + cpu_limit=None, + cpu_usage=None, + ) + assert cgroups_sensor.describe().memory_source is None + + +def test_memory_budget_derived_numbers() -> None: + """Reports what a consumer would otherwise compute, and the same way everywhere.""" + budget = cgroups_sensor.MemoryBudget(limit=1000, working_set=250) + + assert budget.available == 750 + assert budget.used_ratio == 0.25 + + +def test_memory_budget_of_zero() -> None: + """Calls a cgroup that may hold no memory fully used, rather than dividing by its limit.""" + budget = cgroups_sensor.MemoryBudget(limit=0, working_set=0) + + assert budget.available == 0 + assert budget.used_ratio == 1.0 diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..2afe8ba --- /dev/null +++ b/uv.lock @@ -0,0 +1,301 @@ +version = 1 +revision = 3 +requires-python = ">=3.10" + +[[package]] +name = "cgroups-sensor" +version = "0.1.0" +source = { editable = "." } + +[package.dev-dependencies] +dev = [ + { name = "poethepoet" }, + { name = "pytest" }, + { name = "ruff" }, + { name = "ty" }, +] + +[package.metadata] + +[package.metadata.requires-dev] +dev = [ + { name = "poethepoet", specifier = "<1.0.0" }, + { name = "pytest", specifier = "<10.0.0" }, + { name = "ruff", specifier = "~=0.16.0" }, + { name = "ty", specifier = ">=0.0.72,<0.1.0" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "exceptiongroup" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pastel" +version = "0.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/76/f1/4594f5e0fcddb6953e5b8fe00da8c317b8b41b547e2b3ae2da7512943c62/pastel-0.2.1.tar.gz", hash = "sha256:e6581ac04e973cac858828c6202c1e1e81fee1dc7de7683f3e1ffe0bfd8a573d", size = 7555, upload-time = "2020-09-16T19:21:12.43Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/18/a8444036c6dd65ba3624c63b734d3ba95ba63ace513078e1580590075d21/pastel-0.2.1-py2.py3-none-any.whl", hash = "sha256:4349225fcdf6c2bb34d483e523475de5bb04a5c10ef711263452cb37d7dd4364", size = 5955, upload-time = "2020-09-16T19:21:11.409Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "poethepoet" +version = "0.48.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pastel" }, + { name = "pyyaml" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5c/92/93a4af9511b8c7c647874521d9e6c904266be98067c2ee1eb2e74520d208/poethepoet-0.48.0.tar.gz", hash = "sha256:a06f49d244fadfc2e2e7faa78b54e64a9694727e4ce1d50e08f23cea3ded74f1", size = 148679, upload-time = "2026-07-05T21:48:30.106Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/8d/d7c9455b15f8d2d7ce57e7b71a8ef8d02d9992ae4283c9777120620c9022/poethepoet-0.48.0-py3-none-any.whl", hash = "sha256:98da6096d060f49b8d84034770265863fb7dc92a40233b7694b9d216ac68737d", size = 185808, upload-time = "2026-07-05T21:48:28.601Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/a0/39350dd17dd6d6c6507025c0e53aef67a9293a6d37d3511f23ea510d5800/pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b", size = 184227, upload-time = "2025-09-25T21:31:46.04Z" }, + { url = "https://files.pythonhosted.org/packages/05/14/52d505b5c59ce73244f59c7a50ecf47093ce4765f116cdb98286a71eeca2/pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956", size = 174019, upload-time = "2025-09-25T21:31:47.706Z" }, + { url = "https://files.pythonhosted.org/packages/43/f7/0e6a5ae5599c838c696adb4e6330a59f463265bfa1e116cfd1fbb0abaaae/pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8", size = 740646, upload-time = "2025-09-25T21:31:49.21Z" }, + { url = "https://files.pythonhosted.org/packages/2f/3a/61b9db1d28f00f8fd0ae760459a5c4bf1b941baf714e207b6eb0657d2578/pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198", size = 840793, upload-time = "2025-09-25T21:31:50.735Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1e/7acc4f0e74c4b3d9531e24739e0ab832a5edf40e64fbae1a9c01941cabd7/pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b", size = 770293, upload-time = "2025-09-25T21:31:51.828Z" }, + { url = "https://files.pythonhosted.org/packages/8b/ef/abd085f06853af0cd59fa5f913d61a8eab65d7639ff2a658d18a25d6a89d/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0", size = 732872, upload-time = "2025-09-25T21:31:53.282Z" }, + { url = "https://files.pythonhosted.org/packages/1f/15/2bc9c8faf6450a8b3c9fc5448ed869c599c0a74ba2669772b1f3a0040180/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69", size = 758828, upload-time = "2025-09-25T21:31:54.807Z" }, + { url = "https://files.pythonhosted.org/packages/a3/00/531e92e88c00f4333ce359e50c19b8d1de9fe8d581b1534e35ccfbc5f393/pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e", size = 142415, upload-time = "2025-09-25T21:31:55.885Z" }, + { url = "https://files.pythonhosted.org/packages/2a/fa/926c003379b19fca39dd4634818b00dec6c62d87faf628d1394e137354d4/pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c", size = 158561, upload-time = "2025-09-25T21:31:57.406Z" }, + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "ruff" +version = "0.16.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/61/b3/3213589383f8f1b3938781bd1278713f6d18621a14992b3e81fefb8a5ef9/ruff-0.16.3.tar.gz", hash = "sha256:e76d33a347661a84b5be6d043d0347fdc745dfdcf825a8f4fed64b5e26eebdf2", size = 4891904, upload-time = "2026-08-13T15:17:13.381Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bf/96/493770daebd68c0a67f1549fdf519f53be51fc435186c0585bcc272fd76c/ruff-0.16.3-py3-none-linux_armv6l.whl", hash = "sha256:0c5710e247a58a4521e66e124ba9a74655b414f61ba3a2e9e3811e11098f48f7", size = 10902799, upload-time = "2026-08-13T15:16:27.382Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e6/2becf3942fddc29a29b8df47691d456fb1085391a694f74d84513251418c/ruff-0.16.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fe155130631a2471fd2e14a7a664a4dfbd7194b8229c3d7b2a40b21178639081", size = 11135539, upload-time = "2026-08-13T15:16:30.87Z" }, + { url = "https://files.pythonhosted.org/packages/3e/1e/4b8b72f0d006dbf19326aa99f9ca0ee2ff374187c4d301cf529a51aa06fe/ruff-0.16.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e2ed719e14aa64d895c2ee922594a90a43c861a93f0575a95ff8c47cdbd13eb9", size = 10475095, upload-time = "2026-08-13T15:16:33.259Z" }, + { url = "https://files.pythonhosted.org/packages/92/32/2201fa49ba1f6c101ee321e83f051ac7a4b8d07b0ef6b4d3f2772b302275/ruff-0.16.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9e0b1da805eb043654645d74d5de1e5ce2edc686e40790d2b86f56d71cc06a84", size = 10668771, upload-time = "2026-08-13T15:16:35.65Z" }, + { url = "https://files.pythonhosted.org/packages/c3/66/4afc5c8363bd04d45effce1b7c8713ca037d7a6740b7451a2403a6e3a972/ruff-0.16.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a37bdea0bbe21780f590bf437d6412c8c4e1b6cd010f91a65c2c40c5e5f5f870", size = 10699568, upload-time = "2026-08-13T15:16:38.195Z" }, + { url = "https://files.pythonhosted.org/packages/53/fd/c67d246bf36bf1698551c56de39e95cd07f70e64433e0098e6267d77061b/ruff-0.16.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09571e6d1288ed9be475207a3ac04ada404f1cd898104be0f6ab8d7df438575b", size = 11499365, upload-time = "2026-08-13T15:16:40.623Z" }, + { url = "https://files.pythonhosted.org/packages/67/0b/00ecbceb99a263af7b12f6f05ac3c92bc47b905e91adc3f207a836e3bc01/ruff-0.16.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2c18c5a101eb540010638cc1ff3c84944d3adb3df62b8d98ca8f22ba484d3413", size = 12311728, upload-time = "2026-08-13T15:16:43.564Z" }, + { url = "https://files.pythonhosted.org/packages/54/b2/b7b3bb54f4d3f7db504e476ad4ab8de530dceebe2c061384b2757ee419e8/ruff-0.16.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8457c44f15033c85ddbb77b15d451df9e24e4bd03b628396dd3610cedc3b8f82", size = 11699896, upload-time = "2026-08-13T15:16:46.209Z" }, + { url = "https://files.pythonhosted.org/packages/c7/30/4c468429ac195addc5ee1b717b6ab1b66632786737ca3b2ed3443fb0c26a/ruff-0.16.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:294b95c4ae0cda9388525c2047778aa758d6b8d4bb876fd4e9eaa3ebc92343eb", size = 11058736, upload-time = "2026-08-13T15:16:48.823Z" }, + { url = "https://files.pythonhosted.org/packages/43/67/7a113cdaddf24b64d7f75b1242a99d04c82fcef4f6921fdbb832beaffb5f/ruff-0.16.3-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:3d0c7c40c87c2a820509c31ba007968da6e1306468c067b2d82fbfdbcd0e8474", size = 11586911, upload-time = "2026-08-13T15:16:51.913Z" }, + { url = "https://files.pythonhosted.org/packages/f1/c1/2e66f24c0f3ead25a5e660111778685e505e5da353c82802bf49f0cbe7b9/ruff-0.16.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:9f738c0fdfa8eed0b2ce7fb27ee7258208a92a68d7949e62aa15164bc7b389da", size = 10954265, upload-time = "2026-08-13T15:16:54.763Z" }, + { url = "https://files.pythonhosted.org/packages/c2/ba/4cee23bf52cba9a058d3726de623624daf50ef9638868edd86f4126157f6/ruff-0.16.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:fb785f0be25abe69d320415cd4f833b59e17ba7613d9ba6a958023b6bceb0a50", size = 10709886, upload-time = "2026-08-13T15:16:57.339Z" }, + { url = "https://files.pythonhosted.org/packages/82/df/7da7194fa5d9dc0a285f7e6fa5a4722e7c63faac0b45b614ded9314363a1/ruff-0.16.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c5536e3acfbf9563085aa2be7b13c629c3077e902afc5b941ac44024dbb9f506", size = 11210392, upload-time = "2026-08-13T15:17:00.171Z" }, + { url = "https://files.pythonhosted.org/packages/35/85/7795f6e817af050e7517bf3e7aa9b061cce70ef33d280aad902c956c1ecf/ruff-0.16.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a2d85c02f9b8e165d85e6779184d38c4132de12603dab59c51c28e22584f9e4d", size = 11626910, upload-time = "2026-08-13T15:17:03.299Z" }, + { url = "https://files.pythonhosted.org/packages/78/9b/475b927cf27a5cbbda3c7bafb69ed6ff77e1d7923d5d85f17c2749d7ae32/ruff-0.16.3-py3-none-win32.whl", hash = "sha256:388cdf2166642bd9b13d52b5932d3170f34f8abed7e8d9a855f1d84b83645a0a", size = 10931415, upload-time = "2026-08-13T15:17:05.726Z" }, + { url = "https://files.pythonhosted.org/packages/b2/99/e2a2bfc4fbf0a1e8a916bc9ebe6fe6c58cc34c28e0ffc6ce281d572d1c2e/ruff-0.16.3-py3-none-win_amd64.whl", hash = "sha256:e80a7d69ca2a6d1c4d352ec91458cdca6e56c83cdbcabd93e4abe1e53591d948", size = 11445993, upload-time = "2026-08-13T15:17:08.353Z" }, + { url = "https://files.pythonhosted.org/packages/69/3e/4132e539aed78c148854d4997a2685b0ed4dc4e87110b59ce528564e184e/ruff-0.16.3-py3-none-win_arm64.whl", hash = "sha256:b8ca152da82c1acc1fa8d5874b15951935f0eef46f10e6954c83859011b6178a", size = 11399302, upload-time = "2026-08-13T15:17:10.908Z" }, +] + +[[package]] +name = "tomli" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" }, + { url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" }, + { url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" }, + { url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" }, + { url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" }, + { url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" }, + { url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" }, + { url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" }, + { url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" }, + { url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" }, + { url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" }, + { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" }, + { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" }, + { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" }, + { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" }, + { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" }, + { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" }, + { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" }, + { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" }, + { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" }, + { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" }, + { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" }, + { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" }, + { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" }, + { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" }, + { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, +] + +[[package]] +name = "ty" +version = "0.0.72" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d5/df/656e684bafb13c1d146e7d5b5f3e7978ca177232acc84998ff36427e9462/ty-0.0.72.tar.gz", hash = "sha256:ec2b8066b618df18cab4cb8e992f8da45d360332acb23fa34df7fa29cd1b9d3a", size = 6654939, upload-time = "2026-08-14T21:35:42.612Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/3b/f51461239a4e66565d4b362f97a3b55fe7fdba2e944068341f87c62f6743/ty-0.0.72-py3-none-linux_armv6l.whl", hash = "sha256:fda86db153ffd85ee52000cf175d6a3f1c0223772cf7c5b6f726200bf92c7b44", size = 12621989, upload-time = "2026-08-14T21:35:01.676Z" }, + { url = "https://files.pythonhosted.org/packages/ca/fb/79ddf683affc679ca856f3510b5640ec3a88a842ba5f654f5d4bc78f1786/ty-0.0.72-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ceb944c612529b9023acfdc9cf4c0dcbb722549f9d17d46baecd1141baf01d7f", size = 12233910, upload-time = "2026-08-14T21:35:04.334Z" }, + { url = "https://files.pythonhosted.org/packages/5d/45/10562a0d84802158db8fa4ec46de54aa9fdcecdeeaabbfe3639ae7042b66/ty-0.0.72-py3-none-macosx_11_0_arm64.whl", hash = "sha256:108d76218333d6c092e5f1cebf8e9b06f25738613a0236a28e2dd47c936ee52c", size = 12084108, upload-time = "2026-08-14T21:35:06.686Z" }, + { url = "https://files.pythonhosted.org/packages/a1/dc/1fe1aef8d697e3509face271a5331700c7aa1d1e44a4b622707bdfa41d4b/ty-0.0.72-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f3943f186f741a2499a31053872169250c9264a9a49684920e48d8fcf4ef4f5", size = 12132640, upload-time = "2026-08-14T21:35:09.305Z" }, + { url = "https://files.pythonhosted.org/packages/14/46/41ceb265e96969487311a2014bd0e53abb4fbc1395efb2ebe411fcb4db62/ty-0.0.72-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cf283c07dc3cc52ca48a3ad8ab100fb5aec3aebbd03ef6a12d5f910b8e596fc5", size = 12402489, upload-time = "2026-08-14T21:35:11.555Z" }, + { url = "https://files.pythonhosted.org/packages/2b/45/30bf43cb4fd505c5c2dd30fda27dde5f05208686cd21217adec77c954204/ty-0.0.72-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:95f3b6462c38f9f115d10cee21f47fedf715fcf2040daf36eef210359300bc7c", size = 13130835, upload-time = "2026-08-14T21:35:13.746Z" }, + { url = "https://files.pythonhosted.org/packages/31/2f/03bba754d2613f640df168335c41f83f41db150bb515839c60d80e3a7880/ty-0.0.72-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:30caf658feb8ffb250d9e9e47107657a78f5f3425c227df1664d8df2ebe38880", size = 13590392, upload-time = "2026-08-14T21:35:16.839Z" }, + { url = "https://files.pythonhosted.org/packages/04/c7/03c67f00e63005ec41585653dc3096064570b1e6273742baae2798cd242f/ty-0.0.72-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:27bdc012ddfbeec8948e4a6036c0dc39ac7cf2c8ec7c7d48dc7d2fd56d57b399", size = 13309629, upload-time = "2026-08-14T21:35:19.169Z" }, + { url = "https://files.pythonhosted.org/packages/c1/df/102d3b264eb7f2a58dd11952f229bb5150bb5668d176a6154976a6675981/ty-0.0.72-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:802c5970a77d7739e6f499921fbb6984fb7ad8a31d95e1ff42fd46f3642e4f3b", size = 12734028, upload-time = "2026-08-14T21:35:22.099Z" }, + { url = "https://files.pythonhosted.org/packages/61/85/d0737c8c54d0ba67366ddfb9f31d88edf0b02299e65923e6945ae60ebcb5/ty-0.0.72-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:47dce65114fdc615c68ca0edb393b433df0956447e4267df0e264137a789598d", size = 13174832, upload-time = "2026-08-14T21:35:24.71Z" }, + { url = "https://files.pythonhosted.org/packages/1e/31/497f5a96c36d9b586ab6afe0574986835c6fd5b835a89773d2bec4711b49/ty-0.0.72-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:325144fa07e2675d0faa337fcc864213c272a499eb0cfe5bde2fdc62282d27bc", size = 12215005, upload-time = "2026-08-14T21:35:26.892Z" }, + { url = "https://files.pythonhosted.org/packages/df/7d/46e65b17b4966c7cd0140f134380d33d8e84fe6efccd761533ce793dc502/ty-0.0.72-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a5c9f15d0f58e43707d8848274be1821a0ef408eccb8aa7dda28a4a9eddf7640", size = 12421298, upload-time = "2026-08-14T21:35:29.301Z" }, + { url = "https://files.pythonhosted.org/packages/08/2a/12ada4ec17700b3cb1d4fd3bc3e5b1852df9e6885288429318cade87b3c1/ty-0.0.72-py3-none-musllinux_1_2_i686.whl", hash = "sha256:8ee508d64b381871529cc22c412b41071bf5e908b7aa5d66a38f3f6b2573a806", size = 12669242, upload-time = "2026-08-14T21:35:31.444Z" }, + { url = "https://files.pythonhosted.org/packages/1c/1a/4692536880790fb550ed6d44a6096778dc71bb112f2c6d615cebb01a57e5/ty-0.0.72-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:3699e2ec7921d44da79d6b089f7bf239b2cc53c4e45a5a38430adc34ee9e9a55", size = 12988199, upload-time = "2026-08-14T21:35:33.749Z" }, + { url = "https://files.pythonhosted.org/packages/9a/0d/f5e5a50322e9c45865e7b7a428ba6cd6527387cf0f2472492ac3cf746243/ty-0.0.72-py3-none-win32.whl", hash = "sha256:f25f72a67bd36cd247707c4784e52fad0b6b4f42a1b7dd14804110fa95c486ed", size = 11939708, upload-time = "2026-08-14T21:35:36.006Z" }, + { url = "https://files.pythonhosted.org/packages/3f/4e/8af3534b2e4214e6184a5a59c34101e94a68d578f081f97b995866bab1bf/ty-0.0.72-py3-none-win_amd64.whl", hash = "sha256:cdeee869341717e1736cea2e2d7856738c6957c320f584ed2f68c8f90100d2f5", size = 12643876, upload-time = "2026-08-14T21:35:38.141Z" }, + { url = "https://files.pythonhosted.org/packages/ff/ea/a2606e654c7276bd08586391a2525b0af3f3bf60228a8c57b2d248f273f9/ty-0.0.72-py3-none-win_arm64.whl", hash = "sha256:1bd3ac3ed4424a6d6990a85dc388556aea012bd752de21349a84b685951de0d8", size = 12394857, upload-time = "2026-08-14T21:35:40.277Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] From 16b710ca77d502265931d067fe77c27aaa4a137d Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Sat, 22 Aug 2026 00:16:16 +0000 Subject: [PATCH 3/3] e2e markers --- .github/workflows/e2e.yaml | 11 +-- pyproject.toml | 8 +- tests/e2e/conftest.py | 142 +++-------------------------------- tests/e2e/harness.py | 83 +++++++------------- tests/e2e/probe.py | 8 +- tests/e2e/scripts/guest.sh | 22 +++--- tests/e2e/test_docker.py | 22 ++---- tests/e2e/test_kubernetes.py | 39 +++------- tests/e2e/test_machine.py | 88 ++++++++++------------ 9 files changed, 117 insertions(+), 306 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 1b6e04f..026d8b3 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -30,7 +30,6 @@ jobs: env: # The suite itself, and with it the probe on this machine, runs on the version of the matrix. UV_PYTHON: ${{ matrix.python-version }} - E2E_REQUIRE: sudo,systemd E2E_INTERFACE: cgroup-v2 steps: - name: Checkout repository @@ -42,8 +41,8 @@ jobs: - name: Install dependencies run: uv run poe install-dev - # pytest directly, not `poe e2e-tests`: that task names the whole suite, and a path passed to it would - # be collected on top of the directory rather than instead of it. + # This runner is a plain cgroup v2 machine with systemd and passwordless sudo, so every machine test + # holds here and none is excluded. - name: Run the machine tests run: uv run pytest tests/e2e/test_machine.py @@ -52,9 +51,6 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 env: - # What this runner is here to prove. Without it a runner that lost docker would report a green job - # while every container test quietly skipped. - E2E_REQUIRE: docker,sudo,systemd E2E_INTERFACE: cgroup-v2 steps: - name: Checkout repository @@ -67,7 +63,6 @@ jobs: run: uv run poe install-dev - name: Run the container tests - # These parametrize the interpreter themselves, inside the containers, so they run once. run: uv run pytest tests/e2e/test_docker.py kubernetes: @@ -90,8 +85,6 @@ jobs: run: uv run poe install-dev - name: Run the kubernetes tests - env: - E2E_REQUIRE: kubernetes run: uv run pytest tests/e2e/test_kubernetes.py guest: diff --git a/pyproject.toml b/pyproject.toml index 5559eda..19ae82d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -100,11 +100,13 @@ inline-quotes = "single" known-first-party = ["cgroups_sensor"] [tool.pytest.ini_options] -# `--strict-markers` because the gates of the e2e suite are `usefixtures` marks: a mistyped one would run the -# test without its gate. `--strict-config` and `filterwarnings` fail on a typo here and on a deprecation in -# what the package is read through, which for a stdlib-only package is a real signal. addopts = "-r a --verbose --strict-markers --strict-config" filterwarnings = ["error"] +markers = [ + "unified: needs the resource controllers on the cgroup v2 unified hierarchy; cgroup v1 lanes exclude it", + "systemd_slices: needs docker to place containers under systemd slices; cgroup v1 lanes exclude it", + "kubernetes: needs kind and kubectl; only the kubernetes lane runs these", +] # The e2e readings carry a dozen fields, and a truncated one hides the number that explains the failure. verbosity_assertions = 2 diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 905c71a..f37f440 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -2,113 +2,24 @@ import os import subprocess -from pathlib import Path from typing import TYPE_CHECKING import pytest from .harness import ( - CAPABILITIES, IMAGE, MEMORY_LIMIT, - REQUIRED_CAPABILITIES, - TIMEOUT_SECONDS, - command_works, - have, - is_unified, - machine_cpu_count, - unavailable, + pull_image, ) if TYPE_CHECKING: from collections.abc import Callable, Iterator -TWO_CORES = 2 - - -def pytest_sessionstart() -> None: - """Reject a capability name this suite does not know, which would otherwise disarm the gate silently. - - The raise is what ends the run, with pytest's own usage-error code. Nothing reads `session.exitstatus` - this early, unlike in `pytest_sessionfinish` below, where setting it is the only way to fail a run. - """ - unknown = REQUIRED_CAPABILITIES - CAPABILITIES - if unknown: - raise pytest.UsageError(f'E2E_REQUIRE names {", ".join(sorted(unknown))}, not one of {sorted(CAPABILITIES)}') - - -@pytest.fixture(scope='session', autouse=True) -def _linux_only() -> None: - """Skip the whole suite where cgroups cannot exist.""" - if not Path('/proc/self/cgroup').exists(): - pytest.skip('cgroups exist on Linux only') - @pytest.fixture(scope='session') def _docker() -> None: - """Skip unless a working Docker is available, and pull the probe image once.""" - if not have('docker') or not command_works(['docker', 'info']): - unavailable('docker', 'docker is not available') - - subprocess.run( - ['docker', 'pull', '--quiet', IMAGE], - capture_output=True, - timeout=TIMEOUT_SECONDS, - check=False, - ) - - -@pytest.fixture -def _sudo() -> None: - """Skip unless sudo runs without asking for a password.""" - if not have('sudo') or not command_works(['sudo', '-n', 'true']): - unavailable('sudo', 'passwordless sudo is not available') - - -@pytest.fixture -def _systemd_user() -> None: - """Skip unless this user has a systemd manager that can hold a scope.""" - if not have('systemd-run') or not command_works(['systemd-run', '--user', '--scope', '-q', 'true']): - unavailable('systemd', 'a systemd user manager is not available') - - -@pytest.fixture -def _systemd_system() -> None: - """Skip unless a system scope can be started, which needs both sudo and a running systemd. - - The command is tried rather than the binary looked for: a machine can carry `systemd-run` and boot with - another init, and then every scope fails instead of skipping. - """ - if not have('systemd-run') or not command_works(['sudo', '-n', 'systemd-run', '--scope', '-q', 'true']): - unavailable('systemd', 'a system systemd manager is not available') - - -@pytest.fixture -def _unified() -> None: - """Skip unless the controllers live on the cgroup v2 unified hierarchy. - - Under cgroup v1 systemd delegates only `name=systemd` to user sessions, so a user scope gets no cgroup in - the resource controllers and its properties do nothing. - """ - if not is_unified(): - pytest.skip('the controllers are not on the unified hierarchy') - - -@pytest.fixture -def _systemd_cgroup_driver() -> None: - """Skip unless docker puts its containers under systemd slices. - - `--cgroup-parent` takes a slice name only under that driver. - """ - result = subprocess.run( - ['docker', 'info', '--format', '{{.CgroupDriver}}'], - capture_output=True, - text=True, - timeout=60, - check=False, - ) - if result.stdout.strip() != 'systemd': - pytest.skip(f'docker uses the {result.stdout.strip() or "unknown"!r} cgroup driver, this needs systemd') + """Pull the probe image once, before the tests that use it.""" + pull_image(IMAGE) @pytest.fixture @@ -120,18 +31,14 @@ def parent_slice() -> Iterator[str]: name = f'cgroups-sensor-e2e-{os.getpid()}.slice' holder = f'cgroups-sensor-e2e-holder-{os.getpid()}' - subprocess.run( + steps = ( ['sudo', 'systemd-run', '-q', '--unit', holder, '--slice', name, 'sleep', 'infinity'], - capture_output=True, - timeout=60, - check=True, - ) - subprocess.run( ['sudo', 'systemctl', 'set-property', '--runtime', name, f'MemoryMax={MEMORY_LIMIT}'], - capture_output=True, - timeout=60, - check=True, ) + for step in steps: + result = subprocess.run(step, capture_output=True, text=True, timeout=60, check=False) + if result.returncode != 0: + pytest.fail(f'{" ".join(step)} exited {result.returncode}\n{result.stderr.strip()}') yield name @@ -143,18 +50,11 @@ def parent_slice() -> Iterator[str]: ) -@pytest.fixture -def _two_cores() -> None: - """Skip where pinning one core would not restrict anything.""" - if machine_cpu_count() < TWO_CORES: - pytest.skip('a single-core machine cannot be restricted to one core') - - @pytest.fixture def systemd_scope() -> Iterator[Callable[..., list[str]]]: """Build `systemd-run` wrappers, and stop whatever scopes the test started. - A system scope needs sudo. Ask for it with `system=True`, together with the `_sudo` fixture. + A system scope needs sudo. Ask for it with `system=True`. """ units: list[tuple[str, bool]] = [] @@ -175,27 +75,3 @@ def build(*properties: str, system: bool = False) -> list[str]: for unit, system in units: stop = ['systemctl', '--user'] if not system else ['sudo', 'systemctl'] subprocess.run([*stop, 'stop', f'{unit}.scope'], capture_output=True, check=False, timeout=60) - - -_EXECUTED: list[str] = [] - - -def pytest_runtest_logreport(report: pytest.TestReport) -> None: - """Record the tests that ran, for the guard below.""" - if report.when == 'call' and report.passed: - _EXECUTED.append(report.nodeid) - - -def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: - """Fail a run in which nothing ran at all. - - Every test here skips when its environment is missing, so an all-skipped suite would exit 0 and prove - nothing. That is the one outcome this suite must never report as success. Collecting is not running, so a - plain `--collect-only` is left alone. - """ - if session.config.getoption('--collect-only'): - return - - if exitstatus == 0 and not _EXECUTED: - session.exitstatus = 1 - print('\nthe e2e suite proved nothing: every test skipped') diff --git a/tests/e2e/harness.py b/tests/e2e/harness.py index f503e03..f1e6064 100644 --- a/tests/e2e/harness.py +++ b/tests/e2e/harness.py @@ -3,13 +3,12 @@ import json import os import platform -import shutil import subprocess import sys import tarfile import tempfile from dataclasses import dataclass -from functools import lru_cache +from functools import cache from pathlib import Path from typing import Any @@ -48,31 +47,12 @@ TIMEOUT_SECONDS = 300 CANNOT_SET_UP = 77 -"""What a wrapper exits with when the environment refuses to produce the shape a test needs. +"""What a wrapper exits with when it could not produce the shape a test needs. -GNU's convention for "skipped". A test that arranges a cgroup layout before probing has to say which of the -two happened: the layout could not be built here, or it was built and the reading is wrong. Without this the -setup fails silently and the assertion afterwards blames the sensor for it. +Both fail the run, but differently: the layout could not be built, or it was built and the reading is wrong. +Without this the setup breaks silently and the assertion afterwards blames the sensor for it. """ -CAPABILITIES = frozenset({'docker', 'sudo', 'systemd', 'kubernetes'}) -"""What a lane can be told to prove. `E2E_REQUIRE` is checked against this, so a typo cannot quietly disarm it.""" - -REQUIRED_CAPABILITIES = frozenset(name for name in os.environ.get('E2E_REQUIRE', '').split(',') if name) -"""What this run must actually exercise, e.g. `E2E_REQUIRE=docker,systemd`. - -A test skips where its environment is missing, which is what makes one suite run everywhere. In CI that turns -a broken runner into a green job, so each lane names what it is there for and a missing capability fails. -""" - - -def unavailable(capability: str, reason: str) -> None: - """Skip because the environment cannot do this, or fail when this run was supposed to prove it.""" - if capability in REQUIRED_CAPABILITIES: - pytest.fail(f'{reason}, and E2E_REQUIRE names {capability}') - - pytest.skip(reason) - @dataclass(frozen=True) class Reading: @@ -106,9 +86,8 @@ def interfaces(self) -> set[str]: def limit_interfaces(self) -> set[str]: """The mechanisms the limits came from. - The consumed CPU time is left out on purpose. It comes from the hierarchy carrying the limits - wherever that hierarchy counts anything, but where nothing does, the base `cpu.stat` of a - controller-less cgroup2 is all there is - and that is not a reason to fail a lane. + The consumed CPU time is left out. Where no hierarchy counts anything, the base `cpu.stat` of a + controller-less cgroup2 is all there is. """ return { source['interface'] for name, source in self.sources.items() if name != 'cpu_usage' and source is not None @@ -165,7 +144,7 @@ def run(command: list[str], *, env: dict[str, str] | None = None) -> Reading: ) if result.returncode == CANNOT_SET_UP: - pytest.skip(f'the environment cannot set this up: {result.stderr.strip()[-300:]}') + pytest.fail(f'the layout this test needs could not be built: {result.stderr.strip()[-300:]}') if result.returncode != 0: pytest.fail( @@ -204,7 +183,7 @@ def probe_command(python_version: str = PYTHON_VERSION) -> str: return f'/uv/uv run --no-project --python {python_version} python /sensor/probe.py' -@lru_cache(maxsize=1) +@cache def portable_uv() -> Path: """Download the uv build that runs in any image, and hand back its path. @@ -224,7 +203,7 @@ def portable_uv() -> Path: check=False, ).stdout.split() if len(reported) < 2: - unavailable('docker', 'the uv version of this machine cannot be read') + pytest.fail('the uv version of this machine cannot be read') version = reported[1] @@ -234,7 +213,7 @@ def portable_uv() -> Path: UV_DOWNLOADS.mkdir(parents=True, exist_ok=True) archive = UV_DOWNLOADS / 'uv.tar.gz' if not command_works(['curl', '-fsSL', '-o', str(archive), url]): - unavailable('docker', f'{url} cannot be downloaded') + pytest.fail(f'{url} cannot be downloaded') with tarfile.open(archive) as tar: member = next(entry for entry in tar.getmembers() if entry.name.endswith('/uv')) @@ -253,9 +232,9 @@ def probe_in_container( command: list[str] | None = None, ) -> Reading: """Run the probe in a fresh container. The arguments go to `docker run`.""" - cache = UV_DOWNLOADS / 'cache' + uv_cache = UV_DOWNLOADS / 'cache' interpreters = UV_DOWNLOADS / 'python' - cache.mkdir(parents=True, exist_ok=True) + uv_cache.mkdir(parents=True, exist_ok=True) interpreters.mkdir(parents=True, exist_ok=True) return run( @@ -270,7 +249,7 @@ def probe_in_container( '--volume', f'{portable_uv()}:/uv/uv:ro', '--volume', - f'{cache}:/uv/cache', + f'{uv_cache}:/uv/cache', '--volume', f'{interpreters}:/uv/python', '--env', @@ -286,18 +265,13 @@ def probe_in_container( ) -def pull_image(image: str) -> bool: - """Pull one image, and report whether it arrived. +def pull_image(image: str) -> None: + """Pull one image ahead of the run that needs it, whether or not it arrives. - Pulling gets its own generous timeout: inside a virtual machine the network is slow enough that the - default one would report a working registry as a missing image. + Pulling separately gets its own generous timeout, which a slow guest network needs. A failure is no + verdict: the image may already be here, and where it is not, the `docker run` that follows says so. """ - return command_works(['docker', 'pull', '--quiet', image], timeout=TIMEOUT_SECONDS) - - -def have(tool: str) -> bool: - """Whether a command exists on this machine.""" - return shutil.which(tool) is not None + command_works(['docker', 'pull', '--quiet', image], timeout=TIMEOUT_SECONDS) def command_works(command: list[str], *, timeout: int = 60) -> bool: @@ -319,8 +293,8 @@ def command_works(command: list[str], *, timeout: int = 60) -> bool: def machine_cpu_count() -> int: """The cores of this machine, counted independently of the package. - The package answers this with `get_machine_cpu_count()`. This one deliberately does not use it: a test - that asks the subject for the expected value proves nothing. + Not the `get_machine_cpu_count()` of the package: a test that asks the subject for the expected value + proves nothing. """ return os.sysconf('SC_NPROCESSORS_ONLN') @@ -339,14 +313,13 @@ def machine_memory_bytes() -> int: if line.startswith('MemTotal:'): return int(line.split()[1]) * 1024 - pytest.skip('/proc/meminfo carries no MemTotal') + pytest.fail('/proc/meminfo carries no MemTotal') def notices_about(reading: Reading, metric: str) -> list[str]: """The notices that explain a dropped reading of one metric. - A notice of the other metric is no explanation, so the two are told apart here rather than at each call - site. The machine facts belong to the metric they were compared against. + A notice of the other metric is no explanation, so the two are told apart here. """ prefixes = {'memory': ('memory', 'machine-memory'), 'cpu': ('cpu',)}[metric] @@ -356,17 +329,14 @@ def notices_about(reading: Reading, metric: str) -> list[str]: def check_invariants(reading: Reading) -> None: """Check what must hold of every reading, whatever the environment. - Called by every test on top of its own assertions. A reported limit has to be real, and a missing one has - to be explained. + A reported limit has to be real, and a missing one has to be explained. """ - # The probe reads `__version__`, which the package resolves lazily on first access. Nothing else in the - # suite touches that path, and a source tree nothing installed reports `unknown` rather than nothing. + # `__version__` resolves lazily on first access, and nothing else in the suite touches that path. assert reading.version if reading.memory_limit is not None: assert reading.working_set is not None - # The probe charged this much anonymous memory to its own cgroup before reading, so a working set - # below it is not the memory of this process - a raw counter, a stale file, or another cgroup. + # The probe charged this much to its own cgroup first, so a smaller working set is not this process. assert reading.allocated <= reading.working_set <= reading.memory_limit assert reading.machine_memory_bytes is not None assert reading.memory_limit < reading.machine_memory_bytes @@ -388,8 +358,7 @@ def check_invariants(reading: Reading) -> None: if reading.cpu_used_ratio is None: assert 'cpu-usage-scope-mismatch' in reading.notices else: - # The probe keeps a core busy while it measures, so a rate of nothing means the time was counted - # somewhere this process is not. The level it was counted at is named next to the failure. + # The probe keeps a core busy while it measures, so a rate of nothing was counted elsewhere. assert 0.0 < reading.cpu_used_ratio <= 1.0 assert reading.cpu_rate_level is not None assert reading.cpu_limit_level is not None diff --git a/tests/e2e/probe.py b/tests/e2e/probe.py index 4557abd..85eea9a 100644 --- a/tests/e2e/probe.py +++ b/tests/e2e/probe.py @@ -13,16 +13,16 @@ ALLOCATION_BYTES = 32 * 1024 * 1024 """How much memory the probe charges to its own cgroup before reading, so that the working set has a floor. -Anonymous memory, touched: nothing about it is reclaimable file cache, so a working set below this would mean -the reading is not the memory of this process. +Anonymous and touched, so none of it is reclaimable file cache. A working set below this is not the memory of +this process. """ def burn(seconds: float) -> None: """Keep one core busy for a while. - The rate is the one reading that says nothing when it is measured in the wrong cgroup: an idle process - reads as zero everywhere, correct or not. So the probe makes itself busy while it measures. + An idle process reads a rate of zero in every cgroup, right or wrong. So the probe makes itself busy + while it measures. """ deadline = time.monotonic() + seconds while time.monotonic() < deadline: diff --git a/tests/e2e/scripts/guest.sh b/tests/e2e/scripts/guest.sh index 78e8aa2..f686abe 100755 --- a/tests/e2e/scripts/guest.sh +++ b/tests/e2e/scripts/guest.sh @@ -13,8 +13,9 @@ GUEST_USER=bench UBUNTU=https://cloud-images.ubuntu.com/releases/22.04/release FEDORA=https://download.fedoraproject.org/pub/fedora/linux/releases/43/Cloud/x86_64/images -# A profile is one machine to run the suite on. The suite skips whatever a machine cannot do, so each profile -# covers what it can: the Ubuntu ones exist for cgroup v1, the Fedora one for a second distribution. +# A profile is one machine to run the suite on: the Ubuntu ones exist for cgroup v1, the Fedora one for a +# second distribution. Nothing in the suite skips itself, so each profile says by name which tests its machine +# cannot carry, and everything else has to pass. case $PROFILE in ubuntu-v1-hybrid | ubuntu-v1-legacy) # cgroup v1 cannot be produced on a modern host, and systemd honours the flags below only up to v255. @@ -28,9 +29,10 @@ case $PROFILE in INITRD_URL=${GUEST_INITRD_URL:-$UBUNTU/unpacked/ubuntu-22.04-server-cloudimg-amd64-initrd-generic} PACKAGES=docker.io PREPARE='' - # The guest exists for cgroup v1. One that came up on v2 would run a smaller suite and still report - # success, so the suite is told what it has to find. - REQUIRE='docker,sudo' + # What cgroup v1 costs, and it is the point of this profile: user scopes get no resource controllers, and + # docker drives cgroups through cgroupfs, so a slice name means nothing to it. kind is not installed. + # Everything else - docker, sudo, systemd, two cores - this guest has, and a test needing it has to pass. + EXCLUDE='not unified and not systemd_slices and not kubernetes' INTERFACE=cgroup-v1 CGROUP_ARGS='systemd.unified_cgroup_hierarchy=0' [ "$PROFILE" = ubuntu-v1-legacy ] && CGROUP_ARGS="$CGROUP_ARGS systemd.legacy_systemd_cgroup_controller=1" @@ -44,7 +46,8 @@ case $PROFILE in KERNEL_URL='' INITRD_URL='' PACKAGES=moby-engine - REQUIRE='docker,sudo,systemd' + # A full cgroup v2 machine, so only the cluster tests are out: kind is not installed here. + EXCLUDE='not kubernetes' INTERFACE=cgroup-v2 # Only the test plumbing needs this: the suite bind-mounts the repository into containers, which SELinux # denies without relabelling. The sensor itself reads `/proc` and `/sys` and is not affected. @@ -206,10 +209,9 @@ run_suite() { local args='' [ "$#" -eq 0 ] || args=" ${*@Q}" - # The floors travel with the command. Without them the suite would skip whatever the guest cannot do and - # report success, which is exactly what these profiles exist to catch. - local floors="E2E_REQUIRE='$REQUIRE' E2E_INTERFACE='$INTERFACE'" - local suite="cd sensor && uv run poe install-dev && $floors uv run poe e2e-tests$args" + # The profile's exclusions travel with the command, and everything they leave has to pass inside the guest. + local suite="cd sensor && uv run poe install-dev" + suite="$suite && E2E_INTERFACE='$INTERFACE' uv run pytest tests/e2e -m ${EXCLUDE@Q}$args" local status=0 echo 'guest: running the e2e suite inside' diff --git a/tests/e2e/test_docker.py b/tests/e2e/test_docker.py index 18bd6b8..9be21dd 100644 --- a/tests/e2e/test_docker.py +++ b/tests/e2e/test_docker.py @@ -37,7 +37,6 @@ def test_memory_only() -> None: check_invariants(reading) assert reading.memory_limit == MEMORY_LIMIT - # The floor `check_invariants` applies is the memory the probe charged to this cgroup on purpose. assert reading.working_set is not None assert reading.working_set < reading.memory_limit assert reading.cpu_limit is None @@ -52,7 +51,6 @@ def test_cpu_quota() -> None: assert reading.memory_limit is None -@pytest.mark.usefixtures('_two_cores') def test_cpu_set_only() -> None: """Reads a set of allowed cores, which restricts the CPU without any quota.""" reading = probe_in_container('--cpuset-cpus', '0') @@ -63,7 +61,6 @@ def test_cpu_set_only() -> None: assert reading.raw_cpu_quota is None -@pytest.mark.usefixtures('_two_cores') def test_every_axis_at_once() -> None: """Takes the tighter of the two CPU axes. Here the quota is tighter than the set.""" reading = probe_in_container( @@ -88,9 +85,8 @@ def test_host_cgroup_namespace() -> None: check_invariants(reading) assert reading.memory_limit == MEMORY_LIMIT - # Under cgroup v2 the whole hierarchy is mounted, so the container sees its ancestry and the walk has - # several levels. Under cgroup v1 each controller is mounted at the container's own cgroup, which hides - # the ancestry whatever the namespace. + # Under cgroup v2 the whole hierarchy is mounted, so the container sees its ancestry. Under cgroup v1 + # each controller is mounted at the container's own cgroup, which hides it whatever the namespace. if reading.sources['memory']['interface'] == 'cgroup-v2': assert len(reading.sources['memory']['levels']) > 1 @@ -99,11 +95,10 @@ def test_tighter_limit_inside_the_container() -> None: """Takes the tightest limit when two levels carry different ones. The container gets one limit and the probe puts itself under a tighter one. That is the shape kubelet - produces, and the only one here where two levels disagree. + produces. """ # The kernel forbids a cgroup that both holds processes and delegates controllers, so the shell vacates - # the root first, then enables the controller, then moves into the tighter cgroup. `0` is how a process - # names itself to cgroupfs; an explicit PID takes the path meant for moving somebody else. + # the root first, enables the controller, then moves in. `0` is how a process names itself to cgroupfs. setup = f""" set -e if [ -e /sys/fs/cgroup/cgroup.controllers ]; then @@ -135,11 +130,10 @@ def test_tighter_limit_inside_the_container() -> None: def test_distributions(image: str) -> None: """Reads the same limits whatever distribution the process runs on. - The sensor only reads `/proc` and `/sys`, so the C library and the package layout should not matter. This - pins that: musl and glibc, and a distribution outside the Debian family. + The sensor only reads `/proc` and `/sys`, so the C library should not matter. It once did: under musl + `sysconf` reported the affinity instead of the machine. """ - if not pull_image(image): - pytest.skip(f'{image} cannot be pulled') + pull_image(image) reading = probe_in_container( '--memory', @@ -171,7 +165,7 @@ def test_python_versions(python_version: str) -> None: assert reading.cpu_limit == QUOTA_CORES -@pytest.mark.usefixtures('_sudo', '_systemd_cgroup_driver') +@pytest.mark.systemd_slices def test_limit_on_a_parent_cgroup(parent_slice: str) -> None: """Reads a limit set outside the container, on the slice the container was put into. diff --git a/tests/e2e/test_kubernetes.py b/tests/e2e/test_kubernetes.py index a39722a..d7c960d 100644 --- a/tests/e2e/test_kubernetes.py +++ b/tests/e2e/test_kubernetes.py @@ -14,9 +14,8 @@ SRC_DIR, TIMEOUT_SECONDS, check_invariants, - have, parse, - unavailable, + pull_image, ) if TYPE_CHECKING: @@ -24,6 +23,8 @@ from .harness import Reading +pytestmark = [pytest.mark.kubernetes, pytest.mark.usefixtures('_cluster')] + CLUSTER = 'cgroups-sensor-e2e' POD = 'sensor-probe' SRC_MAP = 'sensor-src' @@ -49,9 +50,6 @@ def kubectl(*args: str, check: bool = True) -> str: @pytest.fixture(scope='session') def _cluster() -> Iterator[None]: """Bring up a kind cluster carrying the probe image, and take it down afterwards.""" - if not have('kind') or not have('kubectl') or not have('docker'): - unavailable('kubernetes', 'kind, kubectl and docker are needed to run a cluster') - existing = subprocess.run( ['kind', 'get', 'clusters'], capture_output=True, @@ -70,14 +68,9 @@ def _cluster() -> Iterator[None]: check=False, ) if created.returncode != 0: - unavailable('kubernetes', f'the cluster could not be created: {created.stderr.strip()[-300:]}') + pytest.fail(f'the cluster could not be created: {created.stderr.strip()[-300:]}') - subprocess.run( - ['docker', 'pull', '--quiet', IMAGE], - capture_output=True, - timeout=TIMEOUT_SECONDS, - check=False, - ) + pull_image(IMAGE) subprocess.run( ['kind', 'load', 'docker-image', IMAGE, '--name', CLUSTER], capture_output=True, @@ -85,8 +78,8 @@ def _cluster() -> Iterator[None]: check=False, ) - # The package and the probe travel as config maps, so nothing has to be built into an image. The files go - # in one by one, or a stray `__pycache__` would travel with them. + # The package and the probe travel as config maps, so no image has to be built. The files go in one by + # one, or a stray `__pycache__` travels with them. sources = sorted((SRC_DIR / 'cgroups_sensor').glob('*.py')) kubectl('delete', 'configmap', SRC_MAP, PROBE_MAP, '--ignore-not-found') kubectl('create', 'configmap', SRC_MAP, *[f'--from-file={path}' for path in sources]) @@ -151,22 +144,14 @@ def probe_in_pod(resources: dict[str, Any]) -> Reading: if apply.returncode != 0: pytest.fail(f'the pod could not be created:\n{apply.stderr}') + # Only the phase is watched. `Unschedulable` is no verdict: the scheduler retries, and a pod that waits + # for the node or for the previous pod still runs. What it never got past is in the `describe` below. phase = '' for _ in range(POD_WAIT_SECONDS): phase = kubectl('get', f'pod/{POD}', '-o', 'jsonpath={.status.phase}', check=False).strip() if phase in {'Succeeded', 'Failed'}: break - reason = kubectl( - 'get', - f'pod/{POD}', - '-o', - 'jsonpath={.status.conditions[?(@.type=="PodScheduled")].reason}', - check=False, - ).strip() - if reason == 'Unschedulable': - pytest.skip('the pod does not fit this node') - time.sleep(1) logs = kubectl('logs', f'pod/{POD}', check=False) @@ -179,7 +164,6 @@ def probe_in_pod(resources: dict[str, Any]) -> Reading: return reading -@pytest.mark.usefixtures('_cluster') def test_pod_with_limits() -> None: """Reads container limits from inside the pod's own cgroup namespace. @@ -187,8 +171,8 @@ def test_pod_with_limits() -> None: """ reading = probe_in_pod( { - # Small requests on purpose: kubernetes copies limits into requests when none are given, and a pod - # requesting the whole node can never be scheduled. + # Small on purpose: kubernetes copies limits into requests when none are given, and a pod + # requesting the whole node never schedules. 'requests': {'cpu': '50m', 'memory': '64Mi'}, 'limits': {'cpu': '500m', 'memory': str(MEMORY_LIMIT)}, } @@ -199,7 +183,6 @@ def test_pod_with_limits() -> None: assert reading.cpu_limit == 0.5 -@pytest.mark.usefixtures('_cluster') def test_pod_without_limits() -> None: """Reports no restriction for a pod that sets none, inside the same kubepods hierarchy.""" reading = probe_in_pod({}) diff --git a/tests/e2e/test_machine.py b/tests/e2e/test_machine.py index 699aa7b..19fb55e 100644 --- a/tests/e2e/test_machine.py +++ b/tests/e2e/test_machine.py @@ -37,8 +37,7 @@ def test_unrestricted() -> None: def test_python_versions(python_version: str) -> None: """Reads the same machine facts on every supported interpreter. - The version is a real axis here. `os.cpu_count()` started honoring `PYTHON_CPU_COUNT` in 3.13, and the - filters compare against these numbers. + The version is a real axis: `os.cpu_count()` started honoring `PYTHON_CPU_COUNT` in 3.13. """ reading = probe_here(python_version=python_version) @@ -47,21 +46,20 @@ def test_python_versions(python_version: str) -> None: assert reading.machine_memory_bytes == machine_memory_bytes() -@pytest.mark.usefixtures('_systemd_user', '_unified') +@pytest.mark.unified def test_memory_limit(systemd_scope: Callable[..., list[str]]) -> None: """Reads a memory limit set on the scope this process runs in.""" reading = probe_here(systemd_scope(f'MemoryMax={MEMORY_LIMIT}')) check_invariants(reading) assert reading.memory_limit == MEMORY_LIMIT - # The floor `check_invariants` applies is the memory the probe charged to this cgroup on purpose. assert reading.working_set is not None assert reading.working_set < reading.memory_limit # The machine can carry unrelated notices, e.g. about a CPU set covering every core. assert not notices_about(reading, 'memory') -@pytest.mark.usefixtures('_systemd_user', '_unified') +@pytest.mark.unified def test_cpu_quota(systemd_scope: Callable[..., list[str]]) -> None: """Reads a CPU quota set on the scope this process runs in.""" reading = probe_here(systemd_scope(f'CPUQuota={int(QUOTA_CORES * 100)}%')) @@ -71,7 +69,7 @@ def test_cpu_quota(systemd_scope: Callable[..., list[str]]) -> None: assert reading.raw_cpu_quota == QUOTA_CORES -@pytest.mark.usefixtures('_systemd_user', '_unified') +@pytest.mark.unified def test_memory_limit_above_the_machine(systemd_scope: Callable[..., list[str]]) -> None: """Drops a memory limit larger than the machine, and says so. @@ -86,7 +84,7 @@ def test_memory_limit_above_the_machine(systemd_scope: Callable[..., list[str]]) assert 'memory-limit-covers-machine' in reading.notices -@pytest.mark.usefixtures('_systemd_user', '_unified') +@pytest.mark.unified def test_cpu_quota_above_the_machine(systemd_scope: Callable[..., list[str]]) -> None: """Drops a CPU quota larger than the machine, and says so. @@ -101,12 +99,11 @@ def test_cpu_quota_above_the_machine(systemd_scope: Callable[..., list[str]]) -> assert 'cpu-quota-covers-machine' in reading.notices -@pytest.mark.usefixtures('_sudo', '_systemd_system', '_two_cores') def test_every_axis_at_once(systemd_scope: Callable[..., list[str]]) -> None: """Reads all three limits at once. The tighter of the two CPU axes wins. - A system scope, not a user one: under cgroup v1 a user scope gets no cgroup in the resource controllers, - and `AllowedCPUs=` needs a delegated cpuset that only the unified hierarchy has. + A system scope, not a user one: under cgroup v1 a user scope gets no resource cgroup. `AllowedCPUs=` + needs a delegated cpuset, which only the unified hierarchy has. """ properties = [f'MemoryMax={MEMORY_LIMIT}', f'CPUQuota={int(QUOTA_CORES * 100)}%'] if is_unified(): @@ -122,44 +119,42 @@ def test_every_axis_at_once(systemd_scope: Callable[..., list[str]]) -> None: assert reading.raw_cpu_set_size == 1 -@pytest.mark.usefixtures('_systemd_user', '_unified') def test_limit_on_an_ancestor(systemd_scope: Callable[..., list[str]]) -> None: - """Reads a limit from an ancestor when the own cgroup carries no memory files at all. + """Reads a limit from an ancestor when the own cgroup carries no limit of its own. - `Delegate=yes` lets the probe move into a child cgroup. That child inherits no memory controller, so the - sensor has to walk up past a level with nothing to read. + The probe makes a child cgroup and moves into it, so the sensor has to walk up to find the limit. A system + scope, because under cgroup v1 a user scope gets no resource cgroup at all. """ - wrapper = systemd_scope(f'MemoryMax={MEMORY_LIMIT}', 'Delegate=yes') - move_into_leaf = [ - 'bash', - '-c', - ( - 'set -e; ' - 'own=$(grep "^0::" /proc/self/cgroup | cut -d: -f3); ' - 'leaf="/sys/fs/cgroup$own/leaf"; ' - # What a delegated scope is for. Where systemd delegates it differently - the Fedora guest, with a - # much newer one - the shape cannot be built, and this says so instead of probing the scope itself - # and blaming the reading. The errno of whichever step failed travels out on stderr. - # `0` is the kernel's word for "the writing process". An explicit PID takes the path meant for - # moving somebody else, which the kernel of Fedora 43 refuses with EINVAL where the older one of - # Ubuntu 22.04 allowed it. - f'{{ mkdir -p "$leaf" && echo 0 > "$leaf/cgroup.procs"; }} || {{ ' - # Where it still fails, say what the kernel refused and who owns what a delegated scope hands - # over - the two things that tell a missing delegation apart from a refused move. - 'echo "own=$own type=$(cat "/sys/fs/cgroup$own/cgroup.type" 2>&1)"' - ' "subtree=[$(cat "/sys/fs/cgroup$own/cgroup.subtree_control" 2>&1)]"' - ' "as=$(id -u):$(id -g)" "owner=$(stat -c %U:%G:%a "$leaf/cgroup.procs" 2>&1)" >&2; ' - f'exit {CANNOT_SET_UP}; }}; ' - 'exec "$@"' - ), - '--', - ] - reading = probe_here([*wrapper, *move_into_leaf]) + properties = [f'MemoryMax={MEMORY_LIMIT}'] + if is_unified(): + # Nothing can be created below a scope that is not delegated. cgroup v1 has no such rule. + properties.append('Delegate=yes') + + # Under cgroup v1 the controller is mounted at its own point, so the chain hangs off another path. `0` is + # how a process names itself to cgroupfs; an explicit PID takes the path for moving somebody else, which + # the kernel of Fedora 43 refuses with EINVAL. + move_into_leaf = f""" + set -e + if [ -e /sys/fs/cgroup/cgroup.controllers ]; then + leaf="/sys/fs/cgroup$(grep "^0::" /proc/self/cgroup | cut -d: -f3)/leaf" + else + leaf="/sys/fs/cgroup/memory$(grep ":memory:" /proc/self/cgroup | cut -d: -f3)/leaf" + fi + + if ! {{ mkdir -p "$leaf" && echo 0 > "$leaf/cgroup.procs"; }}; then + # Who we are and who owns the file. That tells a missing delegation from a refused move. + echo "leaf=$leaf as=$(id -u):$(id -g) owner=$(stat -c %U:%G:%a "$leaf/cgroup.procs" 2>&1)" >&2 + exit {CANNOT_SET_UP} + fi + + exec "$@" + """ + reading = probe_here([*systemd_scope(*properties, system=True), 'bash', '-c', move_into_leaf, '--']) check_invariants(reading) assert reading.memory_limit == MEMORY_LIMIT - # The walk starts at the leaf, which carries no memory files at all, and finds the limit above it. + # The walk starts at the leaf and finds the limit above it. levels = reading.sources['memory']['levels'] assert levels[0].endswith('/leaf') assert len(levels) > 1 @@ -170,15 +165,12 @@ def test_limit_on_an_ancestor(systemd_scope: Callable[..., list[str]]) -> None: def test_expected_interface() -> None: """Check that this run reached the cgroup interface it was started for. - A guest booted for cgroup v1 that comes up on v2 runs a smaller suite and reports success, which is the - one way this lane can lie. `E2E_INTERFACE=cgroup-v1` makes it say so instead. + A guest booted for cgroup v1 that comes up on v2 would run a smaller set and still report success. + `E2E_INTERFACE` names what to expect. Without it, this machine is asked what it is. - The limits are what has to come from that interface. The CPU counter is allowed to differ, for the reason - `Reading.limit_interfaces` gives. + Only the limits have to come from that interface, for the reason `Reading.limit_interfaces` gives. """ - expected = os.environ.get('E2E_INTERFACE') - if not expected: - pytest.skip('E2E_INTERFACE names no interface to check') + expected = os.environ.get('E2E_INTERFACE') or ('cgroup-v2' if is_unified() else 'cgroup-v1') reading = probe_here()