diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index 2283adf4..b463478c 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -188,7 +188,7 @@ def trace_open(ctx: c_void_p) -> c_int64: For network packet processing: ```python -from pythonbpf.helper import XDP_PASS +from vmlinux import XDP_PASS @section("xdp") def xdp_pass(ctx: c_void_p) -> c_int64: diff --git a/docs/user-guide/decorators.md b/docs/user-guide/decorators.md index ff8b2422..c884fc6c 100644 --- a/docs/user-guide/decorators.md +++ b/docs/user-guide/decorators.md @@ -108,7 +108,7 @@ def trace_open_return(ctx): For network packet processing at the earliest point: ```python -from pythonbpf.helper import XDP_PASS +from vmlinux import XDP_PASS from ctypes import c_void_p, c_int64 @section("xdp") diff --git a/docs/user-guide/structs.md b/docs/user-guide/structs.md index 5c68c23c..adee503d 100644 --- a/docs/user-guide/structs.md +++ b/docs/user-guide/structs.md @@ -290,7 +290,8 @@ class MyStruct: ```python from pythonbpf import bpf, struct, map, section from pythonbpf.maps import RingBuffer -from pythonbpf.helper import ktime, XDP_PASS +from pythonbpf.helper import ktime +from vmlinux import XDP_PASS from ctypes import c_void_p, c_int64, c_uint8, c_uint16, c_uint32, c_uint64 @bpf diff --git a/examples/snake.py b/examples/snake.py new file mode 100644 index 00000000..59aafaf8 --- /dev/null +++ b/examples/snake.py @@ -0,0 +1,462 @@ +# Snake, with the game in the kernel. +# +# A port of bpfsnake (github.com/amiremohamadi/bpfsnake), a bpftrace script. +# The eBPF side keeps the whole game state in maps and runs the game logic: +# steering, moving, collisions, eating and growing. Python only feeds it the +# arrow key and draws what the kernel left in the maps, in a pygame window or, +# with --terminal, in the terminal. +# +# bpftrace's `interval:ms:120` would be a perf_event program, which pylibbpf +# cannot attach yet. Instead Python drives the clock: every tick it calls +# getppid(), and a kprobe on that syscall, filtered to this process, advances +# the game one step. +# +# The original steered from a kprobe on pty_write, reading the byte a terminal +# echoes for each key. That cannot see keys typed into a pygame window, so +# Python writes the key's code into the `state` map instead, in both modes, +# using the original's codes: 'A' up, 'B' down, 'C' right, 'D' left. +# +# Run from the repository root: +# sudo -E env PYTHONPATH=. /path/to/python examples/snake.py # pygame +# sudo env PYTHONPATH=. /path/to/python examples/snake.py --terminal +# The pygame window needs pygame (pip install pygame), and -E keeps your display +# in the environment. The terminal draws the original's board, and like the +# original the game ends when the snake dies. +# Arrows or WASD steer, Space pauses, R restarts, Esc or Q quits. +# x86_64 only (the syscall name in the kprobe). + +import argparse +import math +import os +import select +import sys +import termios +import time +import tty +from collections import deque +from ctypes import c_int64, c_uint64, c_void_p + +from pythonbpf import BPF, bpf, bpfglobal, map, section +from pythonbpf.helper import pid, random +from pythonbpf.maps import HashMap + +MAX_LENGTH = 32 +WIDTH = 50 # columns, walls included +HEIGHT = 12 # rows, walls included +TICK_MS = 120 + +# Slots in the `state` map. The eBPF function spells them as literals: module +# constants are not visible inside it. +FOODX, FOODY, GAME_OVER, PLAYER, KEY = 0, 1, 2, 3, 4 +UP, DOWN, RIGHT, LEFT = 65, 66, 67, 68 + + +# Body segment i sits at row snakex[i], column snakey[i]; segment 0 is the head. +# A missing key reads as 0, which is a wall, so the body ends at the first 0. +@bpf +@map +def snakex() -> HashMap: + return HashMap(key=c_uint64, value=c_uint64, max_entries=32) + + +@bpf +@map +def snakey() -> HashMap: + return HashMap(key=c_uint64, value=c_uint64, max_entries=32) + + +@bpf +@map +def state() -> HashMap: + return HashMap(key=c_uint64, value=c_uint64, max_entries=5) + + +@bpf +@section("kprobe/__x64_sys_getppid") +def tick(ctx: c_void_p) -> c_int64: + if pid() != state.lookup(3): # PLAYER + return c_int64(0) + if state.lookup(2) != 0: # GAME_OVER + return c_int64(0) + + # Move: every segment takes the place of the one ahead of it + i = 0 + while i < 32 and snakex.lookup(i) != 0 and snakey.lookup(i) != 0: + i += 1 + i -= 1 + while i > 0: + snakex.update(i, snakex.lookup(i - 1)) + snakey.update(i, snakey.lookup(i - 1)) + i -= 1 + + key = state.lookup(4) # KEY + headx = snakex.lookup(0) + heady = snakey.lookup(0) + if key == 65: # up + snakex.update(0, headx - 1) + elif key == 66: # down + snakex.update(0, headx + 1) + elif key == 68: # left + snakey.update(0, heady - 1) + else: # right + snakey.update(0, heady + 1) + + # Game over when the head hits a wall ... + x = snakex.lookup(0) + y = snakey.lookup(0) + if x == 0 or x == 11 or y == 0 or y == 49: + state.update(2, 1) # GAME_OVER + + # ... or the snake's own body + i = 1 + while i < 32 and snakex.lookup(i) != 0 and snakey.lookup(i) != 0: + if x == snakex.lookup(i) and y == snakey.lookup(i): + state.update(2, 1) # GAME_OVER + i += 1 + + # Eat: new food somewhere inside the walls, and the tail grows by one + if x == state.lookup(0) and y == state.lookup(1): # FOODX, FOODY + state.update(0, random() % 10 + 1) + state.update(1, random() % 48 + 1) + i = 2 + while i < 32 and snakex.lookup(i) != 0 and snakey.lookup(i) != 0: + i += 1 + snakex.update(i, snakex.lookup(i - 1)) + snakey.update(i, snakey.lookup(i - 1)) + + return c_int64(0) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +# ---------------------------------------------------------------- userspace + +OPPOSITE = {UP: DOWN, DOWN: UP, LEFT: RIGHT, RIGHT: LEFT} + + +def read(m, k): + return m.lookup(k) or 0 + + +class Game: + """The userspace half: queue turns, drive the kernel's tick, read it back.""" + + def __init__(self, b): + self.b = b + self.restart() + + def restart(self): + b = self.b + for i in range(MAX_LENGTH): + for m in (b["snakex"], b["snakey"]): + try: + m.delete_elem(i) + except Exception: + pass # the key was never set + # BEGIN, as in the original + b["snakex"][0], b["snakey"][0] = 1, 1 + b["snakex"][1], b["snakey"][1] = 1, 2 + b["state"][FOODX], b["state"][FOODY] = 4, 4 + b["state"][GAME_OVER] = 0 + b["state"][KEY] = RIGHT + b["state"][PLAYER] = os.getpid() + self.heading, self.turns = RIGHT, deque(maxlen=3) + self.ticks, self.paused = 0, False + self.last_tick = time.monotonic() + self.snapshot() + + def turn(self, code): + self.turns.append(code) + + def toggle_pause(self): + if not self.over: + self.paused = not self.paused + + def due(self): + return time.monotonic() - self.last_tick >= TICK_MS / 1000 + + def tick(self): + """One step of the game, run by the kernel.""" + self.last_tick = time.monotonic() + if self.over or self.paused: + return + # One queued turn per tick; reversing onto your own neck is ignored + while self.turns: + turn = self.turns.popleft() + if turn not in (self.heading, OPPOSITE[self.heading]): + self.heading = turn + break + self.b["state"][KEY] = self.heading + os.getppid() # interval:ms:120 + self.ticks += 1 + self.snapshot() + + def snapshot(self): + b, body = self.b, [] + for i in range(MAX_LENGTH): + x, y = read(b["snakex"], i), read(b["snakey"], i) + if x == 0 or y == 0: + break + if not body or body[-1] != (x, y): # a fresh tail sits on the last one + body.append((x, y)) + self.body = body + self.food = (read(b["state"], FOODX), read(b["state"], FOODY)) + self.over = bool(read(b["state"], GAME_OVER)) + + def status(self): + n = len(self.body) + return f"length {n}/{MAX_LENGTH} score {n - 2} ticks {self.ticks}" + + +# ----------------------------------------------------------------- terminal + + +# Bytes a terminal sends: an arrow is ESC [ A..D (or ESC O A..D), and its last +# byte happens to be the code the kernel expects. +T_KEYS = {b"w": UP, b"s": DOWN, b"a": LEFT, b"d": RIGHT} + + +def terminal_frame(game): + """The board exactly as the bpftrace script prints it.""" + body = set(game.body) + rows = [] + for x in range(HEIGHT): + row = [] + for y in range(WIDTH): + if x == 0 or y == 0 or x == HEIGHT - 1 or y == WIDTH - 1: + row.append("#") + elif (x, y) == game.food: + row.append("$") + elif (x, y) in body: + row.append("@") + else: + row.append(" ") + rows.append("".join(row) + "\n") + return "\033[H" + "".join(rows) # move cursor to top left + + +def terminal_keys(data, game): + """Apply a burst of bytes read from the terminal. False means quit.""" + i = 0 + while i < len(data): + ch = data[i : i + 1] + if ch == b"\x1b" and data[i + 1 : i + 2] in (b"[", b"O"): + code = data[i + 2 : i + 3] + if code in (b"A", b"B", b"C", b"D"): + game.turn(code[0]) + i += 3 + continue + ch = ch.lower() + if ch in (b"q", b"\x03"): + return False + if ch == b" ": + game.toggle_pause() + elif ch == b"r": + game.restart() + elif ch in T_KEYS: + game.turn(T_KEYS[ch]) + i += 1 + return True + + +def run_terminal(game): + fd = sys.stdin.fileno() + saved = termios.tcgetattr(fd) + out = sys.stdout + try: + tty.setcbreak(fd) # keys arrive at once and are not echoed + out.write("\033[H\033[2J") # clear screen + while True: + out.write(terminal_frame(game)) + out.flush() + if game.over: + return # exit(), as in the original + wait = max(0.0, TICK_MS / 1000 - (time.monotonic() - game.last_tick)) + if select.select([fd], [], [], wait)[0]: + if not terminal_keys(os.read(fd, 64), game): + return + if game.due(): + game.tick() + finally: + termios.tcsetattr(fd, termios.TCSAFLUSH, saved) + + +# ------------------------------------------------------------------- pygame + +CELL = 24 +HUD = 56 + +BG = (15, 23, 42) +TILE = (22, 32, 54) +WALL = (51, 65, 85) +WALL_EDGE = (71, 85, 105) +HEAD = (74, 222, 128) +TAIL = (21, 128, 61) +FOOD = (244, 63, 94) +LEAF = (132, 204, 22) +TEXT = (226, 232, 240) +MUTED = (148, 163, 184) +ACCENT = (56, 189, 248) + + +def cell_rect(x, y, inset=0): + return pygame.Rect( + y * CELL + inset, HUD + x * CELL + inset, CELL - 2 * inset, CELL - 2 * inset + ) + + +def mix(a, b, t): + return tuple(round(a[i] + (b[i] - a[i]) * t) for i in range(3)) + + +def draw_board(screen): + screen.fill(BG) + for x in range(HEIGHT): + for y in range(WIDTH): + if x in (0, HEIGHT - 1) or y in (0, WIDTH - 1): + pygame.draw.rect(screen, WALL, cell_rect(x, y, 1), border_radius=5) + pygame.draw.rect( + screen, WALL_EDGE, cell_rect(x, y, 1), width=1, border_radius=5 + ) + elif (x + y) % 2: + pygame.draw.rect(screen, TILE, cell_rect(x, y)) + + +def draw_food(screen, food, now): + pulse = (math.sin(now / 180) + 1) / 2 + r = cell_rect(*food) + glow = pygame.Surface((CELL * 2, CELL * 2), pygame.SRCALPHA) + pygame.draw.circle( + glow, (*FOOD, int(40 + 50 * pulse)), (CELL, CELL), int(CELL * 0.7 + 4 * pulse) + ) + screen.blit(glow, (r.centerx - CELL, r.centery - CELL)) + pygame.draw.circle(screen, FOOD, r.center, CELL // 2 - 4) + pygame.draw.circle(screen, (255, 160, 180), (r.centerx - 3, r.centery - 3), 3) + pygame.draw.ellipse(screen, LEAF, (r.centerx, r.top + 1, 8, 5)) + + +def draw_snake(screen, body): + n = len(body) + for i in range(n - 1, -1, -1): + color = mix(HEAD, TAIL, i / max(n - 1, 1)) + # Bridge to the segment behind so the body reads as one piece + if i + 1 < n: + a, b = cell_rect(*body[i], 3), cell_rect(*body[i + 1], 3) + pygame.draw.rect(screen, color, a.union(b), border_radius=8) + inset = 2 if i == 0 else 3 + pygame.draw.rect(screen, color, cell_rect(*body[i], inset), border_radius=8) + if n >= 2: + (hx, hy), (nx, ny) = body[0], body[1] + dx, dy = hx - nx, hy - ny # rows, columns + else: + dx, dy = 0, 1 + cx, cy = cell_rect(*body[0]).center + for side in (-1, 1): + # Eyes sit ahead of centre, spread across the direction of travel. + # Screen x follows columns (dy) and screen y follows rows (dx). + ex = cx + dy * 3 + dx * side * 5 + ey = cy + dx * 3 + dy * side * 5 + pygame.draw.circle(screen, (255, 255, 255), (ex, ey), 4) + pygame.draw.circle(screen, BG, (ex + dy, ey + dx), 2) + + +def draw_hud(screen, fonts, game): + big, small = fonts + screen.blit(big.render("bpfsnake", True, TEXT), (16, 12)) + screen.blit(small.render("game logic runs in eBPF", True, MUTED), (160, 22)) + s = small.render(game.status(), True, ACCENT) + screen.blit(s, (WIDTH * CELL - s.get_width() - 16, 22)) + + +def draw_overlay(screen, fonts, title, hint): + big, small = fonts + shade = pygame.Surface(screen.get_size(), pygame.SRCALPHA) + shade.fill((2, 6, 23, 170)) + screen.blit(shade, (0, 0)) + t = big.render(title, True, TEXT) + h = small.render(hint, True, MUTED) + cx, cy = screen.get_width() // 2, screen.get_height() // 2 + screen.blit(t, (cx - t.get_width() // 2, cy - 30)) + screen.blit(h, (cx - h.get_width() // 2, cy + 12)) + + +def run_pygame(game): + pygame.init() + pygame.display.set_caption("bpfsnake") + screen = pygame.display.set_mode((WIDTH * CELL, HUD + HEIGHT * CELL)) + fonts = ( + pygame.font.SysFont("dejavusansmono,monospace", 26, bold=True), + pygame.font.SysFont("dejavusansmono,monospace", 15), + ) + keys = { + pygame.K_UP: UP, + pygame.K_w: UP, + pygame.K_DOWN: DOWN, + pygame.K_s: DOWN, + pygame.K_LEFT: LEFT, + pygame.K_a: LEFT, + pygame.K_RIGHT: RIGHT, + pygame.K_d: RIGHT, + } + clock = pygame.time.Clock() + try: + while True: + for event in pygame.event.get(): + if event.type == pygame.QUIT: + return + if event.type != pygame.KEYDOWN: + continue + if event.key in (pygame.K_ESCAPE, pygame.K_q): + return + if event.key == pygame.K_SPACE: + game.toggle_pause() + elif event.key == pygame.K_r: + game.restart() + elif event.key in keys: + game.turn(keys[event.key]) + if game.due(): + game.tick() + + draw_board(screen) + draw_food(screen, game.food, pygame.time.get_ticks()) + draw_snake(screen, game.body) + draw_hud(screen, fonts, game) + if game.over: + hint = f"length {len(game.body)} · R restart · Esc quit" + draw_overlay(screen, fonts, "game over", hint) + elif game.paused: + draw_overlay(screen, fonts, "paused", "Space resume") + pygame.display.flip() + clock.tick(60) + finally: + pygame.quit() + + +parser = argparse.ArgumentParser(description="Snake, with the game in eBPF.") +parser.add_argument( + "--terminal", + action="store_true", + help="draw in this terminal instead of a pygame window (no pygame needed)", +) +args = parser.parse_args() +if not args.terminal: + try: + import pygame + except ImportError: + sys.exit("snake.py: pygame is not installed; pip install pygame, or --terminal") + +b = BPF() +b.load() +b.attach_all() + +try: + if args.terminal: + run_terminal(Game(b)) + else: + run_pygame(Game(b)) +except KeyboardInterrupt: + pass diff --git a/examples/xdp_pass.py b/examples/xdp_pass.py index ea294fff..1f49f037 100644 --- a/examples/xdp_pass.py +++ b/examples/xdp_pass.py @@ -1,5 +1,5 @@ from pythonbpf import bpf, map, section, bpfglobal, compile, compile_to_ir -from pythonbpf.helper import XDP_PASS +from vmlinux import XDP_PASS from pythonbpf.maps import HashMap from ctypes import c_int64, c_void_p diff --git a/pyproject.toml b/pyproject.toml index 9307f626..3c82f196 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,7 @@ python_files = ["test_*.py"] markers = [ "verifier: requires sudo/root for kernel verifier tests (not run by default)", "vmlinux: requires vmlinux.py for current kernel", + "verifier_match: substring a verifier-level xfail expects in the rejection", ] log_cli = false diff --git a/pythonbpf/allocation_pass.py b/pythonbpf/allocation_pass.py index aaf25391..a927d252 100644 --- a/pythonbpf/allocation_pass.py +++ b/pythonbpf/allocation_pass.py @@ -6,8 +6,9 @@ from pythonbpf.helper import HelperHandlerRegistry from pythonbpf.vmlinux_parser.dependency_node import Field from .expr import VmlinuxHandlerRegistry -from pythonbpf.type_deducer import ctypes_to_ir, IntTy, signedness +from pythonbpf.type_deducer import ctypes_to_ir, is_ctypes, IntTy, signedness from pythonbpf.expr.type_inference import infer_int_type +from pythonbpf.expr.operators import usual_arithmetic_conversions from pythonbpf.maps import BPFMapType logger = logging.getLogger(__name__) @@ -49,57 +50,223 @@ def handle_assign_allocation(compilation_context, builder, stmt, local_sym_tab): ) continue - var_name = target.id + _bind_name( + compilation_context, + stmt, + target, + local_sym_tab, + lambda var_name: _allocate_for_value( + builder, var_name, rval, local_sym_tab, compilation_context + ), + ) - # Already bound in this scope: a parameter, an earlier assignment, or a - # `global` declaration (whose slot is the GlobalVariable). No slot needed. - if var_name in local_sym_tab: - logger.debug(f"'{var_name}' already bound, no allocation needed") - continue - # Not declared `global`, yet named like one: Python creates a local - # that shadows the global for the whole function body, and leaves the - # global untouched. Do the same. - shadows_global = var_name in compilation_context.bpf_globals - if shadows_global: - logger.info( - f"'{var_name}' is assigned without a 'global' declaration, so it " - f"is a local shadowing the @bpfglobal of the same name" - ) +def handle_ann_assign_allocation(compilation_context, builder, stmt, local_sym_tab): + """Handle memory allocation for annotated assignment (`x: c_int32 = 0`). - # Determine type and allocate based on rval - if isinstance(rval, ast.Call): - _allocate_for_call( - builder, var_name, rval, local_sym_tab, compilation_context - ) - elif isinstance(rval, ast.Constant): - _allocate_for_constant(builder, var_name, rval, local_sym_tab) - elif isinstance(rval, ast.BinOp): - _allocate_for_binop( - builder, var_name, rval, local_sym_tab, compilation_context - ) - elif isinstance(rval, ast.Name): - # Variable-to-variable assignment (b = a) - _allocate_for_name( - builder, var_name, rval, local_sym_tab, compilation_context - ) - elif isinstance(rval, ast.Attribute): - # Struct field-to-variable assignment (a = dat.fld) - _allocate_for_attribute( - builder, var_name, rval, local_sym_tab, compilation_context - ) - else: - logger.warning( - f"Unsupported assignment value type for {var_name}: {type(rval).__name__}" - ) + The annotation, not the value, types the slot: `x: c_int32 = 0` is an i32 + even though the literal alone would give an i64. With no value + (`x: c_int64`) the slot is still made, and the name stays unbound until + something assigns it, as in Python. + """ + logger.info(f"Handling annotated assignment for allocation: {ast.dump(stmt)}") - if shadows_global and var_name in local_sym_tab: - # Where the binding ends, so that a read above it is reported the - # way Python reports it. end_lineno, not lineno, so a read on a - # continuation line of a multi-line binding counts as above it too. - local_sym_tab[var_name].shadows_global_from = ( - getattr(stmt, "end_lineno", None) or target.lineno - ) + if not isinstance(stmt.target, ast.Name): + raise SyntaxError( + f"annotated assignment on line {stmt.lineno} must target a plain " + f"name, got {type(stmt.target).__name__}" + ) + + ir_type = annotation_to_ir(stmt.annotation, stmt.lineno) + _bind_name( + compilation_context, + stmt, + stmt.target, + local_sym_tab, + lambda var_name: local_sym_tab.__setitem__( + var_name, + LocalSymbol(_allocate_with_type(builder, var_name, ir_type), ir_type), + ), + ) + + +def handle_for_allocation(compilation_context, builder, stmt, local_sym_tab): + """Handle memory allocation for `for in range(...)`. + + Two slots, both in the entry block so a loop never grows the stack: the + loop variable, and a hidden induction counter the loop actually steps. + Keeping them apart is what makes rebinding the loop variable in the body + leave the trip count alone, as in Python -- and a counter the body cannot + touch is what keeps the loop visibly bounded for the verifier. + """ + start, stop, step = parse_range(stmt) + + # range() yields Python ints; like any undeclared local they are 64-bit, + # signed unless the bounds make C's arithmetic unsigned. + bound_types = [ + infer_int_type(bound, local_sym_tab, compilation_context) + for bound in (start, stop) + if bound is not None + ] + signed = True + if all(ty is not None for ty in bound_types): + common = bound_types[0] + for ty in bound_types[1:]: + common = usual_arithmetic_conversions(common, ty) + signed = signedness(common) + if not signed and step < 0: + raise SyntaxError( + f"range() on line {stmt.lineno} counts down over unsigned bounds; " + f"the counter would wrap instead of stopping" + ) + loop_ty = IntTy(64, signed) + + counter = range_counter_name(stmt) + local_sym_tab[counter] = LocalSymbol( + _allocate_with_type(builder, counter, loop_ty), loop_ty + ) + _bind_name( + compilation_context, + stmt, + stmt.target, + local_sym_tab, + lambda var_name: local_sym_tab.__setitem__( + var_name, + LocalSymbol(_allocate_with_type(builder, var_name, loop_ty), loop_ty), + ), + ) + + +def range_counter_name(stmt): + """Symbol-table name of a range loop's hidden induction counter, unique per + loop so nested loops each get their own.""" + return f"__range_idx_{stmt.lineno}_{stmt.col_offset}" + + +def parse_range(stmt): + """Split `for in range(...)` into (start, stop, step): start is an + expression or None (meaning 0), stop an expression, step a nonzero int. + + step has to be known at compile time, because its sign decides whether the + loop runs while the counter is below stop or above it. + """ + it = stmt.iter + if not ( + isinstance(it, ast.Call) + and isinstance(it.func, ast.Name) + and it.func.id == "range" + ): + raise NotImplementedError( + f"for loop on line {stmt.lineno}: only range(...) can be iterated " + f"so far, got {ast.unparse(it)}" + ) + if not isinstance(stmt.target, ast.Name): + raise NotImplementedError( + f"for loop on line {stmt.lineno}: only a plain name can be the loop " + f"variable, got {ast.unparse(stmt.target)}" + ) + if it.keywords or not 1 <= len(it.args) <= 3: + raise SyntaxError( + f"range() on line {stmt.lineno} takes 1 to 3 positional arguments" + ) + + if len(it.args) == 1: + return None, it.args[0], 1 + start, stop = it.args[0], it.args[1] + if len(it.args) == 2: + return start, stop, 1 + + step_node = it.args[2] + negate = isinstance(step_node, ast.UnaryOp) and isinstance(step_node.op, ast.USub) + literal = step_node.operand if negate else step_node + if not ( + isinstance(literal, ast.Constant) + and isinstance(literal.value, int) + and not isinstance(literal.value, bool) + ): + raise SyntaxError( + f"range() step on line {stmt.lineno} must be an integer literal, " + f"got {ast.unparse(step_node)}" + ) + step = -literal.value if negate else literal.value + if step == 0: + raise ValueError(f"range() arg 3 must not be zero (line {stmt.lineno})") + return start, stop, step + + +def annotation_to_ir(annotation, lineno): + """IR type for a ctypes annotation, written `c_int32` or `ctypes.c_int32`.""" + if isinstance(annotation, ast.Name): + name = annotation.id + elif isinstance(annotation, ast.Attribute): + name = annotation.attr + else: + name = None + if name is None or not is_ctypes(name): + raise SyntaxError( + f"unsupported annotation on line {lineno}: {ast.unparse(annotation)} " + f"(annotate locals with a ctypes integer type such as c_int64)" + ) + return ctypes_to_ir(name) + + +def _bind_name(compilation_context, stmt, target, local_sym_tab, allocate): + """What every statement that binds a bare name shares, around the + statement-specific `allocate(var_name)` that makes the slot. + + A name already bound needs no new slot. A name that is also a @bpfglobal + but was not declared `global` becomes a local shadowing it, and records + where its binding ends so a read above it is reported as Python would. + """ + var_name = target.id + + # Already bound in this scope: a parameter, an earlier assignment, or a + # `global` declaration (whose slot is the GlobalVariable). No slot needed. + if var_name in local_sym_tab: + logger.debug(f"'{var_name}' already bound, no allocation needed") + return + + # Not declared `global`, yet named like one: Python creates a local + # that shadows the global for the whole function body, and leaves the + # global untouched. Do the same. + shadows_global = var_name in compilation_context.bpf_globals + if shadows_global: + logger.info( + f"'{var_name}' is assigned without a 'global' declaration, so it " + f"is a local shadowing the @bpfglobal of the same name" + ) + + allocate(var_name) + + if shadows_global and var_name in local_sym_tab: + # Where the binding ends, so that a read above it is reported the + # way Python reports it. end_lineno, not lineno, so a read on a + # continuation line of a multi-line binding counts as above it too. + local_sym_tab[var_name].shadows_global_from = ( + getattr(stmt, "end_lineno", None) or target.lineno + ) + + +def _allocate_for_value(builder, var_name, rval, local_sym_tab, compilation_context): + """Allocate a slot for `var_name = rval`, typed from the value.""" + if isinstance(rval, ast.Call): + _allocate_for_call(builder, var_name, rval, local_sym_tab, compilation_context) + elif isinstance(rval, ast.Constant): + _allocate_for_constant(builder, var_name, rval, local_sym_tab) + elif isinstance(rval, ast.BinOp): + _allocate_for_binop(builder, var_name, rval, local_sym_tab, compilation_context) + elif isinstance(rval, ast.Name): + # Variable-to-variable assignment (b = a) + _allocate_for_name(builder, var_name, rval, local_sym_tab, compilation_context) + elif isinstance(rval, ast.Attribute): + # Struct field-to-variable assignment (a = dat.fld) + _allocate_for_attribute( + builder, var_name, rval, local_sym_tab, compilation_context + ) + else: + logger.warning( + f"Unsupported assignment value type for {var_name}: {type(rval).__name__}" + ) def _allocate_for_call(builder, var_name, rval, local_sym_tab, compilation_context): @@ -110,7 +277,9 @@ def _allocate_for_call(builder, var_name, rval, local_sym_tab, compilation_conte call_type = rval.func.id # C type constructors - if call_type in ("c_int32", "c_int64", "c_uint32", "c_uint64", "c_void_p"): + if is_ctypes(call_type) and isinstance(ctypes_to_ir(call_type), ir.IntType): + # Any integer ctypes constructor, c_uint16 included, declares a + # slot of that width; the value is converted into it at the store. ir_type = ctypes_to_ir(call_type) var = builder.alloca(ir_type, name=var_name) var.align = ir_type.width // 8 @@ -206,7 +375,7 @@ def _allocate_for_map_method( return map_params = map_sym_tab[map_name].params - if map_params["type"] != BPFMapType.HASH: + if map_params["type"] not in (BPFMapType.HASH, BPFMapType.ARRAY): logger.warning( "Map method lookup used on non-hash map, using fallback allocation" ) diff --git a/pythonbpf/assign_pass.py b/pythonbpf/assign_pass.py index f243d8ae..dc876768 100644 --- a/pythonbpf/assign_pass.py +++ b/pythonbpf/assign_pass.py @@ -5,7 +5,7 @@ from llvmlite import ir from pythonbpf.expr import eval_expr, convert from pythonbpf.helper import emit_probe_read_kernel_str_call -from pythonbpf.type_deducer import ctypes_to_ir +from pythonbpf.type_deducer import field_int_type from pythonbpf.vmlinux_parser.dependency_node import Field logger = logging.getLogger(__name__) @@ -40,6 +40,10 @@ def handle_struct_field_assignment( return val, val_type = val_result + if isinstance(val_type, Field): + field_ty = field_int_type(val_type) + if field_ty is not None: + val_type = field_ty # Special case: i8* string to [N x i8] char array if _is_char_array(field_type) and _is_i8_ptr(val_type): @@ -150,6 +154,13 @@ def handle_variable_assignment( logger.info( f"Evaluated value for {var_name}: {val} of type {val_type}, expected {var_type}" ) + # An integer vmlinux field is, for conversion purposes, its declared IntTy + # (width and sign from the ctype), so it takes the same convert() path as + # every other integer below instead of a special case. + if isinstance(val_type, Field): + field_ty = field_int_type(val_type) + if field_ty is not None: + val_type = field_ty if isinstance(val_type, ir.IntType) and isinstance(var_type, ir.IntType): # The descriptor may be narrower than the constant carrying the value @@ -194,33 +205,13 @@ def handle_variable_assignment( ) return False if isinstance(val_type, Field): - logger.info("Handling assignment to struct field") - field_ir_type = ctypes_to_ir(val_type.type.__name__) - # Sub-register-width context fields are zero-extended to i64 by - # load_ctx_field, so val is already i64 even though the field type - # says otherwise (c_uint for xdp_md, c_ushort for pt_regs.cs/ss). - if ( - isinstance(field_ir_type, ir.IntType) - and field_ir_type.width < 64 - and isinstance(var_type, ir.IntType) - and var_type.width == 64 - ): - builder.store(val, var_ptr) - logger.info( - f"Assigned zero-extended i{field_ir_type.width} context field " - f"to {var_name} (i64)" - ) - return True - # TODO: handling only ctype struct fields for now. Handle other stuff too later. - elif var_type == field_ir_type: - builder.store(val, var_ptr) - logger.info(f"Assigned ctype struct field to {var_name}") - return True - else: - logger.error( - f"Failed to assign ctype struct field to {var_name}: {val_type} != {var_type}" - ) - return False + # Integer fields were normalised to their IntTy above; what is + # left is a pointer, array or struct field, which has no path + # into this slot. + logger.error( + f"Failed to assign ctype struct field to {var_name}: {val_type} != {var_type}" + ) + return False elif isinstance(val_type, ir.IntType) and isinstance(var_type, ir.PointerType): # NOTE: This is assignment to a PTR_TO_MAP_VALUE_OR_NULL logger.info( diff --git a/pythonbpf/context.py b/pythonbpf/context.py index 9675d7c4..76b80812 100644 --- a/pythonbpf/context.py +++ b/pythonbpf/context.py @@ -82,7 +82,12 @@ def __init__(self, module: ir.Module): # Current function context (optional, if needed globally during function processing) self.current_func = None + # Enclosing loops of the statement being lowered, innermost last, as + # (continue target, break target) blocks. + self.loop_stack: list[tuple[ir.Block, ir.Block]] = [] + def reset(self): """Reset state between functions if necessary, though new context per compile is preferred.""" self.scratch_pool.reset() self.current_func = None + self.loop_stack = [] diff --git a/pythonbpf/debuginfo/debug_info_generator.py b/pythonbpf/debuginfo/debug_info_generator.py index 8dc31ee6..ca58e1e8 100644 --- a/pythonbpf/debuginfo/debug_info_generator.py +++ b/pythonbpf/debuginfo/debug_info_generator.py @@ -4,6 +4,7 @@ """ from . import dwarf_constants as dc +from llvmlite.ir.values import DIValue from typing import Any, List @@ -227,14 +228,25 @@ def create_local_variable_debug_info( Example: !DILocalVariable(name: "ctx", arg: 1, file: !3, line: 20, type: !7) """ - return self.module.add_debug_info( - "DILocalVariable", + # Not through add_debug_info, which returns a cached node for equal + # operands: a second function with the same parameter name would get + # the first one's node, already scoped to the first function by + # add_scope_to_local_variable. Scoping it again makes a metadata cycle + # that llvmlite recurses on forever when it hashes the next node. + operands = self.module._fix_di_operands( { "name": name, "arg": arg, "file": self.module._file_metadata, "type": var_type, - }, + }.items() + ) + return DIValue( + self.module, + False, + "DILocalVariable", + sorted(operands), + name=str(len(self.module.metadata)), ) def add_scope_to_local_variable(self, local_variable_debug_info, scope_value): diff --git a/pythonbpf/expr/expr_pass.py b/pythonbpf/expr/expr_pass.py index c454e704..19f0a6b8 100644 --- a/pythonbpf/expr/expr_pass.py +++ b/pythonbpf/expr/expr_pass.py @@ -254,6 +254,7 @@ def get_typed_operand(func, compilation_context, operand, builder, local_sym_tab vmlinux_result = VmlinuxHandlerRegistry.handle_name(operand.id) if vmlinux_result is not None: return vmlinux_result # (i64 constant, its C rank) + raise SyntaxError(f"Undefined variable {operand.id}") elif isinstance(operand, ast.Constant): if isinstance(operand.value, (int, bool)): v = int(operand.value) @@ -382,12 +383,57 @@ def _handle_ctypes_call( return value, expected_type +def _is_map_int_value(operand, compilation_context, local_sym_tab): + """Whether the operand is an integer held in a map: a `m.lookup(k)` call on + a map with an integer value, or a local bound to one. Both evaluate to a + pointer into the map, and comparing one means comparing the value.""" + if isinstance(operand, ast.Name): + sym = local_sym_tab.get(operand.id) + if sym is None or not isinstance(sym.var.type, ir.PointerType): + return False + _, depth = get_base_type_and_depth(sym.var.type) + return depth == 2 and isinstance(sym.metadata, str) and is_ctypes(sym.metadata) + if ( + isinstance(operand, ast.Call) + and isinstance(operand.func, ast.Attribute) + and operand.func.attr == "lookup" + and isinstance(operand.func.value, ast.Name) + ): + map_sym = compilation_context.map_sym_tab.get(operand.func.value.id) + if map_sym is None: + return False + value_type = map_sym.params.get("value") + return isinstance(value_type, str) and is_ctypes(value_type) + return False + + def _handle_compare(func, compilation_context, builder, cond, local_sym_tab): """Handle ast.Compare expressions.""" if len(cond.ops) != 1 or len(cond.comparators) != 1: logger.error("Only single comparisons are supported") return None + + if any( + _is_map_int_value(operand, compilation_context, local_sym_tab) + for operand in (cond.left, cond.comparators[0]) + ): + # A map value compares by what it holds (null-checked, 0 when the key + # is absent), never by address: `x == m.lookup(i)` with both sides + # pointers would otherwise compare two map slots' addresses. + lhs, lhs_ty = get_typed_operand( + func, compilation_context, cond.left, builder, local_sym_tab + ) + rhs, rhs_ty = get_typed_operand( + func, compilation_context, cond.comparators[0], builder, local_sym_tab + ) + cmp_ty = usual_arithmetic_conversions(lhs_ty, rhs_ty) + lhs = to_promoted(builder, lhs, lhs_ty, cmp_ty) + rhs = to_promoted(builder, rhs, rhs_ty, cmp_ty) + return handle_comparator( + func, builder, cond.ops[0], lhs, rhs, signed=signedness(cmp_ty) + ) + lhs = eval_expr( func, compilation_context, diff --git a/pythonbpf/functions/functions_pass.py b/pythonbpf/functions/functions_pass.py index 382ef38e..e9b456c7 100644 --- a/pythonbpf/functions/functions_pass.py +++ b/pythonbpf/functions/functions_pass.py @@ -25,12 +25,16 @@ ) from pythonbpf.allocation_pass import ( handle_assign_allocation, + handle_ann_assign_allocation, + handle_for_allocation, + parse_range, + range_counter_name, allocate_temp_pool, create_targets_and_rvals, LocalSymbol, ) from .function_debug_info import generate_function_debug_info -from .return_utils import handle_none_return, handle_xdp_return, is_xdp_name +from .return_utils import handle_none_return from .function_metadata import get_probe_string, is_global_function, infer_return_type @@ -81,10 +85,11 @@ def count_temps_in_call(call_node, local_sym_tab): return count -def handle_if_allocation( +def handle_block_allocation( compilation_context, builder, stmt, func, ret_type, local_sym_tab ): - """Recursively handle allocations in if/else branches.""" + """Recursively handle allocations in the body and else-branch of an + if, for or while statement.""" if stmt.body: allocate_mem( compilation_context, @@ -116,15 +121,23 @@ def merge_type_counts(count_dict): def update_max_temps_for_stmt(stmt): nonlocal max_temps_needed - if isinstance(stmt, ast.If): + if isinstance(stmt, (ast.If, ast.For, ast.While)): + # A loop header is evaluated like a statement of its own: range() + # bounds once before the loop, a while test on every iteration. + header = {ast.For: "iter", ast.While: "test"}.get(type(stmt)) + if header is not None: + count_temps_in_tree(getattr(stmt, header)) for s in stmt.body: update_max_temps_for_stmt(s) for s in stmt.orelse: update_max_temps_for_stmt(s) return + count_temps_in_tree(stmt) + + def count_temps_in_tree(tree): stmt_temps = {} - for node in ast.walk(stmt): + for node in ast.walk(tree): if isinstance(node, ast.Call): call_temps = count_temps_in_call(node, local_sym_tab) for typ, cnt in call_temps.items(): @@ -135,8 +148,10 @@ def update_max_temps_for_stmt(stmt): update_max_temps_for_stmt(stmt) # Handle allocations - if isinstance(stmt, ast.If): - handle_if_allocation( + if isinstance(stmt, ast.For): + handle_for_allocation(compilation_context, builder, stmt, local_sym_tab) + if isinstance(stmt, (ast.If, ast.For, ast.While)): + handle_block_allocation( compilation_context, builder, stmt, @@ -146,6 +161,10 @@ def update_max_temps_for_stmt(stmt): ) elif isinstance(stmt, ast.Assign): handle_assign_allocation(compilation_context, builder, stmt, local_sym_tab) + elif isinstance(stmt, ast.AnnAssign): + handle_ann_assign_allocation( + compilation_context, builder, stmt, local_sym_tab + ) allocate_temp_pool(builder, max_temps_needed, local_sym_tab) @@ -195,6 +214,24 @@ def handle_assign(func, compilation_context, builder, stmt, local_sym_tab): logger.error(f"Unsupported assignment target: {ast.dump(target)}") +def handle_ann_assign(func, compilation_context, builder, stmt, local_sym_tab): + """Handle `x: T = v`. The allocation pass already made x's slot with the + annotated type, so what is left is an ordinary store of v into it, through + the same helper plain assignment uses (which converts v to the slot's type). + A bare `x: T` binds nothing and emits nothing.""" + if stmt.value is None: + return + if not handle_variable_assignment( + func, + compilation_context, + builder, + stmt.target.id, + stmt.value, + local_sym_tab, + ): + logger.error(f"Failed to handle annotated assignment to {stmt.target.id}") + + def handle_aug_assign(func, compilation_context, builder, stmt, local_sym_tab): """Handle `x += v` and friends by direct lowering: resolve the target's slot, load it, apply the operator, store back. @@ -295,7 +332,7 @@ def handle_cond(func, compilation_context, builder, cond, local_sym_tab): return convert_to_bool(builder, val) -def handle_if(func, compilation_context, builder, stmt, local_sym_tab): +def handle_if(func, compilation_context, builder, stmt, local_sym_tab, ret_type): """Handle if statements in the function body.""" logger.info("Handling if statement") # start = builder.block.parent @@ -313,48 +350,181 @@ def handle_if(func, compilation_context, builder, stmt, local_sym_tab): builder.cbranch(cond, then_block, merge_block) builder.position_at_end(then_block) - for s in stmt.body: - process_stmt(func, compilation_context, builder, s, local_sym_tab, False) + process_block( + func, compilation_context, builder, stmt.body, local_sym_tab, ret_type + ) if not builder.block.is_terminated: builder.branch(merge_block) if else_block: builder.position_at_end(else_block) - for s in stmt.orelse: - process_stmt( - func, - compilation_context, - builder, - s, - local_sym_tab, - False, - ) + process_block( + func, compilation_context, builder, stmt.orelse, local_sym_tab, ret_type + ) if not builder.block.is_terminated: builder.branch(merge_block) builder.position_at_end(merge_block) +def _lower_loop( + func, + compilation_context, + builder, + stmt, + local_sym_tab, + ret_type, + body_block, + continue_block, + end_block, + else_block, +): + """What for and while share once their header is emitted: the body, with + `continue` and `break` bound to this loop, falling through to + continue_block; then the else-branch, which runs only when the loop ends + without a break, so it sits between the exit test and end_block.""" + builder.position_at_end(body_block) + compilation_context.loop_stack.append((continue_block, end_block)) + try: + process_block( + func, compilation_context, builder, stmt.body, local_sym_tab, ret_type + ) + finally: + compilation_context.loop_stack.pop() + if not builder.block.is_terminated: + builder.branch(continue_block) + + if else_block is not None: + # Outside this loop's scope: a break here leaves the enclosing loop. + builder.position_at_end(else_block) + process_block( + func, compilation_context, builder, stmt.orelse, local_sym_tab, ret_type + ) + if not builder.block.is_terminated: + builder.branch(end_block) + + builder.position_at_end(end_block) + + +def handle_while(func, compilation_context, builder, stmt, local_sym_tab, ret_type): + """Handle `while test: body [else: orelse]`. The test is re-evaluated at + the top of every iteration, and is where `continue` goes.""" + cond_block = func.append_basic_block(name="while.cond") + body_block = func.append_basic_block(name="while.body") + else_block = func.append_basic_block(name="while.else") if stmt.orelse else None + end_block = func.append_basic_block(name="while.end") + + builder.branch(cond_block) + builder.position_at_end(cond_block) + cond = handle_cond(func, compilation_context, builder, stmt.test, local_sym_tab) + builder.cbranch(cond, body_block, else_block or end_block) + + _lower_loop( + func, + compilation_context, + builder, + stmt, + local_sym_tab, + ret_type, + body_block, + cond_block, + end_block, + else_block, + ) + + +def handle_for(func, compilation_context, builder, stmt, local_sym_tab, ret_type): + """Handle `for name in range(...): body [else: orelse]`. + + The allocation pass made a hidden induction counter (typed from the + bounds) next to the loop variable. The bounds are evaluated once, before + the loop, as Python does; each iteration copies the counter into the loop + variable, and `continue` goes to the step, not straight back to the test. + """ + start, stop, step = parse_range(stmt) + counter = local_sym_tab[range_counter_name(stmt)] + loop_ty = counter.ir_type + + def bound(expr): + val, ty = get_typed_operand( + func, compilation_context, expr, builder, local_sym_tab + ) + if val is None or not isinstance(ty, ir.IntType): + raise SyntaxError( + f"range() bound on line {stmt.lineno} must be an integer: " + f"{ast.unparse(expr)}" + ) + return convert(builder, val, ty, loop_ty) + + start_val = ir.Constant(loop_ty, 0) if start is None else bound(start) + stop_val = bound(stop) + builder.store(start_val, counter.var) + + target = local_sym_tab[stmt.target.id] + if target.var is None: + raise SyntaxError( + f"cannot use '{stmt.target.id}' as a loop variable: it is the " + f"context parameter" + ) + + cond_block = func.append_basic_block(name="for.cond") + body_block = func.append_basic_block(name="for.body") + inc_block = func.append_basic_block(name="for.inc") + else_block = func.append_basic_block(name="for.else") if stmt.orelse else None + end_block = func.append_basic_block(name="for.end") + + builder.branch(cond_block) + builder.position_at_end(cond_block) + idx = builder.load(counter.var) + # Counting up runs while below stop, counting down while above it. + predicate = "<" if step > 0 else ">" + compare = builder.icmp_signed if signedness(loop_ty) else builder.icmp_unsigned + builder.cbranch( + compare(predicate, idx, stop_val), body_block, else_block or end_block + ) + + # The loop variable is bound to the counter's value, per iteration. + builder.position_at_end(body_block) + builder.store( + convert(builder, builder.load(counter.var), loop_ty, target.ir_type), + target.var, + ) + + builder.position_at_end(inc_block) + next_idx = builder.add(builder.load(counter.var), ir.Constant(loop_ty, step)) + builder.store(next_idx, counter.var) + builder.branch(cond_block) + + _lower_loop( + func, + compilation_context, + builder, + stmt, + local_sym_tab, + ret_type, + body_block, + inc_block, + end_block, + else_block, + ) + + +def handle_loop_jump(compilation_context, builder, stmt): + """Handle `break` and `continue`: branch to the innermost loop's exit or + next-iteration block.""" + keyword = "break" if isinstance(stmt, ast.Break) else "continue" + if not compilation_context.loop_stack: + raise SyntaxError(f"'{keyword}' outside loop (line {stmt.lineno})") + continue_block, break_block = compilation_context.loop_stack[-1] + builder.branch(break_block if keyword == "break" else continue_block) + + def handle_return( func, builder, stmt, local_sym_tab, ret_type, compilation_context=None ): logger.info(f"Handling return statement: {ast.dump(stmt)}") if stmt.value is None: return handle_none_return(builder) - elif ( - isinstance(stmt.value, ast.Name) - and is_xdp_name(stmt.value.id) - and stmt.value.id not in local_sym_tab - and ( - compilation_context is None - or stmt.value.id not in compilation_context.bpf_globals - ) - ): - # The XDP fast path resolves names like XDP_PASS from the helper - # constant table, but only as a fallback: a local or @bpfglobal of the - # same name shadows it, mirroring C (a local shadows an enum constant) - # and the resolution order everywhere else in the compiler. - return handle_xdp_return(stmt, builder, ret_type) else: # Fallback for now if ctx not passed, but caller should pass it if compilation_context is None: @@ -397,12 +567,20 @@ def process_stmt( ) elif isinstance(stmt, ast.Assign): handle_assign(func, compilation_context, builder, stmt, local_sym_tab) + elif isinstance(stmt, ast.AnnAssign): + handle_ann_assign(func, compilation_context, builder, stmt, local_sym_tab) elif isinstance(stmt, ast.AugAssign): handle_aug_assign(func, compilation_context, builder, stmt, local_sym_tab) elif isinstance(stmt, ast.Global): logger.debug(f"global declaration of {', '.join(stmt.names)} already bound") elif isinstance(stmt, ast.If): - handle_if(func, compilation_context, builder, stmt, local_sym_tab) + handle_if(func, compilation_context, builder, stmt, local_sym_tab, ret_type) + elif isinstance(stmt, ast.While): + handle_while(func, compilation_context, builder, stmt, local_sym_tab, ret_type) + elif isinstance(stmt, ast.For): + handle_for(func, compilation_context, builder, stmt, local_sym_tab, ret_type) + elif isinstance(stmt, (ast.Break, ast.Continue)): + handle_loop_jump(compilation_context, builder, stmt) elif isinstance(stmt, ast.Return): did_return = handle_return( func, builder, stmt, local_sym_tab, ret_type, compilation_context @@ -417,6 +595,19 @@ def process_stmt( return did_return +def process_block(func, compilation_context, builder, stmts, local_sym_tab, ret_type): + """Process a nested statement list (an if-branch or loop body). Stops at + the first statement that ends the block -- break, continue or return -- + because whatever follows it in the same list can never run, and would + otherwise be emitted after a terminator.""" + for s in stmts: + if builder.block.is_terminated: + break + process_stmt( + func, compilation_context, builder, s, local_sym_tab, False, ret_type + ) + + # ============================================================================ # SECTION 3: Function Body Processing # ============================================================================ diff --git a/pythonbpf/functions/return_utils.py b/pythonbpf/functions/return_utils.py index a05c704d..5979fa61 100644 --- a/pythonbpf/functions/return_utils.py +++ b/pythonbpf/functions/return_utils.py @@ -1,44 +1,12 @@ import logging -import ast from llvmlite import ir logger: logging.Logger = logging.getLogger(__name__) -XDP_ACTIONS = { - "XDP_ABORTED": 0, - "XDP_DROP": 1, - "XDP_PASS": 2, - "XDP_TX": 3, - "XDP_REDIRECT": 4, -} - def handle_none_return(builder) -> bool: """Handle return or return None -> returns 0.""" builder.ret(ir.Constant(ir.IntType(64), 0)) logger.debug("Generated default return: 0") return True - - -def is_xdp_name(name: str) -> bool: - """Check if a name is an XDP action""" - return name in XDP_ACTIONS - - -def handle_xdp_return(stmt: ast.Return, builder, ret_type) -> bool: - """Handle XDP returns""" - if not isinstance(stmt.value, ast.Name): - return False - - action_name = stmt.value.id - - if action_name not in XDP_ACTIONS: - raise ValueError( - f"Unknown XDP action: {action_name}. Available: {XDP_ACTIONS.keys()}" - ) - - value = XDP_ACTIONS[action_name] - builder.ret(ir.Constant(ret_type, value)) - logger.debug(f"Generated XDP action return: {action_name} = {value}") - return True diff --git a/pythonbpf/helper/__init__.py b/pythonbpf/helper/__init__.py index bd4fe174..d7bdbb7a 100644 --- a/pythonbpf/helper/__init__.py +++ b/pythonbpf/helper/__init__.py @@ -11,6 +11,7 @@ deref, comm, probe_read_str, + probe_read_kernel, random, probe_read, smp_processor_id, @@ -18,8 +19,6 @@ skb_store_bytes, get_current_cgroup_id, get_stack, - XDP_DROP, - XDP_PASS, ) @@ -80,12 +79,11 @@ def helper_call_handler(call, compilation_context, builder, func, local_sym_tab) "deref", "comm", "probe_read_str", + "probe_read_kernel", "random", "probe_read", "smp_processor_id", "uid", "skb_store_bytes", "get_stack", - "XDP_DROP", - "XDP_PASS", ] diff --git a/pythonbpf/helper/bpf_helper_handler.py b/pythonbpf/helper/bpf_helper_handler.py index 92bb1307..afa0acdf 100644 --- a/pythonbpf/helper/bpf_helper_handler.py +++ b/pythonbpf/helper/bpf_helper_handler.py @@ -651,10 +651,21 @@ def bpf_probe_read_kernel_emitter( f"probe_read_kernel expects 2 args (dst, src), got {len(call.args)}" ) - # Get destination buffer (char array -> i8*) - dst_ptr, dst_size = get_or_create_ptr_from_arg( + # Destination: a char array reads its length, a scalar reads its width + dst = get_or_create_ptr_from_arg( func, compilation_context, call.args[0], builder, local_sym_tab ) + if isinstance(dst, tuple): + dst_ptr, dst_size = dst + else: + dst_ptr = dst + pointee = dst_ptr.type.pointee + if not isinstance(pointee, ir.IntType): + raise ValueError( + f"probe_read_kernel destination must be a char array or an " + f"integer, got {pointee}" + ) + dst_size = pointee.width // 8 # Get source pointer (evaluate expression) src_ptr, src_type = get_ptr_from_arg( diff --git a/pythonbpf/helper/helper_utils.py b/pythonbpf/helper/helper_utils.py index 28b60b34..bb5e518d 100644 --- a/pythonbpf/helper/helper_utils.py +++ b/pythonbpf/helper/helper_utils.py @@ -341,6 +341,12 @@ def get_ptr_from_arg(arg, func, compilation_context, builder, local_sym_tab): val, val_type = result + # An address held in a 64-bit integer, such as a pt_regs register field: + # the C spelling is (void *)ctx->si + if isinstance(val.type, ir.IntType) and val.type.width == 64: + ptr_type = ir.PointerType(ir.IntType(8)) + return builder.inttoptr(val, ptr_type), ptr_type + if not isinstance(val_type, ir.PointerType): raise ValueError(f"Expected pointer type, got {val_type}") diff --git a/pythonbpf/helper/helpers.py b/pythonbpf/helper/helpers.py index 253c4b08..57519abc 100644 --- a/pythonbpf/helper/helpers.py +++ b/pythonbpf/helper/helpers.py @@ -27,6 +27,11 @@ def probe_read_str(dst, src): return ctypes.c_int64(0) +def probe_read_kernel(dst, src): + """Safely read sizeof(dst) bytes from kernel memory""" + return ctypes.c_int64(0) + + def random(): """get a pseudorandom u32 number""" return ctypes.c_int32(0) @@ -60,10 +65,3 @@ def get_stack(buf, flags=0): def get_current_cgroup_id(): """Get the current cgroup ID""" return ctypes.c_int64(0) - - -XDP_ABORTED = ctypes.c_int64(0) -XDP_DROP = ctypes.c_int64(1) -XDP_PASS = ctypes.c_int64(2) -XDP_TX = ctypes.c_int64(3) -XDP_REDIRECT = ctypes.c_int64(4) diff --git a/pythonbpf/maps/__init__.py b/pythonbpf/maps/__init__.py index eb2007da..fb64f8de 100644 --- a/pythonbpf/maps/__init__.py +++ b/pythonbpf/maps/__init__.py @@ -1,5 +1,12 @@ -from .maps import HashMap, PerfEventArray, RingBuffer +from .maps import ArrayMap, HashMap, PerfEventArray, RingBuffer from .maps_pass import maps_proc from .map_types import BPFMapType -__all__ = ["HashMap", "PerfEventArray", "maps_proc", "RingBuffer", "BPFMapType"] +__all__ = [ + "ArrayMap", + "HashMap", + "PerfEventArray", + "maps_proc", + "RingBuffer", + "BPFMapType", +] diff --git a/pythonbpf/maps/maps.py b/pythonbpf/maps/maps.py index 583e9570..12cd3a48 100644 --- a/pythonbpf/maps/maps.py +++ b/pythonbpf/maps/maps.py @@ -26,6 +26,26 @@ def update(self, key, value, flags=None): raise KeyError(f"Key {key} not found in map") +class ArrayMap: + def __init__(self, key, value, max_entries): + self.key = key + self.value = value + self.max_entries = max_entries + self.entries = {} + + def lookup(self, key): + return self.entries.get(key) + + def update(self, key, value, flags=None): + self.entries[key] = value + + def delete(self, key): + if key in self.entries: + del self.entries[key] + else: + raise KeyError(f"Key {key} not found in map") + + class PerfEventArray: def __init__(self, key_size, value_size): self.key_type = key_size diff --git a/pythonbpf/maps/maps_pass.py b/pythonbpf/maps/maps_pass.py index 362b34b8..ae0d2202 100644 --- a/pythonbpf/maps/maps_pass.py +++ b/pythonbpf/maps/maps_pass.py @@ -143,6 +143,26 @@ def process_hash_map(map_name, rval, compilation_context): return map_global +@MapProcessorRegistry.register("ArrayMap") +def process_array_map(map_name, rval, compilation_context): + """Process a BPF_ARRAY map declaration: the same lowering as a hash map + with BPF_MAP_TYPE_ARRAY, through the same lookup/update/delete helpers; + the kernel requires a 4-byte key (an index).""" + logger.info(f"Processing ArrayMap: {map_name}") + map_params = _parse_map_params(rval, expected_args=["key", "value", "max_entries"]) + map_params["type"] = BPFMapType.ARRAY + + logger.info(f"Map parameters: {map_params}") + map_global = create_bpf_map(compilation_context, map_name, map_params) + create_map_debug_info( + compilation_context, + map_global.var, + map_name, + map_params, + ) + return map_global + + @MapProcessorRegistry.register("PerfEventArray") def process_perf_event_map(map_name, rval, compilation_context): """Process a BPF_PERF_EVENT_ARRAY map declaration""" diff --git a/tests/README.md b/tests/README.md index 6b63fd45..d972939e 100644 --- a/tests/README.md +++ b/tests/README.md @@ -88,6 +88,51 @@ All xfails use `strict = True`: if a test starts **passing** it shows up as **XP 2. Run `make test` — the file is discovered and tested automatically at all levels. 3. If the test is expected to fail, add it to `tests/test_config.toml` instead of `passing_tests/`. +## Kernel selftest equivalents + +`tests/kernel_selftest_equivalent/` contains PythonBPF versions of important +kernel BPF selftests from `bpf-next/tools/testing/selftests/bpf`. Each file names +its upstream original in a header comment. + +The directory holds two kinds of test, and both are useful: + +- **Ports that pass.** A program PythonBPF can already express. These widen the + range of program types under test — `raw_tp`, `perf_event`, + `tracepoint/sched/*` and others that nothing else exercises. +- **Roadmap tests that fail.** A program describing a feature PythonBPF should + grow next. These must be listed as **strict** expected failures in + `tests/test_config.toml` until the feature lands, at which point they turn up + as XPASS and should be promoted. + +### What a passing port proves — and does not + +A kernel selftest is two halves: the BPF program under `progs/`, and a userspace +driver under `prog_tests/` that loads it through a skeleton, triggers it and +asserts on the result. **Only the BPF half is ported**, because this framework +compiles and verifies programs but never runs them. + +So a passing test here says PythonBPF emits a loadable, verifiable object for +that program type and feature mix. It does not say the program behaves the way +the kernel's version does. Treat it as a compiler assertion, not a semantic one. + +### Globals + +The selftest corpus overwhelmingly reports results through global variables: the +program writes a global and the driver reads it back. Ports keep that shape with +`@bpfglobal` scalars, so a future userspace driver reads the same `.bss`/`.data` +values the kernel's driver does. Anything a global cannot yet hold (arrays, +structs, strings) is a roadmap test, not a workaround. + +The one substitution still in use is `WORKAROUND(atomics)`: upstream counters +incremented with `__sync_fetch_and_add` are plain `x += 1` here, tagged on the +line so the sweep is mechanical once atomics land. `PORTING-NOTES.md` in that +directory records every port, its rewrite if any, and why the rest of the +corpus is out of reach; `tools/selftest-audit.py` regenerates that scoring. + +Anything importing from `vmlinux` belongs in `vmlinux/`, which is registered in +`VMLINUX_TEST_DIRS_PASSING` so it is skipped rather than failed where no +`vmlinux.py` has been generated. + ## Directory structure ``` @@ -104,5 +149,6 @@ tests/ │ ├── compiler.py ← wrappers around compile_to_ir() + _run_llc() │ └── verifier.py ← bpftool subprocess wrapper ├── passing_tests/ ← programs that should compile and verify cleanly -└── failing_tests/ ← programs with known issues (declared in test_config.toml) +├── failing_tests/ ← programs with known issues (declared in test_config.toml) +└── kernel_selftest_equivalent/ ← ports of kernel selftests + feature roadmap tests ``` diff --git a/tests/c-form/loops.bpf.c b/tests/c-form/loops.bpf.c new file mode 100644 index 00000000..e66be7ee --- /dev/null +++ b/tests/c-form/loops.bpf.c @@ -0,0 +1,97 @@ +/* Reference for loops. One program per shape the Python tests use, written + * the way PythonBPF lowers them: a `for i in range(...)` keeps a hidden + * induction counter separate from `i`, so rebinding `i` in the body cannot + * change the trip count. The constant-bound cases fold to a `ret` at -O2; + * `dyn_helper` reads its bound from a .data global and calls a helper per + * iteration, so a real loop (and the verifier's bounded-loop check) survives. + * (A helper-free sum over a runtime bound folds to a closed form that needs + * the __multi3 libcall, which BPF lacks -- so it is not a useful reference.) */ +#define SEC(name) __attribute__((section(name), used)) +typedef unsigned int __u32; +typedef unsigned long long __u64; +typedef long long __s64; + +char LICENSE[] SEC("license") = "GPL"; + +__s64 n = 10; + +/* for i in range(10): total = total + 1 */ +SEC("tracepoint/syscalls/sys_enter_execve") +__s64 range_sum(void *ctx) +{ + __s64 total = 0; + for (__s64 idx = 0; idx < 10; idx += 1) { + __s64 i = idx; + total = total + 1; + } + return total; +} + +/* for i in range(0, 10, 2): total = total + i */ +SEC("tracepoint/syscalls/sys_enter_execve") +__s64 range_step(void *ctx) +{ + __s64 total = 0; + for (__s64 idx = 0; idx < 10; idx += 2) { + __s64 i = idx; + total = total + i; + } + return total; +} + +/* while i < 10: i = i + 1 */ +SEC("tracepoint/syscalls/sys_enter_execve") +__s64 while_basic(void *ctx) +{ + __s64 i = 0; + while (i < 10) + i = i + 1; + return i; +} + +/* while True: total += 1; if total == 10: break */ +SEC("tracepoint/syscalls/sys_enter_execve") +__s64 while_true_break(void *ctx) +{ + __s64 total = 0; + while (1) { + total = total + 1; + if (total == 10) + break; + } + return total; +} + +/* for i in range(10): if i % 2 == 0: continue; total = total + i */ +SEC("tracepoint/syscalls/sys_enter_execve") +__s64 for_continue(void *ctx) +{ + __s64 total = 0; + for (__s64 idx = 0; idx < 10; idx += 1) { + __s64 i = idx; + if (i % 2 == 0) + continue; + total = total + i; + } + return total; +} + +static __u32 (*bpf_get_prandom_u32)(void) = (void *)7; + +/* for i in range(stop): total = total + random() -- a helper call per + * iteration keeps the loop from folding, so the verifier sees a real loop. + * The clamp is what makes it bounded: n is writable from userspace, so + * unclamped the verifier walks iterations until E2BIG. */ +SEC("tracepoint/syscalls/sys_enter_execve") +__s64 dyn_helper(void *ctx) +{ + __s64 total = 0; + __s64 stop = n; + if (stop > 64) + stop = 64; + for (__s64 idx = 0; idx < stop; idx += 1) { + __s64 i = idx; + total = total + bpf_get_prandom_u32(); + } + return total; +} diff --git a/tests/conftest.py b/tests/conftest.py index 42ab30ed..f70dee2d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,6 +16,7 @@ """ import logging +import warnings import pytest @@ -25,11 +26,19 @@ # ── vmlinux availability ──────────────────────────────────────────────────── try: - import vmlinux # noqa: F401 + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + import vmlinux # noqa: F401 VMLINUX_AVAILABLE = True -except ImportError: + VMLINUX_SKIP_REASON = "" +except ImportError as exc: + # No vmlinux.py: the tests that need it are skipped. Any other exception + # propagates. A vmlinux.py that exists but does not import is a defect in + # the generator, and hiding it behind skips would pass CI with no vmlinux + # coverage at all. VMLINUX_AVAILABLE = False + VMLINUX_SKIP_REASON = f"vmlinux.py not importable: {exc}" # ── pytest_generate_tests: parametrize on bpf_test_file ─────────────────── @@ -65,7 +74,10 @@ def pytest_collection_modifyitems(items): # vmlinux skip if case.needs_vmlinux and not VMLINUX_AVAILABLE: item.add_marker( - pytest.mark.skip(reason="vmlinux.py not available for current kernel") + pytest.mark.skip( + reason=VMLINUX_SKIP_REASON + or "vmlinux.py not available for current kernel" + ) ) continue @@ -91,6 +103,10 @@ def pytest_collection_modifyitems(items): raises=Exception, ) ) + # A verifier-level xfail may name the rejection it expects; + # any other rejection is then a real failure, not an xfail. + if item_level == "verifier" and case.xfail_match: + item.add_marker(pytest.mark.verifier_match(case.xfail_match)) # ── caplog level fixture: capture ERROR+ from pythonbpf ─────────────────── diff --git a/tests/failing_tests/loops/break_outside_loop.py b/tests/failing_tests/loops/break_outside_loop.py new file mode 100644 index 00000000..3208f915 --- /dev/null +++ b/tests/failing_tests/loops/break_outside_loop.py @@ -0,0 +1,20 @@ +from pythonbpf import bpf, section, bpfglobal, compile +from ctypes import c_void_p, c_int64 + + +@bpf +@section("tracepoint/syscalls/sys_enter_execve") +def hello(ctx: c_void_p) -> c_int64: + total: c_int64 = 0 + if total == 0: + break # noqa: F701 -- the point of the test + return total + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/failing_tests/loops/range_variable_step.py b/tests/failing_tests/loops/range_variable_step.py new file mode 100644 index 00000000..1e94f8f9 --- /dev/null +++ b/tests/failing_tests/loops/range_variable_step.py @@ -0,0 +1,21 @@ +from pythonbpf import bpf, section, bpfglobal, compile +from ctypes import c_void_p, c_int64 + + +@bpf +@section("tracepoint/syscalls/sys_enter_execve") +def hello(ctx: c_void_p) -> c_int64: + total: c_int64 = 0 + step: c_int64 = 2 + for i in range(0, 10, step): + total = total + i + return total + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/failing_tests/direct_assign.py b/tests/failing_tests/vmlinux/direct_assign.py similarity index 96% rename from tests/failing_tests/direct_assign.py rename to tests/failing_tests/vmlinux/direct_assign.py index a7843133..f3362b40 100644 --- a/tests/failing_tests/direct_assign.py +++ b/tests/failing_tests/vmlinux/direct_assign.py @@ -1,5 +1,5 @@ from pythonbpf import bpf, map, section, bpfglobal, compile -from pythonbpf.helper import XDP_PASS +from vmlinux import XDP_PASS from pythonbpf.maps import HashMap from ctypes import c_void_p, c_int64 diff --git a/tests/failing_tests/named_arg.py b/tests/failing_tests/vmlinux/named_arg.py similarity index 95% rename from tests/failing_tests/named_arg.py rename to tests/failing_tests/vmlinux/named_arg.py index 19139df8..bea5e22f 100644 --- a/tests/failing_tests/named_arg.py +++ b/tests/failing_tests/vmlinux/named_arg.py @@ -1,5 +1,5 @@ from pythonbpf import bpf, map, section, bpfglobal, compile -from pythonbpf.helper import XDP_PASS +from vmlinux import XDP_PASS from pythonbpf.maps import HashMap from ctypes import c_void_p, c_int64 diff --git a/tests/failing_tests/xdp_pass.py b/tests/failing_tests/vmlinux/xdp_pass.py similarity index 97% rename from tests/failing_tests/xdp_pass.py rename to tests/failing_tests/vmlinux/xdp_pass.py index c8510dcd..b37a9b82 100644 --- a/tests/failing_tests/xdp_pass.py +++ b/tests/failing_tests/vmlinux/xdp_pass.py @@ -1,6 +1,6 @@ from pythonbpf import bpf, map, section, bpfglobal, compile_to_ir from pythonbpf.maps import HashMap -from pythonbpf.helper import XDP_PASS +from vmlinux import XDP_PASS from vmlinux import TASK_COMM_LEN # noqa: F401 from vmlinux import struct_qspinlock # noqa: F401 diff --git a/tests/framework/bpf_test_case.py b/tests/framework/bpf_test_case.py index a993166e..cc1609bf 100644 --- a/tests/framework/bpf_test_case.py +++ b/tests/framework/bpf_test_case.py @@ -23,6 +23,7 @@ class BpfTestCase: is_expected_fail: bool = False xfail_reason: str = "" xfail_level: str = "ir" # one of LEVELS + xfail_match: str = "" # verifier level: substring the rejection must contain needs_vmlinux: bool = False skip_reason: str = "" diff --git a/tests/framework/collector.py b/tests/framework/collector.py index bdafc149..2d5287d6 100644 --- a/tests/framework/collector.py +++ b/tests/framework/collector.py @@ -7,7 +7,10 @@ TESTS_DIR = Path(__file__).parent.parent CONFIG_FILE = TESTS_DIR / "test_config.toml" -VMLINUX_TEST_DIRS_PASSING = {"passing_tests/vmlinux"} +VMLINUX_TEST_DIRS_PASSING = { + "passing_tests/vmlinux", + "kernel_selftest_equivalent/vmlinux", +} VMLINUX_TEST_DIRS_FAILING = { "failing_tests/vmlinux", "failing_tests/xdp", @@ -33,7 +36,7 @@ def collect_all_test_files() -> list[BpfTestCase]: xfail_map: dict = config.get("xfail", {}) cases = [] - for subdir in ("passing_tests", "failing_tests"): + for subdir in ("passing_tests", "failing_tests", "kernel_selftest_equivalent"): for py_file in sorted((TESTS_DIR / subdir).rglob("*.py")): if py_file.name == "vmlinux.py": # Not a test case: the per-directory symlink to the master @@ -46,6 +49,7 @@ def collect_all_test_files() -> list[BpfTestCase]: is_expected_fail = xfail_entry is not None xfail_reason = xfail_entry.get("reason", "") if xfail_entry else "" xfail_level = xfail_entry.get("level", "ir") if xfail_entry else "ir" + xfail_match = xfail_entry.get("match", "") if xfail_entry else "" cases.append( BpfTestCase( @@ -53,6 +57,7 @@ def collect_all_test_files() -> list[BpfTestCase]: rel_path=rel, is_expected_fail=is_expected_fail, xfail_reason=xfail_reason, + xfail_match=xfail_match, xfail_level=xfail_level, needs_vmlinux=needs_vmlinux, ) diff --git a/tests/kernel_selftest_equivalent/PORTING-NOTES.md b/tests/kernel_selftest_equivalent/PORTING-NOTES.md new file mode 100644 index 00000000..5aa69455 --- /dev/null +++ b/tests/kernel_selftest_equivalent/PORTING-NOTES.md @@ -0,0 +1,226 @@ +# Porting kernel selftests: what the first spike found + +Four programs from `tools/testing/selftests/bpf/progs/` were ported as an experiment, +to answer two questions before anyone commits to doing this at scale: + +1. Is LLM-assisted porting of kernel selftests viable? +2. What must real global-variable support actually handle? + +Short answers: **viable, with a caveat about what a passing port proves**; and +**four distinct global shapes showed up in four programs**, which is the more +actionable finding. + +## The spike + +| Port | Upstream | Section | Outcome | +|---|---|---|---| +| `tracing/tracepoint_sched_switch.py` | `test_tracepoint.c` | `tracepoint/sched/sched_switch` | passes | +| `tracing/get_cgroup_id.py` | `get_cgroup_id_kern.c` | `tracepoint/syscalls/sys_enter_nanosleep` | passes | +| `tracing/autoattach.py` | `test_autoattach.c` | `raw_tp/sys_enter`, `raw_tp/sys_exit` | passes | +| `vmlinux/perf_skip.py` | `test_perf_skip.c` | `perf_event` | strict xfail — nested ctx access | + +## 1. Is it viable? + +**Yes, for programs inside the envelope — three of four compiled and passed `llc` on the +first attempt.** The mechanical part of a port (decorators, ctypes annotations, map +declarations, helper names) is regular enough to be reliable. + +The failure was not a translation error. `perf_skip` needs `ctx.regs.ip`, which PythonBPF +genuinely cannot express, and no amount of care in the port changes that. That is the +useful kind of failure: it converts into a roadmap test that documents the gap. + +Two caveats that matter more than the pass rate: + +**A passing port proves less than the test it came from.** A kernel selftest is two +halves — the BPF program, and a `prog_tests/` driver that loads it through a skeleton, +triggers it, and asserts on the result. Only the BPF half is portable here, because this +framework compiles and verifies but never runs. Everything ported becomes a compiler +assertion: *PythonBPF emits a loadable, verifiable object for this program type and +feature mix*. That is worth having — it is how the `raw_tp` and `perf_event` program types +came under test at all — but it is not what "we ported the kernel's selftests" sounds +like. Closing that gap needs a runtime test tier, which is a much larger piece of work. + +**Selection is the expensive step, not translation.** Of 820 real programs, 28 are +portable today. Picking those out required scoring the whole corpus against the compiler's +actual envelope; guessing from filenames does not work. The classifier that did it is +worth keeping around and re-running after each feature lands. + +**Recommendation: viable and worth continuing, in small increments tied to features.** +Port a handful, let them reveal the next gap, fix the gap, port more. Bulk porting ahead of +the features would just produce a large pile of xfails. + +## 2. What real globals must support + +At the time of the spike every port that touched a global substituted a one-entry +`HashMap`, tagged `WORKAROUND(globals)`. Integer-scalar `@bpfglobal` support has since +landed and the sweep is done: the three ports below now declare the upstream globals +directly. Four programs produced four distinct shapes: + +| Shape | Example | What globals must support | Status | +|---|---|---|---| +| none | `tracepoint_sched_switch` | — (control case) | passes | +| scalar in + scalar out | `get_cgroup_id` | read a global, write a different one | passes with `@bpfglobal` | +| flags across programs | `autoattach` | two programs in one object sharing global state | passes with `@bpfglobal` | +| scalar in, compared against ctx | `perf_skip` | read-only input set by userspace before attach | global fine; still xfail on `ctx.regs.ip` | +| array + cursor *(next increment)* | `cgroup_preorder` | indexed writes and read-modify-write on a global | needs array globals | + +The last row is not in this spike but is the recommended next port precisely because it is +the most demanding shape: `result[idx++] = N` needs an array global *and* a read-modify-write +cursor, which together constrain the design more than anything here does. + +### A design note worth acting on + +**libbpf implements global variables as single-element `BPF_MAP_TYPE_ARRAY` maps.** +`.bss`, `.data` and `.rodata` become internal array maps at load time. Two consequences: + +- A one-element **`ArrayMap`** is the structurally faithful stand-in for a global, not a + `HashMap`. The `HashMap` stand-ins predate `ArrayMap`, which lowers now + (`BPF_MAP_TYPE_ARRAY`, the same helpers as `HashMap`); real `@bpfglobal` scalars have + landed since, so the migration is to those. +- **Most of the ELF work is already done.** `@bpfglobal` is vestigial — a metadata carrier + for `LICENSE` — but the machinery behind it already emits globals that LLVM places into + `.bss` and `.data` correctly, and that libbpf already recognises: + + ``` + libbpf: map 'g.bss' (global data): at sec_idx 5, offset 0, flags 0. + libbpf: map 'g.data' (global data): at sec_idx 6, offset 0, flags 0. + ``` + + What is missing is narrower than "implement global variables": name resolution in + `expr_pass.get_operand_value` (which resolves against `local_sym_tab`, then vmlinux + enums, then gives up), a Python-level surface for declaring one, and userspace access + through `pylibbpf`. + +## Second batch: everything portable after globals + +With scalar globals in, the audit's Tier 1 and Tier 2 lists were re-read against the +compiler and every program it can express was ported. Sixteen more, fifteen of which +pass at every level; the sixteenth passes at IR and llc and is rejected by the verifier +exactly as its upstream driver asserts it must be: + +| Port | Upstream | Section | Outcome | +|---|---|---|---| +| `xdp/xdp_dummy.py` | `xdp_dummy.c` | `xdp` x2 | passes | +| `xdp/priv_prog.py` | `priv_prog.c` | `xdp` | passes | +| `xdp/xdp_link.py` | `test_xdp_link.c` | `xdp`, `tc` | passes | +| `vmlinux/xdp_tx.py` | `xdp_tx.c` | `xdp` | passes | +| `tc/tc_dummy.py` | `tc_dummy.c` | `tc` | passes | +| `vmlinux/tc_bpf.py` | `test_tc_bpf.c` | `tc`, `tcx/ingress` | passes (direct packet access) | +| `cgroup/cgroup_mprog.py` | `cgroup_mprog.c` | `cgroup/getsockopt` x4 | passes | +| `vmlinux/cgroup_skb_direct_packet_access.py` | `cgroup_skb_direct_packet_access.c` | `cgroup_skb/ingress` | passes, after a compiler fix | +| `socket/signed_loader.py` | `test_signed_loader.c` | `socket` | passes | +| `socket/signed_loader_data.py` | `test_signed_loader_data.c` | `socket` | passes (.data global) | +| `netfilter/netfilter_link_attach.py` | `test_netfilter_link_attach.c` | `netfilter` | passes | +| `tracing/kprobe_multi_empty.py` | `kprobe_multi_empty.c` | `kprobe.multi/` | passes | +| `tracing/uprobe_multi_bench.py` | `uprobe_multi_bench.c` | `uprobe.multi/...` | passes (`count += 1`) | +| `tracing/uprobe_multi_usdt.py` | `uprobe_multi_usdt.c` | `usdt` | passes | +| `tracing/link_pinning.py` | `test_link_pinning.c` | `raw_tp/sys_enter`, `tp_btf/sys_enter` | passes | +| `vmlinux/xdp_devmap_helpers.py` | `test_xdp_devmap_helpers.c` | `xdp` | verifier xfail by design | + +**One compiler bug fell out.** `data_end = skb->data_end` into a `__u32` global failed +with `cannot store i64 to i32*`: context fields are loaded widened to i64, and the +assignment path only accepted them into 64-bit slots or slots of exactly the field's +type, never narrowing. It now goes through `convert()` like every other integer store. +`passing_tests/vmlinux/ctx_field_narrow_store.py` pins it. + +**What is still not portable, and why**, from the same two lists: + +| Program | Blocker | +|---|---| +| `metadata_used.c`, `metadata_unused.c` | `char[]` `.rodata` globals: only integer scalars can be globals | +| `test_log_buf.c`, `cgroup_preorder.c`, `uprobe_multi_pid_filter.c`, `test_build_id.c` | array globals | +| `token_kallsyms.c`, `test_btf_ext.c`, `test_static_linked*.c` | BPF-to-BPF calls (`__weak` / `__noinline` subprogs) | +| `test_trace_ext.c`, `freplace_get_constant.c` | `freplace` needs a target program to load against | +| `test_subskeleton*.c` | extern symbols, `__kconfig`, static linking | +| `test_pkt_md_access.c` | narrow type-punned loads of `__sk_buff` fields | +| `test_xdp_attach_fail.c` | tracepoint `__data_loc` pointer arithmetic on a custom ctx struct | +| `sockopt_multi.c` | writes to context fields and through `optval` | +| `tracing_struct_many_args.c` | `BPF_PROG2` multi-argument entry | +| `bpf_nop_bench.c` | `bpf_loop`-based benchmark macro | +| `test_tcp_estats.c` | large; inlinable helpers and struct-heavy, not attempted yet | + +## Third batch: what the re-run audit found + +`tools/selftest-audit.py` is the corpus classifier, rebuilt and checked in. Run against +the current upstream `progs/` it reports 854 real programs, 25 with no hard blocker. All +but two of those 25 were already ported or are unportable for a reason a regex cannot see +(`bpf_nop_bench.c` hides a loop in a macro, `test_pkt_md_access.c` type-puns narrow loads, +`tracing_struct_int128.c` indexes the raw ctx array and needs bpf_testmod to load). The +programs with exactly one blocker were read by hand for anything a documented rewrite +could absorb. Five more ports, all passing at every level: + +| Port | Upstream | Section | Rewrite | +|---|---|---|---| +| `socket/veristat_foo.py` | `veristat_foo.c` | `socket` x3 | none | +| `tracing/perf_link.py` | `test_perf_link.c` | `perf_event` | `WORKAROUND(atomics)` | +| `tracing/enable_stats.py` | `test_enable_stats.c` | `raw_tracepoint/sys_enter` | `WORKAROUND(atomics)` | +| `cgroup/cgroup_link.py` | `test_cgroup_link.c` | `cgroup_skb/egress` x2 | `WORKAROUND(atomics)` | +| `vmlinux/connect4_dropper.py` | `connect4_dropper.c` | `cgroup/connect4` | `bpf_htons` written as shifts | + +### `WORKAROUND(atomics)` + +Three upstream programs count with `__sync_fetch_and_add(&x, 1)`. PythonBPF has no atomic +operations, so the ports do `x += 1`, a plain read-modify-write, and tag the line. As with +the earlier globals tag this is scaffolding for a mechanical sweep once atomics land: + +```bash +grep -rn "WORKAROUND(atomics)" tests/kernel_selftest_equivalent/ +``` + +It is not a cosmetic substitution: the upstream drivers run these programs from many +CPUs at once and the exact count matters there, which is precisely what a non-atomic +increment loses. + +### The blocker histogram now + +Over 854 real programs, hard blockers only; a program usually hits several: + +| Blocker | Programs | Share | +|---|---|---| +| unsupported helper | 504 | 59% | +| unsupported map type | 273 | 32% | +| kfuncs | 242 | 28% | +| verifier-test annotations | 238 | 28% | +| typed program macros (`BPF_PROG`, `BPF_KPROBE`) | 237 | 28% | +| `goto` | 143 | 17% | +| inline asm | 142 | 17% | +| BPF-to-BPF calls | 128 | 15% | +| struct globals | 126 | 15% | +| CO-RE reads | 116 | 14% | +| loops | 110 | 13% | +| array globals | 109 | 13% | +| atomics | 87 | 10% | + +Earlier revisions of this table over-counted kfuncs (the pattern matched every +`bpf_skb_*`/`bpf_xdp_*` helper) and BPF-to-BPF calls (it matched `SEC("?...")`), and +under-counted real programs by seven (a `//` inside a section name was stripped as a +comment). The portable set of 25 was unaffected. + +Globals no longer appear as a blocker at all. The next unlocks by count are helpers (a +long tail, but `bpf_get_current_task`, `bpf_ktime_get_boot_ns` and the `bpf_probe_read_user*` +family recur), array maps, and typed program arguments. + +## 3. Incidental findings + +- **Nested struct field access fails with a misleading error.** `ctx.regs.ip` reports + `SyntaxError: Undefined variable actual` — naming the assignment target rather than the + nested access that caused it. `_allocate_for_attribute` declines to allocate when the + attribute's base is not a plain `Name`, logging at debug level, and the expression pass + then trips over the missing symbol. The diagnostic should name the real cause. +- **One level of nested-context access already works.** `ctx.sample_period` on + `struct_bpf_perf_event_data` compiles and `llc`s cleanly, so `perf_event` contexts are + usable today for anything that does not need `regs`. +- **`@section` really does accept anything.** `tc`, `socket`, `fentry/…`, `lsm/…`, + `cgroup_skb/egress`, `netfilter` and `tp_btf/…` all compile and land in the ELF verbatim. + Program type is not a constraint; the context type is. + +## Re-running the corpus scoring + +```bash +git clone --depth 1 --filter=blob:none --sparse https://github.com/torvalds/linux +git -C linux sparse-checkout set --no-cone tools/testing/selftests/bpf/progs +python3 tools/selftest-audit.py linux/tools/testing/selftests/bpf/progs --histogram --max-hard 1 +``` + +Re-run it after each feature lands. The envelope it encodes (helper, map and construct +lists at the top of the script) is maintained by hand and must move with the compiler. diff --git a/tests/kernel_selftest_equivalent/cgroup/cgroup_link.py b/tests/kernel_selftest_equivalent/cgroup/cgroup_link.py new file mode 100644 index 00000000..a7870832 --- /dev/null +++ b/tests/kernel_selftest_equivalent/cgroup/cgroup_link.py @@ -0,0 +1,58 @@ +# Ported from Linux tools/testing/selftests/bpf/progs/test_cgroup_link.c +# +# Two cgroup_skb/egress programs, each counting its own runs. Upstream +# attaches one through a cgroup bpf_link, then swaps in the other with +# bpf_link_update() and checks the right counter moves: +# +# int calls = 0; +# int alt_calls = 0; +# +# SEC("cgroup_skb/egress") +# int egress(struct __sk_buff *skb) +# { +# __sync_fetch_and_add(&calls, 1); +# return 1; +# } +# +# WORKAROUND(atomics): the upstream increments are atomic. PythonBPF has no +# atomic operations, so these are plain read-modify-writes of the globals. + +from pythonbpf import bpf, section, bpfglobal, compile +from ctypes import c_void_p, c_int64, c_int32 + + +@bpf +@bpfglobal +def calls() -> c_int32: + return c_int32(0) + + +@bpf +@bpfglobal +def alt_calls() -> c_int32: + return c_int32(0) + + +@bpf +@section("cgroup_skb/egress") +def egress(skb: c_void_p) -> c_int64: + global calls + calls += 1 # WORKAROUND(atomics): __sync_fetch_and_add(&calls, 1) + return c_int64(1) + + +@bpf +@section("cgroup_skb/egress") +def egress_alt(skb: c_void_p) -> c_int64: + global alt_calls + alt_calls += 1 # WORKAROUND(atomics): __sync_fetch_and_add(&alt_calls, 1) + return c_int64(1) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/kernel_selftest_equivalent/cgroup/cgroup_mprog.py b/tests/kernel_selftest_equivalent/cgroup/cgroup_mprog.py new file mode 100644 index 00000000..f4717b5e --- /dev/null +++ b/tests/kernel_selftest_equivalent/cgroup/cgroup_mprog.py @@ -0,0 +1,41 @@ +# Ported from Linux tools/testing/selftests/bpf/progs/cgroup_mprog.c +# +# Four identical cgroup/getsockopt programs. Upstream attaches them in +# various orders with BPF_F_BEFORE/BPF_F_AFTER and checks the resulting +# multi-prog chain; the bodies only need to exist and return "allow". + +from pythonbpf import bpf, section, bpfglobal, compile +from ctypes import c_void_p, c_int64 + + +@bpf +@section("cgroup/getsockopt") +def getsockopt_1(ctx: c_void_p) -> c_int64: + return c_int64(1) + + +@bpf +@section("cgroup/getsockopt") +def getsockopt_2(ctx: c_void_p) -> c_int64: + return c_int64(1) + + +@bpf +@section("cgroup/getsockopt") +def getsockopt_3(ctx: c_void_p) -> c_int64: + return c_int64(1) + + +@bpf +@section("cgroup/getsockopt") +def getsockopt_4(ctx: c_void_p) -> c_int64: + return c_int64(1) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/kernel_selftest_equivalent/maps/array_map_lookup_update.py b/tests/kernel_selftest_equivalent/maps/array_map_lookup_update.py new file mode 100644 index 00000000..f84c2d3b --- /dev/null +++ b/tests/kernel_selftest_equivalent/maps/array_map_lookup_update.py @@ -0,0 +1,35 @@ +# Adapted from bpf-next/tools/testing/selftests/bpf/progs/test_map_ops.c +# and bpf-next/tools/testing/selftests/bpf/progs/bpf_iter_bpf_array_map.c. + +from ctypes import c_int32, c_uint64, c_void_p + +from pythonbpf import bpf, bpfglobal, compile, map, section +from pythonbpf.maps import ArrayMap + + +@bpf +@map +def counters() -> ArrayMap: + return ArrayMap(key=c_int32, value=c_uint64, max_entries=8) + + +@bpf +@section("tracepoint/syscalls/sys_enter_getpid") +def array_map_lookup_update(ctx: c_void_p) -> c_int32: + counters.update(0, 1) + + current = counters.lookup(0) + if current: + next_value = current + 1 + counters.update(0, next_value) + + return c_int32(0) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/kernel_selftest_equivalent/netfilter/netfilter_link_attach.py b/tests/kernel_selftest_equivalent/netfilter/netfilter_link_attach.py new file mode 100644 index 00000000..57408901 --- /dev/null +++ b/tests/kernel_selftest_equivalent/netfilter/netfilter_link_attach.py @@ -0,0 +1,23 @@ +# Ported from Linux tools/testing/selftests/bpf/progs/test_netfilter_link_attach.c +# +# A netfilter program that accepts everything (NF_ACCEPT is 1). Upstream +# attaches it with every combination of protocol family, hook and priority +# and checks which the kernel rejects. + +from pythonbpf import bpf, section, bpfglobal, compile +from ctypes import c_void_p, c_int64 + + +@bpf +@section("netfilter") +def nf_link_attach_test(ctx: c_void_p) -> c_int64: + return c_int64(1) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/kernel_selftest_equivalent/ringbuf/reserve_submit_discard.py b/tests/kernel_selftest_equivalent/ringbuf/reserve_submit_discard.py new file mode 100644 index 00000000..0c3b1314 --- /dev/null +++ b/tests/kernel_selftest_equivalent/ringbuf/reserve_submit_discard.py @@ -0,0 +1,48 @@ +# Adapted from bpf-next/tools/testing/selftests/bpf/progs/test_ringbuf.c. + +from ctypes import c_int32, c_uint64, c_void_p + +from pythonbpf import bpf, bpfglobal, compile, map, section, struct +from pythonbpf.helper import pid +from pythonbpf.maps import RingBuffer + + +@bpf +@struct +class sample_t: + pid: c_uint64 + seq: c_uint64 + value: c_uint64 + + +@bpf +@map +def events() -> RingBuffer: + return RingBuffer(max_entries=4096) + + +@bpf +@section("tracepoint/syscalls/sys_enter_getpid") +def ringbuf_reserve_submit_discard(ctx: c_void_p) -> c_int32: + first = events.reserve(24) + if first: + sample = sample_t(first) + sample.pid = pid() + sample.seq = 0 + sample.value = 7 + events.submit(first, 0) + + second = events.reserve(24) + if second: + events.discard(second, 0) + + return c_int32(0) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/kernel_selftest_equivalent/socket/signed_loader.py b/tests/kernel_selftest_equivalent/socket/signed_loader.py new file mode 100644 index 00000000..8f4974ec --- /dev/null +++ b/tests/kernel_selftest_equivalent/socket/signed_loader.py @@ -0,0 +1,23 @@ +# Ported from Linux tools/testing/selftests/bpf/progs/test_signed_loader.c +# +# A minimal, map-less socket filter. Upstream drives it through libbpf's +# light-skeleton loader to test signed-program loading; a socket filter +# needs no attach resolution and no maps keeps the loader trivial. + +from pythonbpf import bpf, section, bpfglobal, compile +from ctypes import c_void_p, c_int64 + + +@bpf +@section("socket") +def probe(ctx: c_void_p) -> c_int64: + return c_int64(0) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/kernel_selftest_equivalent/socket/signed_loader_data.py b/tests/kernel_selftest_equivalent/socket/signed_loader_data.py new file mode 100644 index 00000000..43bc53d1 --- /dev/null +++ b/tests/kernel_selftest_equivalent/socket/signed_loader_data.py @@ -0,0 +1,37 @@ +# Ported from Linux tools/testing/selftests/bpf/progs/test_signed_loader_data.c +# +# The signed-loader fixture with one initialised global, so the object has a +# .data map that the loader must seed. Upstream checks that a signed loader +# keeps the attested initial value: +# +# __u64 magic = 0x5eed1234abad1deaULL; +# +# SEC("socket") +# int probe(void *ctx) +# { +# return (int)magic; +# } + +from pythonbpf import bpf, section, bpfglobal, compile +from ctypes import c_void_p, c_int32, c_uint64 + + +@bpf +@bpfglobal +def magic() -> c_uint64: + return c_uint64(0x5EED1234ABAD1DEA) + + +@bpf +@section("socket") +def probe(ctx: c_void_p) -> c_int32: + return c_int32(magic) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/kernel_selftest_equivalent/socket/veristat_foo.py b/tests/kernel_selftest_equivalent/socket/veristat_foo.py new file mode 100644 index 00000000..1fad158d --- /dev/null +++ b/tests/kernel_selftest_equivalent/socket/veristat_foo.py @@ -0,0 +1,35 @@ +# Ported from Linux tools/testing/selftests/bpf/progs/veristat_foo.c +# +# Three empty socket filters. Upstream exists only to exercise veristat's +# program-name filters, so the bodies are irrelevant and the names are the +# test. Ported for the same reason: three programs, one section, one object. + +from pythonbpf import bpf, section, bpfglobal, compile +from ctypes import c_void_p, c_int64 + + +@bpf +@section("socket") +def foo(ctx: c_void_p) -> c_int64: + return c_int64(0) + + +@bpf +@section("socket") +def bar(ctx: c_void_p) -> c_int64: + return c_int64(0) + + +@bpf +@section("socket") +def buz(ctx: c_void_p) -> c_int64: + return c_int64(0) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/kernel_selftest_equivalent/tc/tc_dummy.py b/tests/kernel_selftest_equivalent/tc/tc_dummy.py new file mode 100644 index 00000000..8f8797d7 --- /dev/null +++ b/tests/kernel_selftest_equivalent/tc/tc_dummy.py @@ -0,0 +1,22 @@ +# Ported from Linux tools/testing/selftests/bpf/progs/tc_dummy.c +# +# A classifier that returns TC_ACT_OK for everything. Upstream is the fixture +# behind the tc_links and tc_opts attach-order tests. + +from pythonbpf import bpf, section, bpfglobal, compile +from ctypes import c_void_p, c_int64 + + +@bpf +@section("tc") +def entry(skb: c_void_p) -> c_int64: + return c_int64(1) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/kernel_selftest_equivalent/tracing/autoattach.py b/tests/kernel_selftest_equivalent/tracing/autoattach.py new file mode 100644 index 00000000..b6746f5b --- /dev/null +++ b/tests/kernel_selftest_equivalent/tracing/autoattach.py @@ -0,0 +1,65 @@ +# Ported from Linux tools/testing/selftests/bpf/progs/test_autoattach.c +# +# Two programs on different raw tracepoints, each recording that it ran. The +# upstream test asserts both fired after bpf_object__attach_skeleton(): +# +# bool prog1_called = false; +# bool prog2_called = false; +# +# SEC("raw_tp/sys_enter") +# int prog1(const void *ctx) +# { +# prog1_called = true; +# return 0; +# } +# +# SEC("raw_tp/sys_exit") +# int prog2(const void *ctx) +# { +# prog2_called = true; +# return 0; +# } +# +# Both flags are @bpfglobal scalars shared by the two programs in one object. +# They are c_uint64 rather than bool because integer scalars are the only +# global type today; the driver-side check is the same either way. + +from pythonbpf import bpf, section, bpfglobal, compile +from ctypes import c_void_p, c_int64, c_uint64 + + +@bpf +@bpfglobal +def prog1_called() -> c_uint64: + return c_uint64(0) + + +@bpf +@bpfglobal +def prog2_called() -> c_uint64: + return c_uint64(0) + + +@bpf +@section("raw_tp/sys_enter") +def prog1(ctx: c_void_p) -> c_int64: + global prog1_called + prog1_called = 1 + return c_int64(0) + + +@bpf +@section("raw_tp/sys_exit") +def prog2(ctx: c_void_p) -> c_int64: + global prog2_called + prog2_called = 1 + return c_int64(0) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/kernel_selftest_equivalent/tracing/enable_stats.py b/tests/kernel_selftest_equivalent/tracing/enable_stats.py new file mode 100644 index 00000000..a1dc4781 --- /dev/null +++ b/tests/kernel_selftest_equivalent/tracing/enable_stats.py @@ -0,0 +1,43 @@ +# Ported from Linux tools/testing/selftests/bpf/progs/test_enable_stats.c +# +# A raw tracepoint program that counts its runs. Upstream enables +# BPF_STATS_RUN_TIME, triggers the program, and checks run_time_ns and +# run_cnt in bpf_prog_info alongside `count`: +# +# __u64 count = 0; +# +# SEC("raw_tracepoint/sys_enter") +# int test_enable_stats(void *ctx) +# { +# __sync_fetch_and_add(&count, 1); +# return 0; +# } +# +# WORKAROUND(atomics): the upstream increment is atomic. PythonBPF has no +# atomic operations, so this is a plain read-modify-write of the global. + +from pythonbpf import bpf, section, bpfglobal, compile +from ctypes import c_void_p, c_int64, c_uint64 + + +@bpf +@bpfglobal +def count() -> c_uint64: + return c_uint64(0) + + +@bpf +@section("raw_tracepoint/sys_enter") +def test_enable_stats(ctx: c_void_p) -> c_int64: + global count + count += 1 # WORKAROUND(atomics): __sync_fetch_and_add(&count, 1) + return c_int64(0) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/kernel_selftest_equivalent/tracing/get_cgroup_id.py b/tests/kernel_selftest_equivalent/tracing/get_cgroup_id.py new file mode 100644 index 00000000..506978dd --- /dev/null +++ b/tests/kernel_selftest_equivalent/tracing/get_cgroup_id.py @@ -0,0 +1,55 @@ +# Ported from Linux tools/testing/selftests/bpf/progs/get_cgroup_id_kern.c +# +# Upstream records the cgroup id of a process whose pid matches one the +# userspace half of the test set beforehand: +# +# __u64 cg_id; +# __u64 expected_pid; +# +# SEC("tracepoint/syscalls/sys_enter_nanosleep") +# int trace(void *ctx) +# { +# __u32 pid = bpf_get_current_pid_tgid(); +# +# if (expected_pid == pid) +# cg_id = bpf_get_current_cgroup_id(); +# +# return 0; +# } +# +# Both file-scope variables are @bpfglobal scalars, so the userspace half reads +# `cg_id` back out of the object's .bss exactly as the kernel's driver does. + +from pythonbpf import bpf, section, bpfglobal, compile +from pythonbpf.helper import pid, get_current_cgroup_id +from ctypes import c_void_p, c_int64, c_uint64 + + +@bpf +@bpfglobal +def cg_id() -> c_uint64: + return c_uint64(0) + + +@bpf +@bpfglobal +def expected_pid() -> c_uint64: + return c_uint64(0) + + +@bpf +@section("tracepoint/syscalls/sys_enter_nanosleep") +def trace(ctx: c_void_p) -> c_int64: + global cg_id + if expected_pid == pid(): + cg_id = get_current_cgroup_id() + return c_int64(0) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/kernel_selftest_equivalent/tracing/kprobe_multi_empty.py b/tests/kernel_selftest_equivalent/tracing/kprobe_multi_empty.py new file mode 100644 index 00000000..a7d8ffca --- /dev/null +++ b/tests/kernel_selftest_equivalent/tracing/kprobe_multi_empty.py @@ -0,0 +1,22 @@ +# Ported from Linux tools/testing/selftests/bpf/progs/kprobe_multi_empty.c +# +# An empty kprobe.multi program. Upstream attaches it to every function in +# the kernel's available_filter_functions list to benchmark attach time. + +from pythonbpf import bpf, section, bpfglobal, compile +from ctypes import c_void_p, c_int64 + + +@bpf +@section("kprobe.multi/") +def test_kprobe_empty(ctx: c_void_p) -> c_int64: + return c_int64(0) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/kernel_selftest_equivalent/tracing/link_pinning.py b/tests/kernel_selftest_equivalent/tracing/link_pinning.py new file mode 100644 index 00000000..9065e544 --- /dev/null +++ b/tests/kernel_selftest_equivalent/tracing/link_pinning.py @@ -0,0 +1,58 @@ +# Ported from Linux tools/testing/selftests/bpf/progs/test_link_pinning.c +# +# Two programs, one raw_tp and one tp_btf on the same tracepoint, each +# copying a global set by userspace into a global read by userspace. +# Upstream pins the link, closes every fd, then checks the program still +# fires by bumping `in` and watching `out` follow: +# +# int in = 0; +# int out = 0; +# +# SEC("raw_tp/sys_enter") +# int raw_tp_prog(const void *ctx) +# { +# out = in; +# return 0; +# } +# +# `in` is renamed `in_val` because `in` is a Python keyword. + +from pythonbpf import bpf, section, bpfglobal, compile +from ctypes import c_void_p, c_int64, c_int32 + + +@bpf +@bpfglobal +def in_val() -> c_int32: + return c_int32(0) + + +@bpf +@bpfglobal +def out() -> c_int32: + return c_int32(0) + + +@bpf +@section("raw_tp/sys_enter") +def raw_tp_prog(ctx: c_void_p) -> c_int64: + global out + out = in_val + return c_int64(0) + + +@bpf +@section("tp_btf/sys_enter") +def tp_btf_prog(ctx: c_void_p) -> c_int64: + global out + out = in_val + return c_int64(0) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/kernel_selftest_equivalent/tracing/perf_link.py b/tests/kernel_selftest_equivalent/tracing/perf_link.py new file mode 100644 index 00000000..7cbc8b87 --- /dev/null +++ b/tests/kernel_selftest_equivalent/tracing/perf_link.py @@ -0,0 +1,43 @@ +# Ported from Linux tools/testing/selftests/bpf/progs/test_perf_link.c +# +# A perf_event program that counts how often it runs. Upstream attaches it +# through a perf_event bpf_link and checks `run_cnt` moved: +# +# int run_cnt = 0; +# +# SEC("perf_event") +# int handler(struct pt_regs *ctx) +# { +# __sync_fetch_and_add(&run_cnt, 1); +# return 0; +# } +# +# WORKAROUND(atomics): the upstream increment is atomic. PythonBPF has no +# atomic operations, so this is a plain read-modify-write of the global. +# Replace with the atomic form once atomics land; grep for WORKAROUND(atomics). + +from pythonbpf import bpf, section, bpfglobal, compile +from ctypes import c_void_p, c_int64, c_int32 + + +@bpf +@bpfglobal +def run_cnt() -> c_int32: + return c_int32(0) + + +@bpf +@section("perf_event") +def handler(ctx: c_void_p) -> c_int64: + global run_cnt + run_cnt += 1 # WORKAROUND(atomics): __sync_fetch_and_add(&run_cnt, 1) + return c_int64(0) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/kernel_selftest_equivalent/tracing/tracepoint_sched_switch.py b/tests/kernel_selftest_equivalent/tracing/tracepoint_sched_switch.py new file mode 100644 index 00000000..9252e0fd --- /dev/null +++ b/tests/kernel_selftest_equivalent/tracing/tracepoint_sched_switch.py @@ -0,0 +1,27 @@ +# Ported from Linux tools/testing/selftests/bpf/progs/test_tracepoint.c +# +# Upstream is a bare handler on sched/sched_switch, used to prove the program +# attaches to a non-syscall tracepoint. Kept faithful: the point is the +# attachment surface, not the body. +# +# Upstream declares the tracepoint argument layout as a struct taken from +# /sys/kernel/tracing/events/sched/sched_switch/format. PythonBPF does not read +# tracepoint formats, so the context stays opaque. + +from pythonbpf import bpf, section, bpfglobal, compile +from ctypes import c_void_p, c_int64 + + +@bpf +@section("tracepoint/sched/sched_switch") +def oncpu(ctx: c_void_p) -> c_int64: + return c_int64(0) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/kernel_selftest_equivalent/tracing/uprobe_multi_bench.py b/tests/kernel_selftest_equivalent/tracing/uprobe_multi_bench.py new file mode 100644 index 00000000..890bfc80 --- /dev/null +++ b/tests/kernel_selftest_equivalent/tracing/uprobe_multi_bench.py @@ -0,0 +1,39 @@ +# Ported from Linux tools/testing/selftests/bpf/progs/uprobe_multi_bench.c +# +# Count how many times the multi-uprobe fires. Upstream attaches it to +# thousands of uprobe_multi_func_* symbols and reads `count` back: +# +# int count; +# +# SEC("uprobe.multi/./uprobe_multi:uprobe_multi_func_*") +# int uprobe_bench(struct pt_regs *ctx) +# { +# count++; +# return 0; +# } + +from pythonbpf import bpf, section, bpfglobal, compile +from ctypes import c_void_p, c_int64, c_int32 + + +@bpf +@bpfglobal +def count() -> c_int32: + return c_int32(0) + + +@bpf +@section("uprobe.multi/./uprobe_multi:uprobe_multi_func_*") +def uprobe_bench(ctx: c_void_p) -> c_int64: + global count + count += 1 + return c_int64(0) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/kernel_selftest_equivalent/tracing/uprobe_multi_usdt.py b/tests/kernel_selftest_equivalent/tracing/uprobe_multi_usdt.py new file mode 100644 index 00000000..01ed34c0 --- /dev/null +++ b/tests/kernel_selftest_equivalent/tracing/uprobe_multi_usdt.py @@ -0,0 +1,30 @@ +# Ported from Linux tools/testing/selftests/bpf/progs/uprobe_multi_usdt.c +# +# The USDT flavour of the multi-uprobe counter. Upstream attaches it to a +# USDT probe in the test binary and asserts on `count` from userspace. + +from pythonbpf import bpf, section, bpfglobal, compile +from ctypes import c_void_p, c_int64, c_int32 + + +@bpf +@bpfglobal +def count() -> c_int32: + return c_int32(0) + + +@bpf +@section("usdt") +def usdt0(ctx: c_void_p) -> c_int64: + global count + count += 1 + return c_int64(0) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/kernel_selftest_equivalent/vmlinux/cgroup_skb_direct_packet_access.py b/tests/kernel_selftest_equivalent/vmlinux/cgroup_skb_direct_packet_access.py new file mode 100644 index 00000000..c290fb3c --- /dev/null +++ b/tests/kernel_selftest_equivalent/vmlinux/cgroup_skb_direct_packet_access.py @@ -0,0 +1,41 @@ +# Ported from Linux tools/testing/selftests/bpf/progs/cgroup_skb_direct_packet_access.c +# +# A cgroup_skb program that records skb->data_end into a global. Upstream +# asserts from userspace that the value is non-zero, proving cgroup_skb +# programs get direct packet access: +# +# __u32 data_end; +# +# SEC("cgroup_skb/ingress") +# int direct_packet_access(struct __sk_buff *skb) +# { +# data_end = skb->data_end; +# return 1; +# } + +from pythonbpf import bpf, section, bpfglobal, compile +from vmlinux import struct___sk_buff +from ctypes import c_int64, c_uint32 + + +@bpf +@bpfglobal +def data_end() -> c_uint32: + return c_uint32(0) + + +@bpf +@section("cgroup_skb/ingress") +def direct_packet_access(skb: struct___sk_buff) -> c_int64: + global data_end + data_end = skb.data_end + return c_int64(1) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/kernel_selftest_equivalent/vmlinux/connect4_dropper.py b/tests/kernel_selftest_equivalent/vmlinux/connect4_dropper.py new file mode 100644 index 00000000..d893450c --- /dev/null +++ b/tests/kernel_selftest_equivalent/vmlinux/connect4_dropper.py @@ -0,0 +1,53 @@ +# Ported from Linux tools/testing/selftests/bpf/progs/connect4_dropper.c +# +# A cgroup/connect4 hook that rejects TCP connects to one port, which +# userspace writes into `port` before attaching: +# +# int port; +# +# SEC("cgroup/connect4") +# int connect_v4_dropper(struct bpf_sock_addr *ctx) +# { +# if (ctx->type != SOCK_STREAM) +# return VERDICT_PROCEED; +# if (ctx->user_port == bpf_htons(port)) +# return VERDICT_REJECT; +# return VERDICT_PROCEED; +# } +# +# bpf_htons() is a byte swap, written out here as shifts on the low 16 bits; +# SOCK_STREAM is 1. + +from pythonbpf import bpf, section, bpfglobal, compile +from vmlinux import struct_bpf_sock_addr +from ctypes import c_int64, c_int32 + +VERDICT_REJECT = 0 +VERDICT_PROCEED = 1 +SOCK_STREAM = 1 + + +@bpf +@bpfglobal +def port() -> c_int32: + return c_int32(0) + + +@bpf +@section("cgroup/connect4") +def connect_v4_dropper(ctx: struct_bpf_sock_addr) -> c_int64: + if ctx.type != 1: + return c_int64(1) + port_be = ((port & 0xFF) << 8) | ((port >> 8) & 0xFF) + if ctx.user_port == port_be: + return c_int64(0) + return c_int64(1) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/kernel_selftest_equivalent/vmlinux/perf_skip.py b/tests/kernel_selftest_equivalent/vmlinux/perf_skip.py new file mode 100644 index 00000000..fc146710 --- /dev/null +++ b/tests/kernel_selftest_equivalent/vmlinux/perf_skip.py @@ -0,0 +1,56 @@ +# Ported from Linux tools/testing/selftests/bpf/progs/test_perf_skip.c +# +# A perf_event program that reports whether the sampled instruction pointer is +# the one userspace asked about. Upstream: +# +# uintptr_t ip; +# +# SEC("perf_event") +# int handler(struct bpf_perf_event_data *data) +# { +# /* Skip events that have the correct ip. */ +# return ip != PT_REGS_IP(&data->regs); +# } +# +# `ip` is a @bpfglobal the driver sets before attaching; the program only +# reads it, so no `global` statement is needed. +# +# ROADMAP: this is a strict expected failure. `ctx.regs.ip` is two levels of +# struct field access, and PythonBPF supports only one -- +# `_allocate_for_attribute` in allocation_pass.py bails out unless the +# attribute's base is a plain Name. One level works today: `ctx.sample_period` +# on this same context compiles fine. +# +# Note the failure surfaces as `SyntaxError: Undefined variable actual`, naming +# the assignment target rather than the nested access that caused it -- the +# allocation pass declines to allocate and logs at debug level, then the +# expression pass fails later on the missing symbol. Worth improving alongside +# nested access support. + +from pythonbpf import bpf, section, bpfglobal, compile +from vmlinux import struct_bpf_perf_event_data +from ctypes import c_int64, c_uint64 + + +@bpf +@bpfglobal +def ip() -> c_uint64: + return c_uint64(0) + + +@bpf +@section("perf_event") +def handler(ctx: struct_bpf_perf_event_data) -> c_int64: + actual = ctx.regs.ip + if ip == actual: + return c_int64(0) + return c_int64(1) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/kernel_selftest_equivalent/vmlinux/priv_prog.py b/tests/kernel_selftest_equivalent/vmlinux/priv_prog.py new file mode 100644 index 00000000..976144f6 --- /dev/null +++ b/tests/kernel_selftest_equivalent/vmlinux/priv_prog.py @@ -0,0 +1,24 @@ +# Ported from Linux tools/testing/selftests/bpf/progs/priv_prog.c +# +# An XDP program that drops everything. Upstream loads it from an +# unprivileged process to check the CAP_BPF/CAP_NET_ADMIN gating; the +# program itself is the smallest privileged-type program there is. + +from pythonbpf import bpf, section, bpfglobal, compile +from vmlinux import XDP_DROP +from ctypes import c_void_p, c_int64 + + +@bpf +@section("xdp") +def xdp_prog1(xdp: c_void_p) -> c_int64: + return XDP_DROP + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/kernel_selftest_equivalent/vmlinux/tc_bpf.py b/tests/kernel_selftest_equivalent/vmlinux/tc_bpf.py new file mode 100644 index 00000000..cf6710d7 --- /dev/null +++ b/tests/kernel_selftest_equivalent/vmlinux/tc_bpf.py @@ -0,0 +1,43 @@ +# Ported from Linux tools/testing/selftests/bpf/progs/test_tc_bpf.c +# +# `cls` is a dummy classifier for the TC-BPF API test. `pkt_ptr` is the one +# that matters: it derives a packet pointer from skb->data, bounds-checks it +# against skb->data_end, and is loaded without CAP_SYS_ADMIN/CAP_PERFMON to +# prove direct packet access works for a plain tcx program. Upstream: +# +# struct iphdr *iph = (void *)(long)skb->data + sizeof(struct ethhdr); +# +# if ((long)(iph + 1) > (long)skb->data_end) +# return 1; +# return 0; +# +# sizeof(struct ethhdr) + sizeof(struct iphdr) is 14 + 20 = 34. + +from pythonbpf import bpf, section, bpfglobal, compile +from vmlinux import struct___sk_buff +from ctypes import c_void_p, c_int64 + + +@bpf +@section("tc") +def cls(skb: c_void_p) -> c_int64: + return c_int64(0) + + +@bpf +@section("tcx/ingress") +def pkt_ptr(skb: struct___sk_buff) -> c_int64: + data = c_void_p(skb.data) + data_end = c_void_p(skb.data_end) + if data + 34 > data_end: + return c_int64(1) + return c_int64(0) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/kernel_selftest_equivalent/vmlinux/xdp_devmap_helpers.py b/tests/kernel_selftest_equivalent/vmlinux/xdp_devmap_helpers.py new file mode 100644 index 00000000..771b017f --- /dev/null +++ b/tests/kernel_selftest_equivalent/vmlinux/xdp_devmap_helpers.py @@ -0,0 +1,34 @@ +# Ported from Linux tools/testing/selftests/bpf/progs/test_xdp_devmap_helpers.c +# +# Reads xdp_md->egress_ifindex, which only exists for programs loaded with +# expected_attach_type = BPF_XDP_DEVMAP. Upstream loads it *without* that +# type and asserts the load fails, so the program is a negative fixture: +# +# unsigned int len = data_end - data; +# bpf_trace_printk(fmt, sizeof(fmt), +# ctx->ingress_ifindex, ctx->egress_ifindex, len); +# return XDP_PASS; + +from pythonbpf import bpf, section, bpfglobal, compile +from vmlinux import XDP_PASS +from vmlinux import struct_xdp_md +from ctypes import c_int64 + + +@bpf +@section("xdp") +def xdpdm_devlog(ctx: struct_xdp_md) -> c_int64: + length = ctx.data_end - ctx.data + ingress = ctx.ingress_ifindex + egress = ctx.egress_ifindex + print(f"devmap redirect: dev {ingress} -> dev {egress} len {length}") + return XDP_PASS + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/kernel_selftest_equivalent/vmlinux/xdp_dummy.py b/tests/kernel_selftest_equivalent/vmlinux/xdp_dummy.py new file mode 100644 index 00000000..c715cac7 --- /dev/null +++ b/tests/kernel_selftest_equivalent/vmlinux/xdp_dummy.py @@ -0,0 +1,31 @@ +# Ported from Linux tools/testing/selftests/bpf/progs/xdp_dummy.c +# +# Two XDP programs that pass every packet. Upstream is the fixture that a +# dozen prog_tests attach and detach to exercise XDP link plumbing; the +# second program's odd name is deliberate, it is what the kallsyms test looks +# for. + +from pythonbpf import bpf, section, bpfglobal, compile +from vmlinux import XDP_PASS +from ctypes import c_void_p, c_int64 + + +@bpf +@section("xdp") +def xdp_dummy_prog(ctx: c_void_p) -> c_int64: + return XDP_PASS + + +@bpf +@section("xdp") +def __x64_sys_nop(ctx: c_void_p) -> c_int64: + return XDP_PASS + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/kernel_selftest_equivalent/vmlinux/xdp_tx.py b/tests/kernel_selftest_equivalent/vmlinux/xdp_tx.py new file mode 100644 index 00000000..d432eb74 --- /dev/null +++ b/tests/kernel_selftest_equivalent/vmlinux/xdp_tx.py @@ -0,0 +1,25 @@ +# Ported from Linux tools/testing/selftests/bpf/progs/xdp_tx.c +# +# Bounce every packet back out of the interface it arrived on. Upstream is +# the transmit side of the veth XDP tests. +# +# XDP actions are vmlinux enum constants, like every kernel constant. + +from pythonbpf import bpf, section, bpfglobal, compile +from vmlinux import XDP_TX +from ctypes import c_void_p, c_int64 + + +@bpf +@section("xdp") +def xdp_tx(xdp: c_void_p) -> c_int64: + return c_int64(XDP_TX) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/kernel_selftest_equivalent/xdp/xdp_link.py b/tests/kernel_selftest_equivalent/xdp/xdp_link.py new file mode 100644 index 00000000..d94df5ab --- /dev/null +++ b/tests/kernel_selftest_equivalent/xdp/xdp_link.py @@ -0,0 +1,29 @@ +# Ported from Linux tools/testing/selftests/bpf/progs/test_xdp_link.c +# +# One XDP and one TC handler in the same object. Upstream attaches the XDP +# one through bpf_link and checks that legacy netlink attach of the same +# program is refused while the link exists. + +from pythonbpf import bpf, section, bpfglobal, compile +from ctypes import c_void_p, c_int64 + + +@bpf +@section("xdp") +def xdp_handler(xdp: c_void_p) -> c_int64: + return c_int64(0) + + +@bpf +@section("tc") +def tc_handler(skb: c_void_p) -> c_int64: + return c_int64(0) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/failing_tests/assign/annotated_assign.py b/tests/passing_tests/assign/annotated_assign.py similarity index 100% rename from tests/failing_tests/assign/annotated_assign.py rename to tests/passing_tests/assign/annotated_assign.py diff --git a/tests/passing_tests/assign/map_value_compare.py b/tests/passing_tests/assign/map_value_compare.py new file mode 100644 index 00000000..5fdcb0e3 --- /dev/null +++ b/tests/passing_tests/assign/map_value_compare.py @@ -0,0 +1,33 @@ +# A comparison with a map value compares what the map holds, on either side: +# `head == body.lookup(1)` with both sides map lookups used to compare the two +# entries' addresses. +from ctypes import c_int64, c_uint64, c_void_p +from pythonbpf import bpf, map, section, bpfglobal, compile +from pythonbpf.maps import HashMap + + +@bpf +@map +def body() -> HashMap: + return HashMap(key=c_uint64, value=c_uint64, max_entries=10) + + +@bpf +@section("tracepoint/syscalls/sys_enter_getppid") +def prog(ctx: c_void_p) -> c_int64: + head = body.lookup(0) + if head == body.lookup(1): + return c_int64(1) + k = 3 + if body.lookup(2) == body.lookup(k): + return c_int64(2) + return c_int64(0) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/failing_tests/loops/for_break.py b/tests/passing_tests/loops/for_break.py similarity index 100% rename from tests/failing_tests/loops/for_break.py rename to tests/passing_tests/loops/for_break.py diff --git a/tests/failing_tests/loops/for_continue.py b/tests/passing_tests/loops/for_continue.py similarity index 100% rename from tests/failing_tests/loops/for_continue.py rename to tests/passing_tests/loops/for_continue.py diff --git a/tests/failing_tests/loops/for_range_start_stop_step.py b/tests/passing_tests/loops/for_range_start_stop_step.py similarity index 100% rename from tests/failing_tests/loops/for_range_start_stop_step.py rename to tests/passing_tests/loops/for_range_start_stop_step.py diff --git a/tests/failing_tests/loops/for_range_sum.py b/tests/passing_tests/loops/for_range_sum.py similarity index 100% rename from tests/failing_tests/loops/for_range_sum.py rename to tests/passing_tests/loops/for_range_sum.py diff --git a/tests/passing_tests/loops/loop_else.py b/tests/passing_tests/loops/loop_else.py new file mode 100644 index 00000000..80b00619 --- /dev/null +++ b/tests/passing_tests/loops/loop_else.py @@ -0,0 +1,32 @@ +# A loop's else-branch runs when the loop ends without a break: the for loop +# never breaks, so its else adds 100; the while loop breaks, so its else is +# skipped. Returns 3 + 100 + 1 = 104. +from pythonbpf import bpf, section, bpfglobal, compile +from ctypes import c_void_p, c_int64 + + +@bpf +@section("tracepoint/syscalls/sys_enter_execve") +def hello(ctx: c_void_p) -> c_int64: + total: c_int64 = 0 + for i in range(3): + if i == 7: + break + total = total + 1 + else: + total = total + 100 + while total < 1000: + total = total + 1 + break + else: + total = total + 1000 + return total + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/failing_tests/loops/nested_for.py b/tests/passing_tests/loops/nested_for.py similarity index 100% rename from tests/failing_tests/loops/nested_for.py rename to tests/passing_tests/loops/nested_for.py diff --git a/tests/passing_tests/loops/range_global_bound.py b/tests/passing_tests/loops/range_global_bound.py new file mode 100644 index 00000000..0b395479 --- /dev/null +++ b/tests/passing_tests/loops/range_global_bound.py @@ -0,0 +1,37 @@ +# The bound is a @bpfglobal and the body calls a helper, so opt cannot fold +# the loop away: this is the case that puts a real bounded loop in front of +# the verifier. Reference: dyn_helper in tests/c-form/loops.bpf.c. +# +# The clamp is required, in C as here: userspace can write any value to a +# .data global, so without it the verifier assumes a bound up to 2**63 and +# walks iterations until it hits its 1M-instruction limit (E2BIG). +from pythonbpf import bpf, section, bpfglobal, compile +from pythonbpf.helper import random +from ctypes import c_void_p, c_int64 + + +@bpf +@bpfglobal +def n() -> c_int64: + return c_int64(10) + + +@bpf +@section("tracepoint/syscalls/sys_enter_execve") +def hello(ctx: c_void_p) -> c_int64: + total: c_int64 = 0 + stop = n + if stop > 64: + stop = 64 + for i in range(stop): + total = total + random() + return total + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/passing_tests/loops/range_negative_step.py b/tests/passing_tests/loops/range_negative_step.py new file mode 100644 index 00000000..fa89739c --- /dev/null +++ b/tests/passing_tests/loops/range_negative_step.py @@ -0,0 +1,21 @@ +# Counting down: 10 + 7 + 4 + 1 = 22. +from pythonbpf import bpf, section, bpfglobal, compile +from ctypes import c_void_p, c_int64 + + +@bpf +@section("tracepoint/syscalls/sys_enter_execve") +def hello(ctx: c_void_p) -> c_int64: + total: c_int64 = 0 + for i in range(10, 0, -3): + total = total + i + return total + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/passing_tests/loops/rebind_loop_var.py b/tests/passing_tests/loops/rebind_loop_var.py new file mode 100644 index 00000000..b994f213 --- /dev/null +++ b/tests/passing_tests/loops/rebind_loop_var.py @@ -0,0 +1,23 @@ +# Rebinding the loop variable does not change the trip count: range() steps a +# counter of its own, as in Python. Returns 10. +from pythonbpf import bpf, section, bpfglobal, compile +from ctypes import c_void_p, c_int64 + + +@bpf +@section("tracepoint/syscalls/sys_enter_execve") +def hello(ctx: c_void_p) -> c_int64: + total: c_int64 = 0 + for i in range(10): + total = total + 1 + i = 100 # noqa: F841 -- rebinding is the point of the test + return total + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/failing_tests/loops/while_basic.py b/tests/passing_tests/loops/while_basic.py similarity index 100% rename from tests/failing_tests/loops/while_basic.py rename to tests/passing_tests/loops/while_basic.py diff --git a/tests/failing_tests/loops/while_true_break.py b/tests/passing_tests/loops/while_true_break.py similarity index 100% rename from tests/failing_tests/loops/while_true_break.py rename to tests/passing_tests/loops/while_true_break.py diff --git a/tests/passing_tests/return/if_narrow.py b/tests/passing_tests/return/if_narrow.py new file mode 100644 index 00000000..c8fdfa72 --- /dev/null +++ b/tests/passing_tests/return/if_narrow.py @@ -0,0 +1,22 @@ +# A return inside an if-branch is lowered in the function's declared return +# type (i32 here), not a default i64 that llc rejects. +from pythonbpf import bpf, section, bpfglobal, compile +from ctypes import c_void_p, c_int32 + + +@bpf +@section("tracepoint/syscalls/sys_enter_execve") +def hello(ctx: c_void_p) -> c_int32: + x = 1 + if x == 1: + return 7 + return 0 + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/passing_tests/return/xdp_name_shadowed.py b/tests/passing_tests/return/xdp_name_shadowed.py index 0caacd82..b188dd26 100644 --- a/tests/passing_tests/return/xdp_name_shadowed.py +++ b/tests/passing_tests/return/xdp_name_shadowed.py @@ -1,7 +1,9 @@ -# A local named after an XDP action must shadow the helper constant table, -# in return position too. clang agrees: a local legally shadows an enum -# constant, and the local's value is what returns (tests/c-form reference). -# Before the fix this returned the hardcoded 2 while XDP_PASS held 55. +# A local named after an XDP action is just a local: it shadows the vmlinux +# enum constant of that name (when vmlinux is imported) exactly as a local +# shadows an enum constant in C, and its value is what returns. This once +# went through a special-cased return path that ignored the local and +# returned the hardcoded 2 while XDP_PASS held 55; that path is gone, and +# return resolves names like every other expression. from pythonbpf import bpf, section, bpfglobal, compile from ctypes import c_void_p, c_int64 diff --git a/tests/passing_tests/vmlinux/ctx_field_narrow_store.py b/tests/passing_tests/vmlinux/ctx_field_narrow_store.py new file mode 100644 index 00000000..5f0d8bff --- /dev/null +++ b/tests/passing_tests/vmlinux/ctx_field_narrow_store.py @@ -0,0 +1,32 @@ +# A context field is loaded widened to i64, but it can be stored into a slot +# of its own declared width, or narrower: the store truncates like any other +# integer conversion. The cgroup_skb_direct_packet_access.c selftest does +# exactly this with `__u32 data_end = skb->data_end`. +from ctypes import c_int64, c_uint16, c_uint32 +from pythonbpf import bpf, section, bpfglobal, compile +from vmlinux import struct_xdp_md + + +@bpf +@bpfglobal +def ifindex() -> c_uint32: + return c_uint32(0) + + +@bpf +@section("xdp") +def prog(ctx: struct_xdp_md) -> c_int64: + global ifindex + ifindex = ctx.ingress_ifindex + queue = c_uint16(0) + queue = ctx.rx_queue_index + return c_int64(queue) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/passing_tests/vmlinux/probe_read_kernel_scalar.py b/tests/passing_tests/vmlinux/probe_read_kernel_scalar.py new file mode 100644 index 00000000..c1bf40d6 --- /dev/null +++ b/tests/passing_tests/vmlinux/probe_read_kernel_scalar.py @@ -0,0 +1,34 @@ +# probe_read_kernel into a scalar reads the scalar's width, and the source may +# be an address held in a register field (C: (void *)ctx->si). bpfsnake reads +# one typed byte this way: `@key = *((int8*)arg1)`. +from ctypes import c_int8, c_int64 +from pythonbpf import bpf, section, bpfglobal, compile +from pythonbpf.helper import probe_read_kernel +from vmlinux import struct_pt_regs + + +@bpf +@bpfglobal +def last() -> c_int64: + return c_int64(0) + + +# pty_write(struct tty_struct *tty, const u8 *buf, size_t c) +@bpf +@section("kprobe/pty_write") +def prog(ctx: struct_pt_regs) -> c_int64: + global last + if ctx.dx == 1: + byte = c_int8(0) + probe_read_kernel(byte, ctx.si) + last = byte + return c_int64(0) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/passing_tests/return/xdp.py b/tests/passing_tests/vmlinux/return_xdp.py similarity index 88% rename from tests/passing_tests/return/xdp.py rename to tests/passing_tests/vmlinux/return_xdp.py index 3c0f5d8c..3978c729 100644 --- a/tests/passing_tests/return/xdp.py +++ b/tests/passing_tests/vmlinux/return_xdp.py @@ -1,6 +1,6 @@ from pythonbpf import bpf, section, bpfglobal, compile from ctypes import c_void_p, c_int64 -from pythonbpf.helper import XDP_PASS +from vmlinux import XDP_PASS @bpf diff --git a/tests/passing_tests/vmlinux/shared_ctx_name.py b/tests/passing_tests/vmlinux/shared_ctx_name.py new file mode 100644 index 00000000..ab93fc17 --- /dev/null +++ b/tests/passing_tests/vmlinux/shared_ctx_name.py @@ -0,0 +1,27 @@ +# Two programs whose context parameters share a name and a vmlinux type. Their +# debug info used to share one cached DILocalVariable, scoped to both +# functions, and llvmlite recursed forever hashing the resulting cycle. +from ctypes import c_int64 +from pythonbpf import bpf, section, bpfglobal, compile +from vmlinux import struct_pt_regs + + +@bpf +@section("kprobe/do_unlinkat") +def first(ctx: struct_pt_regs) -> c_int64: + return c_int64(0) + + +@bpf +@section("kprobe/do_rmdir") +def second(ctx: struct_pt_regs) -> c_int64: + return c_int64(0) + + +@bpf +@bpfglobal +def LICENSE() -> str: + return "GPL" + + +compile() diff --git a/tests/test_config.toml b/tests/test_config.toml index 6f9bec08..309caff0 100644 --- a/tests/test_config.toml +++ b/tests/test_config.toml @@ -6,6 +6,8 @@ # level "ir" = fails during pythonbpf IR generation (exception or ERROR log) # level "llc" = IR generates but llc rejects it # level "verifier" = IR and llc both succeed, but the kernel verifier rejects it +# A verifier-level entry may add match = "..." : the rejection must contain +# that text, otherwise the test fails instead of counting as expected. # # A failure at one level implies failure at every later one, so the declared # level marks that level and all later ones xfail. @@ -24,7 +26,7 @@ "failing_tests/vmlinux/args_test.py" = {reason = "struct_trace_event_raw_sys_enter args field access not supported", level = "ir"} -"failing_tests/xdp_pass.py" = {reason = "XDP program using vmlinux structs (struct_xdp_md) and complex map/struct interaction not yet supported", level = "ir"} +"failing_tests/vmlinux/xdp_pass.py" = {reason = "XDP program using vmlinux structs (struct_xdp_md) and complex map/struct interaction not yet supported", level = "ir"} "failing_tests/globals_read_before_shadow.py" = {reason = "Reading a name above the assignment that makes it a local shadowing a global is UnboundLocalError in Python, and a compile error here", level = "ir"} @@ -36,27 +38,19 @@ "failing_tests/globals_parameter_and_global.py" = {reason = "A parameter may not be declared global (Python: name is parameter and global)", level = "ir"} -"failing_tests/assign/annotated_assign.py" = {reason = "Annotated assignment (ast.AnnAssign, e.g. `x: c_int64 = 0`) is not handled by process_stmt", level = "ir"} +"failing_tests/loops/for_map_items.py" = {reason = "Loops iterate range() only: no sugar over bpf_for_each_map_elem() (a callback subprogram plus a way to capture outer locals) exists yet", level = "ir"} -# Loop constructs: ast.For/ast.While are not handled by the statement dispatcher -# (pythonbpf/functions/functions_pass.py process_stmt only matches Expr, Assign, -# AugAssign, If, Return) and the pre-pass variable scan does not descend into -# loop bodies. A loop is silently skipped rather than erroring, so anything -# that reads a variable assigned in/around the loop afterward blows up later -# with "Undefined variable ..." at the `return` statement. These document the -# intended surface for when loops are implemented. -"failing_tests/loops/for_range_sum.py" = {reason = "for/while loops not implemented: ast.For is silently skipped by the statement dispatcher", level = "ir"} +"failing_tests/loops/break_outside_loop.py" = {reason = "`break` outside a loop is a SyntaxError in Python, and a compile error here", level = "ir"} -"failing_tests/loops/for_range_start_stop_step.py" = {reason = "for/while loops not implemented: range() with start/stop/step is not supported", level = "ir"} +"failing_tests/loops/range_variable_step.py" = {reason = "range() step must be an integer literal: its sign decides the loop test", level = "ir"} -"failing_tests/loops/while_basic.py" = {reason = "for/while loops not implemented: ast.While is silently skipped by the statement dispatcher", level = "ir"} +"kernel_selftest_equivalent/ringbuf/reserve_submit_discard.py" = {reason = "RingBuffer reserve/typed record/discard workflow is planned but not implemented yet", level = "ir"} -"failing_tests/loops/while_true_break.py" = {reason = "for/while loops not implemented: while True with a break-based exit is not supported", level = "ir"} +"kernel_selftest_equivalent/vmlinux/perf_skip.py" = {reason = "Nested struct field access (ctx.regs.ip) not supported; one level such as ctx.sample_period works", level = "ir"} -"failing_tests/loops/nested_for.py" = {reason = "for/while loops not implemented: nested for loops are not supported", level = "ir"} - -"failing_tests/loops/for_break.py" = {reason = "for/while loops not implemented: break is not supported", level = "ir"} - -"failing_tests/loops/for_continue.py" = {reason = "for/while loops not implemented: continue is not supported", level = "ir"} - -"failing_tests/loops/for_map_items.py" = {reason = "for/while loops not implemented: no sugar over bpf_for_each_map_elem()-style map iteration exists yet", level = "ir"} +# Upstream is a negative fixture: reading xdp_md->egress_ifindex is only legal +# for programs loaded with expected_attach_type = BPF_XDP_DEVMAP, and the +# prog_tests driver asserts that a plain load *fails*. The kernel rejects it +# here with "invalid bpf_context access off=20 size=4", which is the pass +# condition upstream. PythonBPF has no way to set expected_attach_type yet. +"kernel_selftest_equivalent/vmlinux/xdp_devmap_helpers.py" = {reason = "Negative fixture: egress_ifindex needs expected_attach_type=BPF_XDP_DEVMAP, which cannot be set yet; the verifier rejection is upstream's pass condition", level = "verifier", match = "invalid bpf_context access"} diff --git a/tests/test_signedness_ir.py b/tests/test_signedness_ir.py index 2166ed88..13341bd8 100644 --- a/tests/test_signedness_ir.py +++ b/tests/test_signedness_ir.py @@ -68,6 +68,12 @@ [r"\bsub i64", r"trunc i64 .* to i32", r"zext i32 .* to i64"], [], ), + # c_uint16(0) declares a 16-bit slot, so a u32 ctx field stored into it + # is truncated to 16 bits (C: __u16 queue = ctx->rx_queue_index). + "vmlinux/ctx_field_narrow_store.py": ( + [r'%"queue" = alloca i16', r"trunc i64 .* to i16"], + [r'%"queue" = alloca i64'], + ), # A bool widens with zext and an integer narrows to it by != 0, never by # trunc; sext of an i1 would return -1 for True. "signedness/bool_int.py": ( @@ -87,6 +93,18 @@ [r"deref_0_not_null", r"ret i64 %"], [r"ret i64\*"], ), + # One byte, sized from the c_int8 destination, read from the address in + # a register field. + "vmlinux/probe_read_kernel_scalar.py": ( + [r'%"byte" = alloca i8', r"inttoptr i64 .* to i8\*", r"i32 1, i8\*"], + [], + ), + # Both sides of each comparison are loaded before the compare; the + # pointers into the map are never compared themselves. + "assign/map_value_compare.py": ( + [r"icmp eq i64 %"], + [r"icmp (eq|ne) i64\* %\S+, %"], + ), } diff --git a/tests/test_verifier.py b/tests/test_verifier.py index 3966e3f6..bd540583 100644 --- a/tests/test_verifier.py +++ b/tests/test_verifier.py @@ -54,7 +54,7 @@ def _get_rejection_reason(verifier_test_file: Path, output) -> str: _verifier_test_files(), ids=_verifier_test_ids(), ) -def test_kernel_verifier(verifier_test_file: Path, tmp_path, caplog): +def test_kernel_verifier(verifier_test_file: Path, tmp_path, caplog, request): """Compile the BPF test and verify it passes the kernel verifier.""" ll_path = tmp_path / "output.ll" obj_path = tmp_path / "output.o" @@ -70,4 +70,13 @@ def test_kernel_verifier(verifier_test_file: Path, tmp_path, caplog): assert obj_path.exists() and obj_path.stat().st_size > 0 ok, output = verify_object(obj_path) + expected = request.node.get_closest_marker("verifier_match") + if not ok and expected is not None and expected.args[0] not in output.stderr: + # pytest.fail raises an OutcomeException, which the xfail marker's + # raises=Exception does not swallow: a rejection for the wrong + # reason is a failure, not the expected one. + pytest.fail( + f"{verifier_test_file.name}: rejected, but not for the expected reason " + f"{expected.args[0]!r}:\n{output.stderr}" + ) assert ok, _get_rejection_reason(verifier_test_file, output) diff --git a/tools/selftest-audit.py b/tools/selftest-audit.py new file mode 100755 index 00000000..26f6798a --- /dev/null +++ b/tools/selftest-audit.py @@ -0,0 +1,329 @@ +#!/usr/bin/env python3 +"""Score the kernel's BPF selftest programs against what PythonBPF can express. + +Point it at a checkout of tools/testing/selftests/bpf/progs and it prints, for +every real program there, the constructs that keep it out of PythonBPF today. +Programs with no hard blocker are the porting candidates; the soft flags on +them say what a port has to rewrite by hand. + + python3 tools/selftest-audit.py path/to/linux/tools/testing/selftests/bpf/progs + python3 tools/selftest-audit.py progs/ --json > audit.json + python3 tools/selftest-audit.py progs/ --histogram + +This is a heuristic scan of the C source, not a compiler: it has false +negatives in both directions, and the envelope it encodes (the helper, map and +construct lists below) must be kept in step with the compiler by hand. Re-run +it after a feature lands to see what the feature unlocked. The lists were last +reconciled against pythonbpf/helper and pythonbpf/maps when scalar @bpfglobal +support and integer signedness merged. +""" + +import argparse +import json +import re +import sys +from collections import Counter +from pathlib import Path + +# ── the envelope ───────────────────────────────────────────────────────────── + +# Kernel helpers pythonbpf/helper can emit (bpf_trace_printk is `print`). +SUPPORTED_HELPERS = { + "bpf_get_current_cgroup_id", + "bpf_get_current_comm", + "bpf_get_current_pid_tgid", + "bpf_get_current_uid_gid", + "bpf_get_prandom_u32", + "bpf_get_smp_processor_id", + "bpf_get_stack", + "bpf_ktime_get_ns", + "bpf_map_delete_elem", + "bpf_map_lookup_elem", + "bpf_map_update_elem", + "bpf_perf_event_output", + "bpf_printk", + "bpf_trace_printk", + "bpf_probe_read", + "bpf_probe_read_kernel", + "bpf_probe_read_kernel_str", + "bpf_ringbuf_output", + "bpf_ringbuf_reserve", + "bpf_ringbuf_submit", + "bpf_skb_store_bytes", +} + +# BPF_MAP_TYPE_* that pythonbpf/maps lowers. +SUPPORTED_MAP_TYPES = {"ARRAY", "HASH", "PERF_EVENT_ARRAY", "RINGBUF"} + +# Things that look like helper calls but are libbpf macros, not helpers. +NOT_HELPERS = { + "bpf_htons", + "bpf_ntohs", + "bpf_htonl", + "bpf_ntohl", + "bpf_be64_to_cpu", + "bpf_cpu_to_be64", + "bpf_printk_", +} + +# (label, kind, regex). kind is "hard" (a language gap) or "soft" (a porting +# cost a careful rewrite can absorb). Order does not matter. +PATTERNS = [ + # control flow + ("loop", "hard", r"\b(for|while)\s*\(|\bbpf_for\b|\bbpf_repeat\b|\bbpf_loop\s*\("), + ("goto", "hard", r"\bgoto\s+\w+"), + ("switch", "hard", r"\bswitch\s*\("), + ("ternary", "soft", r"\?[^?:]*:"), + # entry-point shape + ( + "typed_prog_macro", + "hard", + r"\bBPF_(PROG|PROG2|KPROBE|KRETPROBE|KSYSCALL|KPROBE_SYSCALL|UPROBE|URETPROBE|USDT|" + r"TRACE_\w+|ITER\w*|LSM\w*)\s*\(", + ), + ("struct_ops", "hard", r'SEC\s*\(\s*"\.?struct_ops'), + ("freplace", "hard", r'SEC\s*\(\s*"freplace'), + ("sleepable_or_special_sec", "soft", r'SEC\s*\(\s*"\?'), + # verifier-test harness + ( + "verifier_annotation", + "hard", + r"\b__(failure|success|msg|retval|naked|log_level|flag|arch_\w+|description|" + r"jited|xlated|caps_unpriv|load_if_JITed|not_msg|failure_unpriv|success_unpriv)\b", + ), + # functions + ("subprog_call", "hard", r"\b__noinline\b|\b__weak\b"), + ( + "static_helper", + "soft", + r"\bstatic\s+(__always_inline\s+|inline\s+|__noinline\s+)?\w[\w\s\*]*\s+\**\w+\s*\([^;]*\)\s*\{", + ), + # kernel features + ( + "kfunc", + "hard", + r"__ksym\b|bpf_experimental\.h|bpf_kfuncs\.h|\bbpf_(obj_new|obj_drop|refcount|task_from|" + r"task_acquire|task_release|cgroup_acquire|cgroup_release|cpumask_\w+|rbtree_\w+|list_\w+|" + r"rcu_read_lock|rcu_read_unlock|arena_\w+|key_put|lookup_user_key|dynptr_\w+|iter_\w+|" + r"wq_\w+|timer_\w+|throw|percpu_obj_\w+|res_spin_\w+|preempt_\w+|local_irq_\w+|" + r"session_\w+|get_dentry_xattr|get_file_xattr|kptr_xchg|sk_assign|" + r"xdp_metadata_\w+|xdp_flow_lookup|skb_flow_lookup)\s*\(", + ), + ("inline_asm", "hard", r"\basm\s*(volatile)?\s*\(|__asm__"), + ("atomic", "hard", r"__sync_\w+|__atomic_\w+|\bbpf_spin_(lock|unlock)\b"), + ("tail_call", "hard", r"\bbpf_tail_call\w*\s*\("), + ( + "core_read", + "hard", + r"\bBPF_CORE_READ\w*\b|\bbpf_core_\w+|__builtin_preserve\w*|\bbpf_probe_read_user\w*", + ), + ( + "builtin", + "hard", + r"__builtin_(memcpy|memset|memcmp|bswap\w*|ctz|clz|popcount|expect)\b|\b(memcpy|memset|memcmp)\s*\(", + ), + ( + "endian_macro", + "hard", + r"\bbpf_(htons|ntohs|htonl|ntohl|be64_to_cpu|cpu_to_be64)\s*\(", + ), + ("kconfig_or_extern", "hard", r"__kconfig\b|^\s*extern\s"), + ("arena_or_iter_sec", "hard", r'SEC\s*\(\s*"(iter|arena)'), + # data + ("ctx_field_write", "soft", r"\bctx\s*->\s*\w+\s*(\+|-|\||&|\^|<<|>>)?=[^=]"), + ("local_struct", "soft", r"^\s+struct\s+\w+\s+\w+\s*(=\s*\{|;)"), + ( + "local_array", + "soft", + r"^\s+(const\s+)?(char|__u8|__u16|__u32|__u64|int|long|unsigned|u8|u16|u32|u64|__s\d+)\s+(?!_?_?license\b)\w+\s*\[[^\]]*\]", + ), + ( + "string_or_char_global", + "hard", + r"^(static\s+)?(volatile\s+)?(const\s+)?(volatile\s+)?char\s+\w+\s*\[[^\]]*\]\s*(SEC\s*\(\s*\"\.rodata\"\s*\))?\s*=", + ), +] + +# File-scope declarations that are not ordinary scalar globals. +ARRAY_GLOBAL = re.compile( + r"^(static\s+)?(volatile\s+)?(const\s+)?(volatile\s+)?" + r"(struct\s+\w+|__?[us]\d+|u\d+|s\d+|int|long|short|char|bool|unsigned\s+\w+|" + r"uintptr_t|size_t|__wsum|__be\d+|__le\d+|\w+_t)\s*\**\s*\w+\s*\[", + re.M, +) +STRUCT_GLOBAL = re.compile( + r"^(static\s+)?(volatile\s+)?(const\s+)?(volatile\s+)?struct\s+\w+\s+\w+\s*(=|;)", + re.M, +) +ANON_STRUCT_GLOBAL = re.compile(r"^struct\s*\{", re.M) +MAP_DECL = re.compile(r"__uint\s*\(\s*type\s*,\s*BPF_MAP_TYPE_(\w+)\s*\)") +SEC_RE = re.compile(r'SEC\s*\(\s*"([^"]+)"\s*\)') +HELPER_CALL = re.compile(r"\b(bpf_\w+)\s*\(") +INCLUDE_C = re.compile(r'^\s*#include\s+"[^"]+\.c"', re.M) +BTF_DUMP_FIXTURE = re.compile(r"btf_dump|btf__|__attribute__\(\(btf_decl_tag", re.I) + + +_STRING_OR_COMMENT = re.compile(r'("(?:\\.|[^"\\\n])*")|/\*.*?\*/|//[^\n]*', re.S) + + +def strip_comments(src: str) -> str: + """Remove C comments, leaving string literals alone: a // inside a string + such as SEC("uprobe//proc/self/exe:func") is part of the section name.""" + return _STRING_OR_COMMENT.sub(lambda m: m.group(1) or "", src) + + +def classify(path: Path) -> dict | None: + raw = path.read_text(errors="replace") + src = strip_comments(raw) + secs = [s for s in SEC_RE.findall(src) if s not in ("license", ".maps", "version")] + is_prog = bool(secs) and not INCLUDE_C.search(src) + if not is_prog: + return None # wrapper shim, header-only fixture, or library file + + hard: set[str] = set() + soft: set[str] = set() + detail: dict[str, list[str]] = {} + + for label, kind, rx in PATTERNS: + if re.search(rx, src, re.M): + (hard if kind == "hard" else soft).add(label) + + maps = set(MAP_DECL.findall(src)) + bad_maps = sorted(m for m in maps if m not in SUPPORTED_MAP_TYPES) + if bad_maps: + hard.add("unsupported_map") + detail["unsupported_map"] = bad_maps + if re.search(r'SEC\s*\(\s*"\.maps"\s*\)', src) and not maps: + # a map with no __uint(type) is a legacy bpf_map_def or an extern + hard.add("legacy_map_def") + + helpers = set(HELPER_CALL.findall(src)) - NOT_HELPERS + unsupported = sorted( + h + for h in helpers + if h not in SUPPORTED_HELPERS + and not h.startswith(("bpf_core_", "bpf_probe_read_user")) + ) + if unsupported: + hard.add("unsupported_helper") + detail["unsupported_helper"] = unsupported + + # file-scope globals that scalar @bpfglobal cannot hold. Map declarations + # are anonymous structs too, so take them out first. + no_maps = re.sub( + r"struct\s*\{[^}]*\}\s*\w+\s*SEC\s*\(\s*\"\.maps\"\s*\)\s*;", + "", + src, + flags=re.S, + ) + body_stripped = re.sub(r"\{[^{}]*\}", "{}", no_maps) # crude: drop innermost bodies + for _ in range(6): + body_stripped = re.sub(r"\{[^{}]*\}", "{}", body_stripped) + top = "\n".join( + line + for line in body_stripped.splitlines() + if not line.lstrip().startswith(("#", "SEC", "}", "{")) + ) + if ARRAY_GLOBAL.search(top) and not re.search( + r"^char\s+_?_?license", top, re.M | re.I + ): + hard.add("array_global") + elif ARRAY_GLOBAL.search(top): + # licence/version aside, any other array is still a blocker + others = [ + m.group(0) + for m in ARRAY_GLOBAL.finditer(top) + if "license" not in m.group(0).lower() + ] + if others: + hard.add("array_global") + if STRUCT_GLOBAL.search(top) or ANON_STRUCT_GLOBAL.search(top): + hard.add("struct_global") + + # a global written from a static helper etc. is fine; a global at all is + # informational now that scalars are supported + if re.search( + r"^(volatile\s+)?(const\s+)?(volatile\s+)?(__?[us]\d+|u\d+|s\d+|int|long|short|bool|unsigned\s+\w+|uintptr_t|size_t)\s+\w+\s*(=[^=]|;)", + top, + re.M, + ): + soft.add("scalar_global") + + return { + "file": path.name, + "bytes": len(raw), + "sections": sorted(set(secs)), + "hard": sorted(hard), + "soft": sorted(soft), + "detail": detail, + } + + +def main() -> int: + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument( + "progs", type=Path, help="tools/testing/selftests/bpf/progs checkout" + ) + ap.add_argument( + "--json", action="store_true", help="emit one JSON object per program" + ) + ap.add_argument( + "--histogram", action="store_true", help="print the blocker histogram" + ) + ap.add_argument( + "--all", + action="store_true", + help="list every program, not only the portable ones", + ) + ap.add_argument( + "--max-hard", + type=int, + default=0, + help="list programs with at most this many hard blockers", + ) + args = ap.parse_args() + + results = [] + skipped = 0 + for c in sorted(args.progs.glob("*.c")): + r = classify(c) + if r is None: + skipped += 1 + else: + results.append(r) + + if args.json: + for r in results: + print(json.dumps(r)) + return 0 + + real = len(results) + clean = [r for r in results if not r["hard"]] + print( + f"{real} real programs ({skipped} shims/fixtures skipped); {len(clean)} with no hard blocker\n" + ) + + if args.histogram: + hist = Counter(b for r in results for b in r["hard"]) + for label, n in hist.most_common(): + print(f" {label:28s} {n:4d} {100 * n / real:4.0f}%") + print() + + rows = ( + results if args.all else [r for r in results if len(r["hard"]) <= args.max_hard] + ) + rows.sort(key=lambda r: (len(r["hard"]), r["bytes"])) + for r in rows: + flags = " ".join(r["hard"]) or "-" + soft = " ".join(r["soft"]) or "-" + extra = "; ".join(f"{k}={','.join(v)}" for k, v in r["detail"].items()) + print( + f"{r['file']:44s} {r['bytes']:6d} hard: {flags:30s} soft: {soft} {extra}" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main())