Merge #105 (selftest roadmap tests, ArrayMap, narrow ctx stores) for bpfsnake

Author: varun-r-mallya

docs/getting-started/quickstart.md

@@ -188,7 +188,7 @@ def trace_open(ctx: c_void_p) -> c_int64:
 For network packet processing:
 
 ```python
-from pythonbpf.helper import XDP_PASS
+from vmlinux import XDP_PASS
 
 @section("xdp")
 def xdp_pass(ctx: c_void_p) -> c_int64:

docs/user-guide/decorators.md

@@ -108,7 +108,7 @@ def trace_open_return(ctx):
 For network packet processing at the earliest point:
 
 ```python
-from pythonbpf.helper import XDP_PASS
+from vmlinux import XDP_PASS
 from ctypes import c_void_p, c_int64
 
 @section("xdp")

docs/user-guide/structs.md

@@ -290,7 +290,8 @@ class MyStruct:
 ```python
 from pythonbpf import bpf, struct, map, section
 from pythonbpf.maps import RingBuffer
-from pythonbpf.helper import ktime, XDP_PASS
+from pythonbpf.helper import ktime
+from vmlinux import XDP_PASS
 from ctypes import c_void_p, c_int64, c_uint8, c_uint16, c_uint32, c_uint64
 
 @bpf

examples/xdp_pass.py

@@ -1,5 +1,5 @@
 from pythonbpf import bpf, map, section, bpfglobal, compile, compile_to_ir
-from pythonbpf.helper import XDP_PASS
+from vmlinux import XDP_PASS
 from pythonbpf.maps import HashMap
 from ctypes import c_int64, c_void_p
 

pyproject.toml

@@ -63,6 +63,7 @@ python_files = ["test_*.py"]
 markers = [
     "verifier: requires sudo/root for kernel verifier tests (not run by default)",
     "vmlinux: requires vmlinux.py for current kernel",
+    "verifier_match: substring a verifier-level xfail expects in the rejection",
 ]
 log_cli = false
 

pythonbpf/allocation_pass.py

@@ -6,7 +6,7 @@
 from pythonbpf.helper import HelperHandlerRegistry
 from pythonbpf.vmlinux_parser.dependency_node import Field
 from .expr import VmlinuxHandlerRegistry
-from pythonbpf.type_deducer import ctypes_to_ir, IntTy, signedness
+from pythonbpf.type_deducer import ctypes_to_ir, is_ctypes, IntTy, signedness
 from pythonbpf.expr.type_inference import infer_int_type
 from pythonbpf.maps import BPFMapType
 
@@ -110,7 +110,9 @@ def _allocate_for_call(builder, var_name, rval, local_sym_tab, compilation_conte
         call_type = rval.func.id
 
         # C type constructors
-        if call_type in ("c_int32", "c_int64", "c_uint32", "c_uint64", "c_void_p"):
+        if is_ctypes(call_type) and isinstance(ctypes_to_ir(call_type), ir.IntType):
+            # Any integer ctypes constructor, c_uint16 included, declares a
+            # slot of that width; the value is converted into it at the store.
             ir_type = ctypes_to_ir(call_type)
             var = builder.alloca(ir_type, name=var_name)
             var.align = ir_type.width // 8
@@ -206,7 +208,7 @@ def _allocate_for_map_method(
         return
 
     map_params = map_sym_tab[map_name].params
-    if map_params["type"] != BPFMapType.HASH:
+    if map_params["type"] not in (BPFMapType.HASH, BPFMapType.ARRAY):
         logger.warning(
             "Map method lookup used on non-hash map, using fallback allocation"
         )

pythonbpf/assign_pass.py

@@ -5,7 +5,7 @@
 from llvmlite import ir
 from pythonbpf.expr import eval_expr, convert
 from pythonbpf.helper import emit_probe_read_kernel_str_call
-from pythonbpf.type_deducer import ctypes_to_ir
+from pythonbpf.type_deducer import field_int_type
 from pythonbpf.vmlinux_parser.dependency_node import Field
 
 logger = logging.getLogger(__name__)
@@ -40,6 +40,10 @@ def handle_struct_field_assignment(
         return
 
     val, val_type = val_result
+    if isinstance(val_type, Field):
+        field_ty = field_int_type(val_type)
+        if field_ty is not None:
+            val_type = field_ty
 
     # Special case: i8* string to [N x i8] char array
     if _is_char_array(field_type) and _is_i8_ptr(val_type):
@@ -150,6 +154,13 @@ def handle_variable_assignment(
     logger.info(
         f"Evaluated value for {var_name}: {val} of type {val_type}, expected {var_type}"
     )
+    # An integer vmlinux field is, for conversion purposes, its declared IntTy
+    # (width and sign from the ctype), so it takes the same convert() path as
+    # every other integer below instead of a special case.
+    if isinstance(val_type, Field):
+        field_ty = field_int_type(val_type)
+        if field_ty is not None:
+            val_type = field_ty
 
     if isinstance(val_type, ir.IntType) and isinstance(var_type, ir.IntType):
         # The descriptor may be narrower than the constant carrying the value
@@ -194,33 +205,13 @@ def handle_variable_assignment(
                 )
                 return False
         if isinstance(val_type, Field):
-            logger.info("Handling assignment to struct field")
-            field_ir_type = ctypes_to_ir(val_type.type.__name__)
-            # Sub-register-width context fields are zero-extended to i64 by
-            # load_ctx_field, so val is already i64 even though the field type
-            # says otherwise (c_uint for xdp_md, c_ushort for pt_regs.cs/ss).
-            if (
-                isinstance(field_ir_type, ir.IntType)
-                and field_ir_type.width < 64
-                and isinstance(var_type, ir.IntType)
-                and var_type.width == 64
-            ):
-                builder.store(val, var_ptr)
-                logger.info(
-                    f"Assigned zero-extended i{field_ir_type.width} context field "
-                    f"to {var_name} (i64)"
-                )
-                return True
-            # TODO: handling only ctype struct fields for now. Handle other stuff too later.
-            elif var_type == field_ir_type:
-                builder.store(val, var_ptr)
-                logger.info(f"Assigned ctype struct field to {var_name}")
-                return True
-            else:
-                logger.error(
-                    f"Failed to assign ctype struct field to {var_name}: {val_type} != {var_type}"
-                )
-                return False
+            # Integer fields were normalised to their IntTy above; what is
+            # left is a pointer, array or struct field, which has no path
+            # into this slot.
+            logger.error(
+                f"Failed to assign ctype struct field to {var_name}: {val_type} != {var_type}"
+            )
+            return False
         elif isinstance(val_type, ir.IntType) and isinstance(var_type, ir.PointerType):
             # NOTE: This is assignment to a PTR_TO_MAP_VALUE_OR_NULL
             logger.info(

pythonbpf/expr/expr_pass.py

@@ -254,6 +254,7 @@ def get_typed_operand(func, compilation_context, operand, builder, local_sym_tab
             vmlinux_result = VmlinuxHandlerRegistry.handle_name(operand.id)
             if vmlinux_result is not None:
                 return vmlinux_result  # (i64 constant, its C rank)
+            raise SyntaxError(f"Undefined variable {operand.id}")
     elif isinstance(operand, ast.Constant):
         if isinstance(operand.value, (int, bool)):
             v = int(operand.value)

pythonbpf/functions/functions_pass.py

@@ -30,7 +30,7 @@
     LocalSymbol,
 )
 from .function_debug_info import generate_function_debug_info
-from .return_utils import handle_none_return, handle_xdp_return, is_xdp_name
+from .return_utils import handle_none_return
 from .function_metadata import get_probe_string, is_global_function, infer_return_type
 
 
@@ -341,20 +341,6 @@ def handle_return(
     logger.info(f"Handling return statement: {ast.dump(stmt)}")
     if stmt.value is None:
         return handle_none_return(builder)
-    elif (
-        isinstance(stmt.value, ast.Name)
-        and is_xdp_name(stmt.value.id)
-        and stmt.value.id not in local_sym_tab
-        and (
-            compilation_context is None
-            or stmt.value.id not in compilation_context.bpf_globals
-        )
-    ):
-        # The XDP fast path resolves names like XDP_PASS from the helper
-        # constant table, but only as a fallback: a local or @bpfglobal of the
-        # same name shadows it, mirroring C (a local shadows an enum constant)
-        # and the resolution order everywhere else in the compiler.
-        return handle_xdp_return(stmt, builder, ret_type)
     else:
         # Fallback for now if ctx not passed, but caller should pass it
         if compilation_context is None:

pythonbpf/functions/return_utils.py

@@ -1,44 +1,12 @@
 import logging
-import ast
 
 from llvmlite import ir
 
 logger: logging.Logger = logging.getLogger(__name__)
 
-XDP_ACTIONS = {
-    "XDP_ABORTED": 0,
-    "XDP_DROP": 1,
-    "XDP_PASS": 2,
-    "XDP_TX": 3,
-    "XDP_REDIRECT": 4,
-}
-
 
 def handle_none_return(builder) -> bool:
     """Handle return or return None -> returns 0."""
     builder.ret(ir.Constant(ir.IntType(64), 0))
     logger.debug("Generated default return: 0")
     return True
-
-
-def is_xdp_name(name: str) -> bool:
-    """Check if a name is an XDP action"""
-    return name in XDP_ACTIONS
-
-
-def handle_xdp_return(stmt: ast.Return, builder, ret_type) -> bool:
-    """Handle XDP returns"""
-    if not isinstance(stmt.value, ast.Name):
-        return False
-
-    action_name = stmt.value.id
-
-    if action_name not in XDP_ACTIONS:
-        raise ValueError(
-            f"Unknown XDP action: {action_name}. Available: {XDP_ACTIONS.keys()}"
-        )
-
-    value = XDP_ACTIONS[action_name]
-    builder.ret(ir.Constant(ret_type, value))
-    logger.debug(f"Generated XDP action return: {action_name} = {value}")
-    return True