From aea7fcadb1bdb03d601eec5f7918bbe199d6f291 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:53:05 +0900 Subject: [PATCH] Feat: Embed package-relative source paths and report why a module has no C (#112, #128) --- .../src/main/python/typedpython/__main__.py | 8 +- .../src/main/python/typedpython/cgen.py | 13 +- .../src/main/python/typedpython/frontend.py | 13 ++ .../main/python/typedpython/incremental.py | 7 +- .../src/main/python/typedpython/pipeline.py | 72 ++++++- .../src/test/python/test_module_report.py | 195 ++++++++++++++++++ 6 files changed, 291 insertions(+), 17 deletions(-) create mode 100644 python-multiplatform-ksp/src/test/python/test_module_report.py diff --git a/python-multiplatform-ksp/src/main/python/typedpython/__main__.py b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py index 6534afebf..5b3ef98f4 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/__main__.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py @@ -82,8 +82,12 @@ def _build(paths: list[Path], cache: Path, out: Path | None) -> int: why = f" ({', '.join(m.reasons)})" if m.reasons else "" what = m.extension.name if m.extension else "no extension (interpreted)" print(f"{m.status:7} {m.name}: {what}{why}") - for name, reason in sorted(m.skipped.items()): - print(f" interpreted: {name} — {reason}") + rel = m.path.relative_to(root).as_posix() + if m.kind == "not_marked": + print(f" {rel}: not marked for compilation (no `# typedpython: compiled`, no @compiled)") + for item in m.skipped_items: + where = f"{item.file}:{item.line}" if item.line is not None else item.file + print(f" {where}: {item.name}: {item.message}") if out is not None and m.extension is not None: out.mkdir(parents=True, exist_ok=True) shutil.copy2(m.extension, out / m.extension.name) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py index 6359ed9d4..0ded53910 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py @@ -410,9 +410,16 @@ def kw_index(self, kw: tuple[str, ...]) -> int: return self.kwtuples.index(kw) -def generate(module: ir.Module, source_path: Path) -> str: - """One C file implementing `module`; `source_path` is the original Python source, embedded.""" +def generate(module: ir.Module, source_path: Path, display_path: str | None = None) -> str: + """One C file implementing `module`; `source_path` is the original Python source, embedded. + + `display_path` is the path embedded as the code object's filename (tracebacks of the interpreted + fallback): the module's path relative to the project root, posix separators (#112). It never + defaults to the absolute host path, so the C does not depend on where the project was built; + left out, it is the file name. + """ source_path = Path(source_path) + display_path = source_path.name if display_path is None else display_path source = source_path.read_text(encoding="utf-8") mg = _ModGen(module) for f in module.functions: @@ -438,7 +445,7 @@ def generate(module: ir.Module, source_path: Path) -> str: w("#pragma STDC FP_CONTRACT OFF") w("") w(f"static const char tp_source[] =\n {c_string(source)};") - w(f"static const char tp_source_path[] = {c_string(str(source_path))};") + w(f"static const char tp_source_path[] = {c_string(display_path)};") w("") n_names, n_kw = len(mg.names), len(mg.kwtuples) w("/* Module state: interned names (globals, attributes, keywords, function names) and the") diff --git a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py index fa86512c0..bf2bf5555 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py @@ -293,6 +293,19 @@ def _whole_module(source: str) -> bool: return next((line.strip() for line in source.splitlines() if line.strip()), "") == MARKER +def is_opted_in(path: Path) -> bool: + """Whether the front end considers the module for compilation at all: the `# typedpython: compiled` + marker as its first non-empty line, or a module-level `def` / `class` decorated `@compiled` + (the same tests `_targets` and `_classes` apply).""" + source = Path(path).read_text() + if _whole_module(source): + return True + return any( + isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) + and any(isinstance(d, ast.Name) and d.id == "compiled" for d in node.decorator_list) + for node in ast.parse(source, filename=str(path)).body) + + def _classes( tree: ast.Module, source: str, info: _ModuleInfo, ) -> tuple[dict[str, ir.ClassDecl], dict[str, str]]: diff --git a/python-multiplatform-ksp/src/main/python/typedpython/incremental.py b/python-multiplatform-ksp/src/main/python/typedpython/incremental.py index fad342e05..594de1ab2 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/incremental.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/incremental.py @@ -7,8 +7,11 @@ The front end, the C back end and the C compiler are passed in as callables, so this file depends on none of them (only on `ir`, the contract). +The module's relative path is in the key because the C embeds it (#112): the same content moved to +another path must not reuse an extension carrying the old path. + Key encoding (v1): sha256 of the compact, key-sorted JSON of - {"v": 1, "source": sha256-hex(source bytes), + {"v": 1, "path": , "source": sha256-hex(source bytes), "deps": [[, ], ...] sorted, "compiler_version": str, "platform": str, "flags": [str, ...] in the given order} Interface hash (v1): sha256 of the compact, key-sorted JSON of @@ -267,7 +270,7 @@ def build_project(self, sources: list[Path], project_root: Path, *, dep_hashes[rel(d)] = "src:" + _sha(d.read_bytes()) comps = {"source": source_hash, "deps": dep_hashes, "compiler_version": compiler_version, "platform": platform, "flags": list(flags)} - key = _sha(_canon({"v": _V, "source": source_hash, "deps": sorted(dep_hashes.items()), + key = _sha(_canon({"v": _V, "path": rel(p), "source": source_hash, "deps": sorted(dep_hashes.items()), "compiler_version": compiler_version, "platform": platform, "flags": list(flags)})) name = _module_name(p, root) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py b/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py index db3e2ffe1..aca7a74a5 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py @@ -12,6 +12,7 @@ """ import hashlib import json +import re import sysconfig from dataclasses import dataclass, field from pathlib import Path @@ -33,6 +34,11 @@ class Result: diagnostics: list[verify.Diagnostic] = field(default_factory=list) +def _relative(path: Path, root: Path) -> str: + """The module's path relative to the project root, posix separators (what the C embeds).""" + return Path(path).resolve().relative_to(Path(root).resolve()).as_posix() + + def _gate_or_raise(path: Path) -> None: errors = [d for d in gate.check([path], mode="compiled") if d.severity == "error"] if errors: @@ -47,8 +53,12 @@ def compile_module( python_include: Path | None = None, ext_suffix: str | None = None, runtime_dir: Path | None = None, + project_root: Path | None = None, ) -> Result: + """`project_root`: the root the embedded source path is relative to (#112); default: the source's + own directory, so the file name alone is embedded.""" path = Path(path).resolve() + root = path.parent if project_root is None else Path(project_root).resolve() _gate_or_raise(path) lowered = frontend.lower(path) @@ -57,7 +67,7 @@ def compile_module( if not proved.functions: return Result(path, proved, None, skipped, diagnostics) - c_source = cgen.generate(proved, path) + c_source = cgen.generate(proved, path, _relative(path, root)) extension = cbuild.build( c_source, path.stem, Path(out_dir), runtime_dir=runtime_dir, python_include=python_include, ext_suffix=ext_suffix, @@ -98,6 +108,36 @@ def _tree_hash(*dirs: Path) -> str: _NO_EXTENSION = ".noext" # cache artefact of a module with nothing compiled +@dataclass(frozen=True) +class SkippedItem: + """A function or class that stays interpreted, and why (pypackpack's native-level report).""" + name: str + file: str # package-relative, posix + line: int | None + message: str # without the "line N: " prefix + + +_LINE_PREFIX = re.compile(r"line (\d+): ") + + +def skipped_items(skipped: dict[str, str], diagnostics: list[verify.Diagnostic], file: str) -> list[SkippedItem]: + """Structure `ir.Module.skipped` (name -> text, which stays as it is: the incremental interface + hash reads its keys only). The front end writes `line N: reason`; the verifier writes `rule: + message` and records the line in its diagnostic for the function.""" + lines: dict[str, int] = {} + for d in diagnostics: + lines.setdefault(d.function, d.source_line) + out = [] + for name, text in sorted(skipped.items()): + m = _LINE_PREFIX.match(text) + if m: + line, message = int(m.group(1)), text[m.end():] + else: + line, message = lines.get(name), text + out.append(SkippedItem(name, file, line, message)) + return out + + @dataclass class ModuleResult: path: Path @@ -107,6 +147,8 @@ class ModuleResult: reasons: list[str] = field(default_factory=list) # why it was rebuilt (empty if reused) skipped: dict[str, str] = field(default_factory=dict) # function -> why it stays interpreted interface: str = "" + kind: str = "compiled" # "compiled" | "nothing_compiled" | "not_marked" + skipped_items: list[SkippedItem] = field(default_factory=list) # `skipped`, structured @dataclass @@ -129,16 +171,19 @@ def compile_project( ) -> ProjectResult: sources = [Path(s).resolve() for s in sources] cache_dir = Path(cache_dir) - verified: dict[Path, ir.Module] = {} + verified: dict[Path, list[SkippedItem]] = {} + verified_module_skipped: dict[Path, dict[str, str]] = {} + root = Path(project_root).resolve() def lower(path: Path) -> ir.Module: _gate_or_raise(path) - proved, _ = verify.verify(frontend.lower(path)) - verified[path] = proved # the VERIFIED module: its signatures are what gets compiled + proved, diagnostics = verify.verify(frontend.lower(path)) + verified_module_skipped[path] = dict(proved.skipped) + verified[path] = skipped_items(proved.skipped, diagnostics, _relative(path, root)) # the VERIFIED module: its signatures are what gets compiled return proved def generate(module: ir.Module, path: Path) -> str: - return cgen.generate(module, path) if module.functions else "" + return cgen.generate(module, path, _relative(path, root)) if module.functions else "" def compile_c(c_source: str, name: str, out_dir: Path) -> Path: if not c_source: @@ -166,11 +211,18 @@ def compile_c(c_source: str, name: str, out_dir: Path) -> Path: for m in report.modules: record = side / f"{m.key}.json" if m.status == "rebuilt": - record.write_text(json.dumps(dict(verified[m.path].skipped))) + record.write_text(json.dumps({ + "skipped": dict(verified_module_skipped[m.path]), + "items": [vars(i) for i in verified[m.path]]})) try: - skipped = json.loads(record.read_text()) - except (OSError, ValueError): - skipped = {} + data = json.loads(record.read_text()) + skipped = dict(data["skipped"]) + items = [SkippedItem(**i) for i in data["items"]] + except (OSError, ValueError, KeyError, TypeError): + skipped, items = {}, [] ext = None if m.artifact is None or m.artifact.suffix == _NO_EXTENSION else m.artifact - out.append(ModuleResult(m.path, m.name, ext, m.status, list(m.reasons), skipped, m.interface)) + kind = ("not_marked" if not frontend.is_opted_in(m.path) + else "compiled" if ext is not None else "nothing_compiled") + out.append(ModuleResult(m.path, m.name, ext, m.status, list(m.reasons), skipped, m.interface, + kind, items)) return ProjectResult(out) diff --git a/python-multiplatform-ksp/src/test/python/test_module_report.py b/python-multiplatform-ksp/src/test/python/test_module_report.py new file mode 100644 index 000000000..195b7a434 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_module_report.py @@ -0,0 +1,195 @@ +"""Package-relative embedded source path (#112) and the structured per-module report (#128). + +Real Pyrefly, real C builds. SPEC N-8 (embedded path), N-10 (cache), pypackpack's native level. +""" +import re +import shutil +import traceback +import textwrap +from pathlib import Path + +from typedpython import cbuild, cgen, frontend, pipeline, verify + +SRC = "# typedpython: compiled\n" + textwrap.dedent("""\ + def twice(n: int) -> int: + return n * 2 + + + def boom(s: str) -> str: + raise ValueError("boom " + s) + """) + + +def make_project(root: Path) -> Path: + mod = root / "app" / "physics" / "nbody.py" + mod.parent.mkdir(parents=True) + (root / "app" / "__init__.py").write_text("") + (mod.parent / "__init__.py").write_text("") + mod.write_text(SRC) + return mod + + +# --- #112 ---------------------------------------------------------------------------------------- + +def test_generated_c_is_identical_from_two_absolute_roots_and_embeds_the_relative_path( + tmp_path, monkeypatch): + seen = [] + real = cbuild.build + + def build(c_source, *a, **k): + seen.append(c_source) + return real(c_source, *a, **k) + + monkeypatch.setattr(cbuild, "build", build) + roots = [tmp_path / "one" / "rootA", tmp_path / "two" / "deeper" / "rootB"] + for i, root in enumerate(roots): + root.mkdir(parents=True) + mod = make_project(root) + pipeline.compile_project([mod], root, tmp_path / f"cache{i}") + assert len(seen) == 2 and seen[0] == seen[1] + assert 'tp_source_path[] = "app/physics/nbody.py"' in seen[0] + assert str(tmp_path) not in seen[0] # nothing else embeds a host path + + +def test_compile_module_embeds_the_path_relative_to_project_root(tmp_path): + mod = make_project(tmp_path / "r") + r = pipeline.compile_module(mod, tmp_path / "o2", project_root=tmp_path / "r") + ext = cbuild.load("nbody", r.extension) + try: + ext.boom("x") + except ValueError as e: + assert traceback.extract_tb(e.__traceback__)[-1].filename == "app/physics/nbody.py" + else: + raise AssertionError("boom did not raise") + d = pipeline.compile_module(mod, tmp_path / "o3") # default root: the file's directory + try: + cbuild.load("nbody", d.extension).boom("x") + except ValueError as e: + assert traceback.extract_tb(e.__traceback__)[-1].filename == "nbody.py" + + +def test_default_embedded_path_is_the_file_name(tmp_path): + mod = make_project(tmp_path / "r") + lowered, _ = verify.verify(frontend.lower(mod)) + assert 'tp_source_path[] = "nbody.py"' in cgen.generate(lowered, mod) + assert str(tmp_path) not in cgen.generate(lowered, mod) + + +def test_a_moved_file_with_the_same_content_does_not_reuse_the_extension(tmp_path): + root = tmp_path / "p" + (root / "a").mkdir(parents=True) + (root / "b").mkdir() + (root / "a" / "m.py").write_text(SRC) + cache = tmp_path / "cache" + first = pipeline.compile_project([root / "a" / "m.py"], root, cache) + assert first.modules[0].status == "rebuilt" + shutil.move(root / "a" / "m.py", root / "b" / "m.py") + second = pipeline.compile_project([root / "b" / "m.py"], root, cache) + assert second.modules[0].status == "rebuilt" + ext = cbuild.load("m", second.modules[0].extension) + try: + ext.boom("x") + except ValueError as e: + assert traceback.extract_tb(e.__traceback__)[-1].filename == "b/m.py" + + +# --- #128 ---------------------------------------------------------------------------------------- + +MIXED = "# typedpython: compiled\n" + textwrap.dedent("""\ + def ok(n: int) -> int: + return n + 1 + + + def greet(name: str) -> str: + return "hi " + name + """) + + +def build_one(tmp_path, rel, source, cache="cache"): + root = tmp_path / "proj" + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(source) + r = pipeline.compile_project([path], root, tmp_path / cache) + return root, path, r.modules[0] + + +def test_kind_compiled(tmp_path): + _, _, m = build_one(tmp_path, "pkg/mixed.py", MIXED) + assert m.kind == "compiled" and m.extension is not None + [item] = m.skipped_items + assert item.name == "greet" and item.file == "pkg/mixed.py" + assert item.line == 7 and not item.message.startswith("line ") + assert m.skipped["greet"].startswith("line 7: ") # ModuleResult.skipped unchanged + + +def test_kind_nothing_compiled_with_structured_items(tmp_path): + _, _, m = build_one(tmp_path, "pkg/plain.py", + "# typedpython: compiled\n\ndef f(s: str) -> str:\n return s + 'x'\n") + assert m.kind == "nothing_compiled" and m.extension is None + [item] = m.skipped_items + assert (item.name, item.file, item.line) == ("f", "pkg/plain.py", 4) + assert item.message and not re.match(r"line \d+", item.message) + + +def test_kind_not_marked_when_there_is_neither_marker_nor_decorator(tmp_path): + _, _, m = build_one(tmp_path, "pkg/unmarked.py", "def f(x: int) -> int:\n return x\n") + assert m.kind == "not_marked" and m.extension is None and m.skipped_items == [] + + +def test_a_compiled_decorator_alone_opts_in(tmp_path): + src = "@compiled\ndef f(s: str) -> str:\n return s + 'x'\n" + _, _, m = build_one(tmp_path, "pkg/deco.py", src) + assert m.kind == "nothing_compiled" + assert [(i.name, i.line) for i in m.skipped_items] == [("f", 3)] + + +def test_class_items_carry_name_and_line(tmp_path): + src = ("# typedpython: compiled\n\n\nclass Node:\n __slots__ = ('a',)\n" + " def __init__(self, a: int) -> None:\n self.a = a\n") + _, _, m = build_one(tmp_path, "pkg/cls.py", src) + cls = [i for i in m.skipped_items if i.name == "Node"] + assert cls and cls[0].line == 4 and cls[0].file == "pkg/cls.py" + + +def test_verifier_skips_carry_a_line_and_no_prefix(tmp_path): + src = "# typedpython: compiled\n\n\ndef f(x: int) -> int:\n return g(x)\n\n\ndef g(x: int) -> int:\n return x\n" + _, _, m = build_one(tmp_path, "pkg/v.py", src) + for i in m.skipped_items: + assert isinstance(i.line, int) and not i.message.startswith("line ") + + +def test_structure_round_trips_through_the_cache_for_reused_modules(tmp_path): + root, path, first = build_one(tmp_path, "pkg/mixed.py", MIXED) + second = pipeline.compile_project([path], root, tmp_path / "cache").modules[0] + assert second.status == "reused" + assert second.kind == first.kind == "compiled" + assert second.skipped_items == first.skipped_items and second.skipped_items + assert second.skipped == first.skipped + + +def test_not_marked_and_nothing_compiled_survive_reuse(tmp_path): + root, path, _ = build_one(tmp_path, "pkg/plain.py", + "# typedpython: compiled\n\ndef f(s: str) -> str:\n return s + 'x'\n") + again = pipeline.compile_project([path], root, tmp_path / "cache").modules[0] + assert again.status == "reused" and again.kind == "nothing_compiled" and again.skipped_items + root2, path2, _ = build_one(tmp_path, "pkg/un.py", "def f(x: int) -> int:\n return x\n") + again2 = pipeline.compile_project([path2], root2, tmp_path / "cache").modules[0] + assert again2.status == "reused" and again2.kind == "not_marked" + + +def test_cli_build_prints_file_line_name_message_and_not_marked(tmp_path): + import os + import subprocess + import sys + root = tmp_path / "proj" + (root / "pkg").mkdir(parents=True) + (root / "pkg" / "mixed.py").write_text(MIXED) + (root / "pkg" / "un.py").write_text("def f(x: int) -> int:\n return x\n") + src = Path(__file__).parents[2] / "main" / "python" + out = subprocess.run( + [sys.executable, "-m", "typedpython", "build", str(root), "--cache", str(tmp_path / "c")], + capture_output=True, text=True, env=dict(os.environ, PYTHONPATH=str(src))) + assert out.returncode == 0, out.stdout + out.stderr + assert re.search(r"^\s*pkg/mixed\.py:7: greet: \S", out.stdout, re.M), out.stdout + assert re.search(r"pkg/un\.py.*not marked for compilation", out.stdout), out.stdout