Skip to content

Commit 970be90

Browse files
authored
Merge pull request #133 from thisisthepy/feat/tp-module-report
Feat: Package-relative source paths and a structured 'why no C' report
2 parents de9f1fa + a50be42 commit 970be90

6 files changed

Lines changed: 291 additions & 17 deletions

File tree

‎python-multiplatform-ksp/src/main/python/typedpython/__main__.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,8 +82,12 @@ def _build(paths: list[Path], cache: Path, out: Path | None) -> int:
8282
why = f" ({', '.join(m.reasons)})" if m.reasons else ""
8383
what = m.extension.name if m.extension else "no extension (interpreted)"
8484
print(f"{m.status:7} {m.name}: {what}{why}")
85-
for name, reason in sorted(m.skipped.items()):
86-
print(f" interpreted: {name} — {reason}")
85+
rel = m.path.relative_to(root).as_posix()
86+
if m.kind == "not_marked":
87+
print(f" {rel}: not marked for compilation (no `# typedpython: compiled`, no @compiled)")
88+
for item in m.skipped_items:
89+
where = f"{item.file}:{item.line}" if item.line is not None else item.file
90+
print(f" {where}: {item.name}: {item.message}")
8791
if out is not None and m.extension is not None:
8892
out.mkdir(parents=True, exist_ok=True)
8993
shutil.copy2(m.extension, out / m.extension.name)

‎python-multiplatform-ksp/src/main/python/typedpython/cgen.py‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -410,9 +410,16 @@ def kw_index(self, kw: tuple[str, ...]) -> int:
410410
return self.kwtuples.index(kw)
411411

412412

413-
def generate(module: ir.Module, source_path: Path) -> str:
414-
"""One C file implementing `module`; `source_path` is the original Python source, embedded."""
413+
def generate(module: ir.Module, source_path: Path, display_path: str | None = None) -> str:
414+
"""One C file implementing `module`; `source_path` is the original Python source, embedded.
415+
416+
`display_path` is the path embedded as the code object's filename (tracebacks of the interpreted
417+
fallback): the module's path relative to the project root, posix separators (#112). It never
418+
defaults to the absolute host path, so the C does not depend on where the project was built;
419+
left out, it is the file name.
420+
"""
415421
source_path = Path(source_path)
422+
display_path = source_path.name if display_path is None else display_path
416423
source = source_path.read_text(encoding="utf-8")
417424
mg = _ModGen(module)
418425
for f in module.functions:
@@ -438,7 +445,7 @@ def generate(module: ir.Module, source_path: Path) -> str:
438445
w("#pragma STDC FP_CONTRACT OFF")
439446
w("")
440447
w(f"static const char tp_source[] =\n {c_string(source)};")
441-
w(f"static const char tp_source_path[] = {c_string(str(source_path))};")
448+
w(f"static const char tp_source_path[] = {c_string(display_path)};")
442449
w("")
443450
n_names, n_kw = len(mg.names), len(mg.kwtuples)
444451
w("/* Module state: interned names (globals, attributes, keywords, function names) and the")

‎python-multiplatform-ksp/src/main/python/typedpython/frontend.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -293,6 +293,19 @@ def _whole_module(source: str) -> bool:
293293
return next((line.strip() for line in source.splitlines() if line.strip()), "") == MARKER
294294

295295

296+
def is_opted_in(path: Path) -> bool:
297+
"""Whether the front end considers the module for compilation at all: the `# typedpython: compiled`
298+
marker as its first non-empty line, or a module-level `def` / `class` decorated `@compiled`
299+
(the same tests `_targets` and `_classes` apply)."""
300+
source = Path(path).read_text()
301+
if _whole_module(source):
302+
return True
303+
return any(
304+
isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))
305+
and any(isinstance(d, ast.Name) and d.id == "compiled" for d in node.decorator_list)
306+
for node in ast.parse(source, filename=str(path)).body)
307+
308+
296309
def _classes(
297310
tree: ast.Module, source: str, info: _ModuleInfo,
298311
) -> tuple[dict[str, ir.ClassDecl], dict[str, str]]:

‎python-multiplatform-ksp/src/main/python/typedpython/incremental.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,11 @@
77
The front end, the C back end and the C compiler are passed in as callables, so this file depends on
88
none of them (only on `ir`, the contract).
99
10+
The module's relative path is in the key because the C embeds it (#112): the same content moved to
11+
another path must not reuse an extension carrying the old path.
12+
1013
Key encoding (v1): sha256 of the compact, key-sorted JSON of
11-
{"v": 1, "source": sha256-hex(source bytes),
14+
{"v": 1, "path": <module path relative to project root, posix>, "source": sha256-hex(source bytes),
1215
"deps": [[<dep path relative to project root, posix>, <dep interface hash>], ...] sorted,
1316
"compiler_version": str, "platform": str, "flags": [str, ...] in the given order}
1417
Interface hash (v1): sha256 of the compact, key-sorted JSON of
@@ -267,7 +270,7 @@ def build_project(self, sources: list[Path], project_root: Path, *,
267270
dep_hashes[rel(d)] = "src:" + _sha(d.read_bytes())
268271
comps = {"source": source_hash, "deps": dep_hashes, "compiler_version": compiler_version,
269272
"platform": platform, "flags": list(flags)}
270-
key = _sha(_canon({"v": _V, "source": source_hash, "deps": sorted(dep_hashes.items()),
273+
key = _sha(_canon({"v": _V, "path": rel(p), "source": source_hash, "deps": sorted(dep_hashes.items()),
271274
"compiler_version": compiler_version, "platform": platform,
272275
"flags": list(flags)}))
273276
name = _module_name(p, root)

‎python-multiplatform-ksp/src/main/python/typedpython/pipeline.py‎

Lines changed: 62 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
"""
1313
import hashlib
1414
import json
15+
import re
1516
import sysconfig
1617
from dataclasses import dataclass, field
1718
from pathlib import Path
@@ -33,6 +34,11 @@ class Result:
3334
diagnostics: list[verify.Diagnostic] = field(default_factory=list)
3435

3536

37+
def _relative(path: Path, root: Path) -> str:
38+
"""The module's path relative to the project root, posix separators (what the C embeds)."""
39+
return Path(path).resolve().relative_to(Path(root).resolve()).as_posix()
40+
41+
3642
def _gate_or_raise(path: Path) -> None:
3743
errors = [d for d in gate.check([path], mode="compiled") if d.severity == "error"]
3844
if errors:
@@ -47,8 +53,12 @@ def compile_module(
4753
python_include: Path | None = None,
4854
ext_suffix: str | None = None,
4955
runtime_dir: Path | None = None,
56+
project_root: Path | None = None,
5057
) -> Result:
58+
"""`project_root`: the root the embedded source path is relative to (#112); default: the source's
59+
own directory, so the file name alone is embedded."""
5160
path = Path(path).resolve()
61+
root = path.parent if project_root is None else Path(project_root).resolve()
5262
_gate_or_raise(path)
5363

5464
lowered = frontend.lower(path)
@@ -57,7 +67,7 @@ def compile_module(
5767
if not proved.functions:
5868
return Result(path, proved, None, skipped, diagnostics)
5969

60-
c_source = cgen.generate(proved, path)
70+
c_source = cgen.generate(proved, path, _relative(path, root))
6171
extension = cbuild.build(
6272
c_source, path.stem, Path(out_dir),
6373
runtime_dir=runtime_dir, python_include=python_include, ext_suffix=ext_suffix,
@@ -98,6 +108,36 @@ def _tree_hash(*dirs: Path) -> str:
98108
_NO_EXTENSION = ".noext" # cache artefact of a module with nothing compiled
99109

100110

111+
@dataclass(frozen=True)
112+
class SkippedItem:
113+
"""A function or class that stays interpreted, and why (pypackpack's native-level report)."""
114+
name: str
115+
file: str # package-relative, posix
116+
line: int | None
117+
message: str # without the "line N: " prefix
118+
119+
120+
_LINE_PREFIX = re.compile(r"line (\d+): ")
121+
122+
123+
def skipped_items(skipped: dict[str, str], diagnostics: list[verify.Diagnostic], file: str) -> list[SkippedItem]:
124+
"""Structure `ir.Module.skipped` (name -> text, which stays as it is: the incremental interface
125+
hash reads its keys only). The front end writes `line N: reason`; the verifier writes `rule:
126+
message` and records the line in its diagnostic for the function."""
127+
lines: dict[str, int] = {}
128+
for d in diagnostics:
129+
lines.setdefault(d.function, d.source_line)
130+
out = []
131+
for name, text in sorted(skipped.items()):
132+
m = _LINE_PREFIX.match(text)
133+
if m:
134+
line, message = int(m.group(1)), text[m.end():]
135+
else:
136+
line, message = lines.get(name), text
137+
out.append(SkippedItem(name, file, line, message))
138+
return out
139+
140+
101141
@dataclass
102142
class ModuleResult:
103143
path: Path
@@ -107,6 +147,8 @@ class ModuleResult:
107147
reasons: list[str] = field(default_factory=list) # why it was rebuilt (empty if reused)
108148
skipped: dict[str, str] = field(default_factory=dict) # function -> why it stays interpreted
109149
interface: str = ""
150+
kind: str = "compiled" # "compiled" | "nothing_compiled" | "not_marked"
151+
skipped_items: list[SkippedItem] = field(default_factory=list) # `skipped`, structured
110152

111153

112154
@dataclass
@@ -129,16 +171,19 @@ def compile_project(
129171
) -> ProjectResult:
130172
sources = [Path(s).resolve() for s in sources]
131173
cache_dir = Path(cache_dir)
132-
verified: dict[Path, ir.Module] = {}
174+
verified: dict[Path, list[SkippedItem]] = {}
175+
verified_module_skipped: dict[Path, dict[str, str]] = {}
176+
root = Path(project_root).resolve()
133177

134178
def lower(path: Path) -> ir.Module:
135179
_gate_or_raise(path)
136-
proved, _ = verify.verify(frontend.lower(path))
137-
verified[path] = proved # the VERIFIED module: its signatures are what gets compiled
180+
proved, diagnostics = verify.verify(frontend.lower(path))
181+
verified_module_skipped[path] = dict(proved.skipped)
182+
verified[path] = skipped_items(proved.skipped, diagnostics, _relative(path, root)) # the VERIFIED module: its signatures are what gets compiled
138183
return proved
139184

140185
def generate(module: ir.Module, path: Path) -> str:
141-
return cgen.generate(module, path) if module.functions else ""
186+
return cgen.generate(module, path, _relative(path, root)) if module.functions else ""
142187

143188
def compile_c(c_source: str, name: str, out_dir: Path) -> Path:
144189
if not c_source:
@@ -166,11 +211,18 @@ def compile_c(c_source: str, name: str, out_dir: Path) -> Path:
166211
for m in report.modules:
167212
record = side / f"{m.key}.json"
168213
if m.status == "rebuilt":
169-
record.write_text(json.dumps(dict(verified[m.path].skipped)))
214+
record.write_text(json.dumps({
215+
"skipped": dict(verified_module_skipped[m.path]),
216+
"items": [vars(i) for i in verified[m.path]]}))
170217
try:
171-
skipped = json.loads(record.read_text())
172-
except (OSError, ValueError):
173-
skipped = {}
218+
data = json.loads(record.read_text())
219+
skipped = dict(data["skipped"])
220+
items = [SkippedItem(**i) for i in data["items"]]
221+
except (OSError, ValueError, KeyError, TypeError):
222+
skipped, items = {}, []
174223
ext = None if m.artifact is None or m.artifact.suffix == _NO_EXTENSION else m.artifact
175-
out.append(ModuleResult(m.path, m.name, ext, m.status, list(m.reasons), skipped, m.interface))
224+
kind = ("not_marked" if not frontend.is_opted_in(m.path)
225+
else "compiled" if ext is not None else "nothing_compiled")
226+
out.append(ModuleResult(m.path, m.name, ext, m.status, list(m.reasons), skipped, m.interface,
227+
kind, items))
176228
return ProjectResult(out)

0 commit comments

Comments
 (0)