fix(pycg): key the shard module graph by file path, not module_name

PyModule.module_name is only the file stem (py_file.stem), which collides
heavily across a real project — every __init__.py, models.py, views.py shares
a name. Keying the partition graph by module_name collapsed all same-stem
files into a single node and, via the last-wins module_name->file_path map,
silently dropped every colliding file but one from the shards.

Observed on odoo: a 1028-file symbol table produced a graph of only 399
nodes (4 fat shards), so ~600 files were never handed to PyCG.

Key graph nodes by file_path (unique) instead; carry module_name as a node
attribute for readable reporting. plan_shards now emits file-path shards
directly (no name->file remap) with a parallel module_shards name view.

Add a regression test asserting every file lands in exactly one shard under
stem collisions, and update graph tests for file-keyed nodes.

Author: rahlk

codeanalyzer/semantic_analysis/pycg/shard_planner.py

@@ -61,7 +61,7 @@
 
 
 # ----------------------------------------------------------------------------
-# Symbol-table walking: callable / class signature -> owning module
+# Symbol-table walking: callable / class signature -> defining file
 # ----------------------------------------------------------------------------
 
 def _walk_callable_sigs(c: PyCallable) -> Iterator[str]:
@@ -80,22 +80,25 @@ def _walk_class_sigs(cls: PyClass) -> Iterator[str]:
         yield from _walk_class_sigs(inner)
 
 
-def _signature_to_module(symbol_table: Dict[str, PyModule]) -> Dict[str, str]:
-    """Map every callable/class signature in the project to its module name.
+def _signature_to_file(symbol_table: Dict[str, PyModule]) -> Dict[str, str]:
+    """Map every callable/class signature in the project to its defining file.
 
-    Built by walking each module's full nesting tree, so the mapping is exact
-    rather than relying on longest-prefix matching against module names (which
-    is ambiguous when one module name is a prefix of another).
+    Built by walking each module's full nesting tree, recording the file that
+    actually defines each callable.  Files are the partition unit because
+    ``PyModule.module_name`` is only the file *stem* (``py_file.stem``), which
+    collides heavily across a real project (every ``__init__.py``, ``models.py``
+    …) — keying the module graph by name would collapse unrelated files into
+    one node and silently drop files from shards.  ``file_path`` is unique.
     """
-    sig_to_mod: Dict[str, str] = {}
+    sig_to_file: Dict[str, str] = {}
     for module in symbol_table.values():
         for fn in module.functions.values():
             for sig in _walk_callable_sigs(fn):
-                sig_to_mod[sig] = module.module_name
+                sig_to_file[sig] = module.file_path
         for cls in module.classes.values():
             for sig in _walk_class_sigs(cls):
-                sig_to_mod[sig] = module.module_name
-    return sig_to_mod
+                sig_to_file[sig] = module.file_path
+    return sig_to_file
 
 
 # ----------------------------------------------------------------------------
@@ -133,22 +136,24 @@ def build_module_graph(
     symbol_table: Dict[str, PyModule],
     jedi_edges: List[PyCallEdge],
 ) -> nx.DiGraph:
-    """Project Jedi callable→callable edges onto a weighted module DiGraph.
-
-    Every project module is a node (isolated modules included).  Edge weight is
-    the summed Jedi weight of cross-module call sites; intra-module edges and
-    edges touching external/library symbols (no symbol-table entry) are
-    dropped — they cannot influence how the project is partitioned.
+    """Project Jedi callable→callable edges onto a weighted file DiGraph.
+
+    Nodes are file paths (the unique, collision-free partition unit — see
+    :func:`_signature_to_file`); each carries a ``module_name`` attribute for
+    readable reporting.  Every project file is a node (isolated files
+    included).  Edge weight is the summed Jedi weight of cross-file call sites;
+    intra-file edges and edges touching external/library symbols (no
+    symbol-table entry) are dropped — they cannot influence the partition.
     """
-    sig_to_mod = _signature_to_module(symbol_table)
+    sig_to_file = _signature_to_file(symbol_table)
 
     g = nx.DiGraph()
     for module in symbol_table.values():
-        g.add_node(module.module_name, file_path=module.file_path)
+        g.add_node(module.file_path, module_name=module.module_name)
 
     for edge in jedi_edges:
-        src = sig_to_mod.get(edge.source)
-        dst = sig_to_mod.get(edge.target)
+        src = sig_to_file.get(edge.source)
+        dst = sig_to_file.get(edge.target)
         if src is None or dst is None or src == dst:
             continue
         if g.has_edge(src, dst):
@@ -355,23 +360,25 @@ def plan_shards(
     if merge_small:
         unit_shards = _merge_small(unit_shards, hu, unit_size, budget)
 
-    # Expand SCC units back to modules, then to file paths.
-    module_shards: List[List[str]] = []
+    # Expand SCC units back to file paths (graph nodes are files).
+    file_shards: List[List[str]] = []
     for units in unit_shards:
-        mods: Set[str] = set()
+        files: Set[str] = set()
         for scc in units:
-            mods |= unit_members[scc]
-        if mods:
-            module_shards.append(sorted(mods))
+            files |= unit_members[scc]
+        if files:
+            file_shards.append(sorted(files))
 
-    mod_to_file = {n: g.nodes[n]["file_path"] for n in g.nodes}
-    file_shards = [[mod_to_file[m] for m in mods] for mods in module_shards]
+    # Parallel view in module names (file stems) for human-readable reporting.
+    module_shards = [
+        [g.nodes[f].get("module_name", f) for f in files] for files in file_shards
+    ]
 
     # Metrics: how much Jedi edge weight does this partition sever?
     shard_of: Dict[str, int] = {}
-    for idx, mods in enumerate(module_shards):
-        for m in mods:
-            shard_of[m] = idx
+    for idx, files in enumerate(file_shards):
+        for f in files:
+            shard_of[f] = idx
     cut_weight = 0.0
     for u, v, w in g.edges(data="weight", default=1):
         if shard_of.get(u) != shard_of.get(v):

test/test_shard_planner.py

@@ -31,8 +31,9 @@ def _edge(src: str, dst: str, w: int = 1) -> PyCallEdge:
     return PyCallEdge(source=src, target=dst, weight=w, provenance=["jedi"])
 
 
-def _cut_ratio(g: nx.DiGraph, module_shards: List[List[str]]) -> float:
-    shard_of = {m: i for i, mods in enumerate(module_shards) for m in mods}
+def _cut_ratio(g: nx.DiGraph, file_shards: List[List[str]]) -> float:
+    # Graph nodes are file paths; partitions are lists of file paths.
+    shard_of = {f: i for i, files in enumerate(file_shards) for f in files}
     total = cut = 0.0
     for u, v, w in g.edges(data="weight", default=1):
         total += w
@@ -45,27 +46,41 @@ def _cut_ratio(g: nx.DiGraph, module_shards: List[List[str]]) -> float:
 # build_module_graph
 # ----------------------------------------------------------------------------
 
-def test_module_graph_projects_callables_to_modules():
+def test_module_graph_keys_nodes_by_file_not_name():
+    # Two distinct files share the stem "models" (module_name collision); the
+    # graph must keep them as separate nodes keyed by path, not collapse them.
+    st = {
+        "/pkg_a/models.py": _module("models", "/pkg_a/models.py", ["f"]),
+        "/pkg_b/models.py": _module("models", "/pkg_b/models.py", ["h"]),
+    }
+    edges = [
+        _edge("models.f", "models.h", 3),  # but which file? mapped by definition
+    ]
+    g = build_module_graph(st, edges)
+    assert set(g.nodes) == {"/pkg_a/models.py", "/pkg_b/models.py"}
+    assert g.nodes["/pkg_a/models.py"]["module_name"] == "models"
+
+
+def test_module_graph_projects_callables_to_files():
     st = {
         "/a.py": _module("a", "/a.py", ["f", "g"]),
         "/b.py": _module("b", "/b.py", ["h"]),
     }
     edges = [
-        _edge("a.f", "b.h", 3),   # cross-module -> kept
-        _edge("a.f", "a.g", 5),   # intra-module -> dropped
+        _edge("a.f", "b.h", 3),   # cross-file -> kept
+        _edge("a.f", "a.g", 5),   # intra-file -> dropped
         _edge("a.g", "ext.lib.x"),  # external target -> dropped
     ]
     g = build_module_graph(st, edges)
-    assert set(g.nodes) == {"a", "b"}
-    assert g.has_edge("a", "b") and g["a"]["b"]["weight"] == 3
-    assert not g.has_edge("a", "a")
+    assert set(g.nodes) == {"/a.py", "/b.py"}
+    assert g.has_edge("/a.py", "/b.py") and g["/a.py"]["/b.py"]["weight"] == 3
     assert g.number_of_edges() == 1
 
 
-def test_isolated_modules_are_nodes():
+def test_isolated_files_are_nodes():
     st = {"/a.py": _module("a", "/a.py", ["f"]), "/b.py": _module("b", "/b.py", ["g"])}
     g = build_module_graph(st, [])  # no edges
-    assert set(g.nodes) == {"a", "b"}
+    assert set(g.nodes) == {"/a.py", "/b.py"}
     assert g.number_of_edges() == 0
 
 
@@ -105,9 +120,11 @@ def test_every_module_assigned_exactly_once():
 def test_beats_naive_per_package_cut_ratio():
     st, edges = _coupled_clusters_project()
     g = build_module_graph(st, edges)
+    # Naive baseline: one shard per top-level directory (e.g. /pkg_a/...).
     naive = {}
     for m in st.values():
-        naive.setdefault(m.module_name.split(".")[0], []).append(m.module_name)
+        top = m.file_path.split("/")[1]
+        naive.setdefault(top, []).append(m.file_path)
     naive_ratio = _cut_ratio(g, list(naive.values()))
 
     plan = plan_shards(st, edges, budget=4)
@@ -150,6 +167,21 @@ def test_determinism():
     assert norm(a) == norm(b)
 
 
+def test_no_file_dropped_on_stem_collision():
+    # Regression: module_name is only the file stem, so many files collide on
+    # name (every __init__.py, models.py, ...). Keying by name would drop all
+    # but one per stem. Every FILE must land in exactly one shard.
+    st, edges = {}, []
+    for pkg in ("a", "b", "c", "d"):
+        for stem in ("__init__", "models", "views"):
+            path = f"/{pkg}/{stem}.py"
+            st[path] = _module(stem, path, ["f"])
+    plan = plan_shards(st, edges, budget=5)
+    assigned = sorted(f for shard in plan.shards for f in shard)
+    assert assigned == sorted(st.keys())          # all 12 files present
+    assert len(assigned) == len(set(assigned))    # none duplicated
+
+
 def test_empty_project():
     plan = plan_shards({}, [], budget=4)
     assert plan.shards == []