feat(pycg): bound the fixpoint with --pycg-max-iter; stop following in-tree deps

Two robustness fixes for level-2 PyCG, motivated by odoo divergence analysis.

1. max_iter cap (--pycg-max-iter, default 50). PyCG runs its PostProcessor
   fixpoint with max_iter=-1 (until convergence). Its abstract domain is
   field-sensitive access paths with no k-limiting/widening, so on heavy
   metaclass/mixin code the def set balloons (measured: 23 odoo ORM files ->
   7.3k defs pass 0, 8.4k pass 1) and convergence may need many O(defs^2)
   passes. Capping passes returns a sound-but-incomplete graph and guarantees
   termination even with --pycg-shard-timeout 0 (which previously hung forever
   on a single diverging shard). Threaded through _run_pycg_batch and the Ray
   worker. Note: the wall-clock timeout is still the guard for shards whose
   individual passes exceed it.

2. Dependency exclusion. PyCG bounds analysis to its package dir via
   "if mod_dir not in mod.__file__". The whole-project path used
   package=project_dir, but an in-tree .codeanalyzer venv / site-packages
   lives under project_dir, so PyCG followed imports into dependencies and
   exploded. Run the whole-project path inside a symlink mini-project (as the
   shards already do) whose root mirrors only the SKIP_DIRS-filtered source,
   so deps resolve outside mod_dir and stay ghost nodes.

Add test/test_pycg_sharding.py (max_iter threading; in-tree dep stays a ghost
and its internals are never analysed).

Author: rahlk

codeanalyzer/__main__.py

@@ -200,6 +200,22 @@ def main(
             ),
         ),
     ] = ShardStrategy.JEDI,
+    pycg_max_iter: Annotated[
+        int,
+        typer.Option(
+            "--pycg-max-iter",
+            help=(
+                "Cap on PyCG's fixpoint passes per shard/project (level 2; "
+                "default 50). PyCG iterates until its points-to state stops "
+                "changing, but its access-path domain has no convergence bound, "
+                "so heavy metaclass/mixin code (e.g. an ORM) can loop with each "
+                "pass costing seconds. The cap returns a sound-but-incomplete "
+                "call graph instead of looping until the timeout kills it. "
+                "Set to -1 for PyCG's unbounded run-to-convergence behaviour."
+            ),
+            min=-1,
+        ),
+    ] = 50,
 ):
     options = AnalysisOptions(
         input=input,
@@ -224,6 +240,7 @@ def main(
         pycg_shard_ceiling=pycg_shard_ceiling,
         pycg_shard_timeout=pycg_shard_timeout,
         pycg_shard_strategy=pycg_shard_strategy,
+        pycg_max_iter=pycg_max_iter,
     )
 
     _set_log_level(options.verbosity)

codeanalyzer/core.py

@@ -683,6 +683,7 @@ def _get_pycg_call_graph(
                 shard_ceiling=self.options.pycg_shard_ceiling,
                 shard_timeout=self.options.pycg_shard_timeout,
                 shard_strategy=self.options.pycg_shard_strategy,
+                max_iter=self.options.pycg_max_iter,
                 using_ray=self.using_ray,
             )
             return pycg.build_call_graph_edges(symbol_table, jedi_edges=jedi_edges)

codeanalyzer/options/options.py

@@ -61,3 +61,4 @@ class AnalysisOptions:
     pycg_shard_ceiling: int = 100
     pycg_shard_timeout: int = 120
     pycg_shard_strategy: ShardStrategy = ShardStrategy.JEDI
+    pycg_max_iter: int = 50

codeanalyzer/semantic_analysis/pycg/pycg_analysis.py

@@ -154,13 +154,14 @@ def _pycg_shard_worker(
     entry_points: List[str],
     package_dir: str,
     prefix: str,
+    max_iter: int = -1,
 ) -> List[tuple]:
     """Run PyCG on one shard; called in a Ray worker process.
 
     Returns a list of ``(source, target, weight)`` tuples that the caller
     converts to :class:`PyCallEdge` objects.  This function is a plain
     module-level callable so it can be pickled by Ray without capturing any
-    class-level state.
+    class-level state.  *max_iter* caps PyCG's fixpoint passes (-1 = unbounded).
     """
     import importlib
     import sys
@@ -191,7 +192,7 @@ def _pycg_shard_worker(
     cg = CallGraphGenerator(
         entry_points=entry_points,
         package=package_dir,
-        max_iter=-1,
+        max_iter=max_iter,
         operation="call-graph",
     )
     cg.analyze()
@@ -366,7 +367,22 @@ class PyCG:
     # --pycg-shard-timeout.  Set to 0 to disable.
     _PYCG_SHARD_TIMEOUT: int = 120
 
-    # Directory names that should never be fed to PyCG as entry points.
+    # Cap on PyCG's outer fixpoint passes.  PyCG runs PostProcessor until the
+    # def/scope/MRO state stops changing; its abstract domain (field-sensitive
+    # access paths, no k-limiting or widening) has no ascending-chain bound, so
+    # on heavy metaclass/mixin code (e.g. an ORM) the def set can balloon into
+    # the thousands and each O(defs^2) pass costs seconds — convergence, if it
+    # comes, takes many passes.  A finite cap turns "loop until killed" into a
+    # sound-but-incomplete result that still returns the edges found so far.
+    # 50 is generous — well-behaved code converges in well under 20 passes —
+    # while bounding the pathological case.  Override via --pycg-max-iter;
+    # -1 restores PyCG's unbounded run-to-convergence behaviour.
+    _PYCG_MAX_ITER: int = 50
+
+    # Directory names that should never be fed to PyCG as entry points, nor
+    # followed into during import resolution (an in-tree .codeanalyzer venv /
+    # site-packages lives under project_dir and would otherwise be pulled into
+    # the package bound and analysed — see _shard_symlink_root).
     _SKIP_DIRS: frozenset = frozenset({
         ".codeanalyzer", ".git", "__pycache__",
         "venv", ".venv", "virtualenv", "env", ".env",
@@ -382,6 +398,7 @@ def __init__(
         shard_ceiling: Optional[int] = None,
         shard_timeout: Optional[int] = None,
         shard_strategy: str = "jedi",
+        max_iter: Optional[int] = None,
         using_ray: bool = False,
     ) -> None:
         self.project_dir = Path(project_dir).resolve()
@@ -393,6 +410,7 @@ def __init__(
         self.shard_timeout = (
             shard_timeout if shard_timeout is not None else self._PYCG_SHARD_TIMEOUT
         )
+        self.max_iter = max_iter if max_iter is not None else self._PYCG_MAX_ITER
         # "jedi": partition the Jedi module graph (SCC + Louvain) so coupled
         # modules co-compute and few edges are severed (see shard_planner).
         # "package": legacy one-shard-per-package-directory grouping.
@@ -519,7 +537,7 @@ def _run_pycg_batch(
             cg = self._CallGraphGenerator(
                 entry_points=entry_points,
                 package=str(package_dir),
-                max_iter=-1,
+                max_iter=self.max_iter,
                 operation="call-graph",
             )
             cg.analyze()
@@ -656,7 +674,7 @@ def _build_sharded_planned_ray(self, plan: "ShardPlan") -> List[PyCallEdge]:
                         continue
                     root, eps = _materialize_shard_root(files, self.project_dir)
                     roots.append(root)
-                    fut = remote_fn.remote(eps, str(root), "")
+                    fut = remote_fn.remote(eps, str(root), "", self.max_iter)
                     futures.append(fut)
                     meta[fut] = (idx, n)
 
@@ -844,7 +862,7 @@ def _build_sharded_ray(self, shards: Dict[Path, List[str]]) -> List[PyCallEdge]:
                     progress.advance()
                     continue
                 prefix = self._package_prefix(pkg_root, self.project_dir)
-                fut = remote_fn.remote(files, str(pkg_root), prefix)
+                fut = remote_fn.remote(files, str(pkg_root), prefix, self.max_iter)
                 futures.append(fut)
                 meta[fut] = (pkg_label, n)
 
@@ -989,14 +1007,15 @@ def build_call_graph_edges(
                 )
                 return []
         else:
-            # Small project (≤ ceiling): whole-project analysis.
+            # Small project (≤ ceiling): whole-project analysis.  Run inside a
+            # symlink mini-project mirroring only the (already SKIP_DIRS-filtered)
+            # entry points, so PyCG's package bound covers project source alone.
+            # Pointing PyCG at project_dir directly would put an in-tree
+            # .codeanalyzer venv / site-packages *under* mod_dir, and PyCG would
+            # follow imports into those dependencies and explode the analysis.
             logger.info("PyCG: starting whole-project call graph analysis (%d files)", n_files)
-            try:
-                edges = self._run_pycg_batch(
-                    entry_points, self.project_dir, resolver, prefix=""
-                )
-            except PyCGExceptions.PyCGAnalysisError as exc:
-                raise
+            with _shard_symlink_root(entry_points, self.project_dir) as (root, eps):
+                edges = self._run_pycg_batch(eps, root, resolver, prefix="")
 
         elapsed = time.perf_counter() - t0
         logger.info("✅ PyCG: %d edges in %.1fs", len(edges), elapsed)

test/test_pycg_sharding.py

@@ -0,0 +1,61 @@
+"""Tests for PyCG executor scoping: dependency exclusion and the max_iter cap.
+
+These drive the real PyCG wrapper, so they require ``pycg`` (a level-2 install
+dependency).  They are deliberately tiny (a few files) so they run fast.
+"""
+from pathlib import Path
+
+import pytest
+
+pytest.importorskip("PyCG")
+
+from codeanalyzer.semantic_analysis.pycg.pycg_analysis import (
+    PyCG,
+    _PyCGCallableResolver,
+    _shard_symlink_root,
+)
+
+
+def test_max_iter_default_and_override(tmp_path):
+    p = PyCG(tmp_path)
+    assert p.max_iter == PyCG._PYCG_MAX_ITER == 50
+    assert PyCG(tmp_path, max_iter=7).max_iter == 7
+    assert PyCG(tmp_path, max_iter=-1).max_iter == -1
+
+
+def test_pycg_does_not_follow_into_in_tree_dependency(tmp_path):
+    """An in-tree ``.codeanalyzer`` venv under project_dir must stay a ghost.
+
+    PyCG bounds analysis to its ``package`` directory; running inside the
+    symlink mini-project keeps that bound on project source only, so imports
+    into a bundled dependency are recorded as ghost edges but never analysed.
+    Regression guard for the dep-reach blowup.
+    """
+    proj = tmp_path
+    app = proj / "app"
+    app.mkdir()
+    (app / "__init__.py").write_text("")
+    (app / "main.py").write_text("import bigdep\ndef run():\n    return bigdep.work()\n")
+
+    # A bundled dependency with many internal functions: if PyCG followed into
+    # it, dozens of bigdep.fN definitions/edges would appear.
+    dep = proj / ".codeanalyzer" / "venv" / "site-packages" / "bigdep"
+    dep.mkdir(parents=True)
+    body = "".join(f"def f{i}(x):\n    return f{(i + 1) % 50}(x)\n" for i in range(50))
+    body += "def work():\n    return f0(1)\n"
+    (dep / "__init__.py").write_text(body)
+
+    pycg = PyCG(proj)
+    pycg._ensure_pycg_loaded()
+    resolver = _PyCGCallableResolver(set())
+    entry_points = [str(app / "__init__.py"), str(app / "main.py")]
+    with _shard_symlink_root(entry_points, proj) as (root, eps):
+        edges = pycg._run_pycg_batch(eps, root, resolver, prefix="")
+
+    nodes = {n for e in edges for n in (e.source, e.target)}
+    # bigdep is reachable as a ghost target ...
+    assert any(n.startswith("bigdep") for n in nodes)
+    # ... but none of its internals were analysed.
+    assert not [n for n in nodes if n.startswith("bigdep.f")]
+    # and the real app edge is present.
+    assert any(e.source == "app.main.run" and e.target == "bigdep.work" for e in edges)