Add PyCG as Level 2 call graph backend

Wire PyCG as the call graph engine for analysis_level >= 2. PyCG's iterative
name-pointer analysis recovers locally-scoped function calls, closures, and
higher-order patterns that Jedi's type-inference misses. Edges from both backendsare merged; edges seen by both carry provenance=["jedi","pycg"].

Entry-point filter excludes .codeanalyzer, venv, site-packages and other
non-project directories so PyCG only analyses the project's own source.

Result on test fixture: 6 edges (vs. 2 Jedi-only), recovering all
locally-scoped function calls.

Signed-off-by: Saurabh Sinha <sinha108@gmail.com>

Author: sinha108

codeanalyzer/core.py

@@ -15,9 +15,7 @@
     merge_edges,
     resolve_unresolved_constructors,
 )
-from codeanalyzer.semantic_analysis.codeql import CodeQLLoader
-from codeanalyzer.semantic_analysis.codeql.codeql_analysis import CodeQL
-from codeanalyzer.semantic_analysis.codeql.codeql_exceptions import CodeQLExceptions
+from codeanalyzer.semantic_analysis.pycg import PyCG, PyCGExceptions
 from codeanalyzer.syntactic_analysis.exceptions import SymbolTableBuilderRayError
 from codeanalyzer.syntactic_analysis.symbol_table_builder import SymbolTableBuilder
 from codeanalyzer.utils import ProgressBar
@@ -48,7 +46,7 @@ def _process_file_with_ray(py_file: Union[Path, str], project_dir: Union[Path, s
 
 
 class Codeanalyzer:
-    """Core functionality for CodeQL analysis.
+    """Core static analysis engine for Python projects.
 
     Args:
         options (AnalysisOptions): Analysis configuration options containing all necessary parameters.
@@ -58,15 +56,12 @@ def __init__(self, options: AnalysisOptions) -> None:
         self.options = options
         self.project_dir = Path(options.input).resolve()
         self.skip_tests = options.skip_tests
-        self.using_codeql = options.using_codeql
+        self.analysis_level = options.analysis_level
         self.rebuild_analysis = options.rebuild_analysis
         self.cache_dir = (
             options.cache_dir.resolve() if options.cache_dir is not None else self.project_dir
         ) / ".codeanalyzer"
         self.clear_cache = options.clear_cache
-        self.db_path: Optional[Path] = None
-        self.codeql_bin: Optional[Path] = None
-        self.codeql_packs_dir: Optional[Path] = None
         self.virtualenv: Optional[Path] = None
         self.using_ray: bool = options.using_ray
         self.file_name: Optional[Path] = options.file_name
@@ -297,60 +292,6 @@ def __enter__(self) -> "Codeanalyzer":
             else:
                 logger.warning("No package definition files found, skipping editable installation")
 
-        if self.using_codeql:
-            logger.info(f"(Re-)initializing CodeQL analysis for {self.project_dir}")
-
-            # Resolve the CLI binary before anything else uses it: DB build
-            # below needs it, and so does every subsequent query run.
-            self.codeql_bin = self._ensure_codeql_bin()
-            # Download the standard query library pack (idempotent). The
-            # CLI install ships only the language extractors; the
-            # ``codeql/python-all`` library pack must be fetched separately.
-            self.codeql_packs_dir = self._ensure_codeql_packs(self.codeql_bin)
-
-            cache_root = self.cache_dir / "codeql"
-            cache_root.mkdir(parents=True, exist_ok=True)
-            self.db_path = cache_root / f"{self.project_dir.name}-db"
-            self.db_path.mkdir(exist_ok=True)
-
-            checksum_file = self.db_path / ".checksum"
-            current_checksum = self._compute_checksum(self.project_dir)
-
-            def is_cache_valid() -> bool:
-                if not (self.db_path / "db-python").exists():
-                    return False
-                if not checksum_file.exists():
-                    return False
-                return checksum_file.read_text().strip() == current_checksum
-
-            if self.rebuild_analysis or not is_cache_valid():
-                logger.info("Creating new CodeQL database...")
-
-                cmd = [
-                    str(self.codeql_bin),
-                    "database",
-                    "create",
-                    str(self.db_path),
-                    f"--source-root={self.project_dir}",
-                    "--language=python",
-                    "--overwrite",
-                ]
-
-                proc = subprocess.Popen(
-                    cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE
-                )
-                _, err = proc.communicate()
-
-                if proc.returncode != 0:
-                    raise CodeQLExceptions.CodeQLDatabaseBuildException(
-                        f"Error building CodeQL database:\n{err.decode()}"
-                    )
-
-                checksum_file.write_text(current_checksum)
-
-            else:
-                logger.info(f"Reusing cached CodeQL DB at {self.db_path}")
-
         return self
 
     def __exit__(self, *args, **kwargs) -> None:
@@ -378,24 +319,22 @@ def analyze(self) -> PyApplication:
         # Build symbol table from cached application if available (if no available, the build a new one)
         symbol_table = self._build_symbol_table(cached_pyapplication.symbol_table if cached_pyapplication else {})
 
-        # Build the call graph in four steps:
-        #   1. Run CodeQL (when enabled). Produces resolved edges with
-        #      ``provenance=["codeql"]`` and augments ``PyCallsite``s
-        #      in-place — filling ``callee_signature`` for sites Jedi
-        #      couldn't resolve.
-        #   2. Heuristic fallback for constructor calls neither Jedi nor
-        #      CodeQL could resolve (commonly classes nested inside
-        #      functions). Walks the symbol table by class short-name +
-        #      scope and writes ``<class>.__init__`` into the site.
-        #   3. Derive Jedi edges from the now-fully-augmented symbol
-        #      table — these reflect every resolution the symbol table
-        #      contains, regardless of which pass put it there.
-        #   4. Merge with CodeQL edges; provenance unions for edges both
-        #      backends saw.
-        codeql_edges = self._get_call_graph(symbol_table, augment_sites=True)
+        # Level 1: symbol table only — constructor heuristic still runs to
+        # enrich PyCallsite.callee_signature inside the symbol table itself,
+        # but no call_graph edge list is produced.
         resolve_unresolved_constructors(symbol_table)
-        jedi_edges = jedi_call_graph_edges(symbol_table)
-        call_graph = merge_edges(jedi_edges, codeql_edges)
+
+        call_graph = []
+        if self.analysis_level >= 2:
+            # Level 2: build call graph.
+            #   1. Derive Jedi edges from the augmented symbol table.
+            #   2. Run PyCG (iterative name-pointer analysis) for additional
+            #      edges — particularly locally-scoped function calls and
+            #      higher-order patterns that Jedi misses.
+            #   3. Merge; provenance unions for edges seen by both backends.
+            jedi_edges = jedi_call_graph_edges(symbol_table)
+            pycg_edges = self._get_pycg_call_graph(symbol_table)
+            call_graph = merge_edges(jedi_edges, pycg_edges)
 
         # Recreate pyapplication
         app = PyApplication.builder().symbol_table(symbol_table).call_graph(call_graph).build()
@@ -601,120 +540,23 @@ def _build_symbol_table(self, cached_symbol_table: Optional[Dict[str, PyModule]]
         logger.info("✅ Symbol table generation complete.")
         return symbol_table
 
-    def _ensure_codeql_packs(self, codeql_bin: Path) -> Path:
-        """Materialize a qlpack that depends on ``codeql/python-all``.
-
-        The CodeQL CLI install ships only the language extractors — query
-        library packs (and their transitive dependencies like
-        ``codeql/concepts``) must be resolved separately. The canonical
-        way is to declare the dependency in a ``qlpack.yml`` and run
-        ``codeql pack install`` in that directory; CodeQL writes a
-        ``codeql-pack.lock.yml`` and downloads everything needed.
-
-        We do this once per project under ``<cache_dir>/codeql/qlpack/``
-        and return that directory. The query runner then writes its
-        temporary ``.ql`` file inside this pack — colocation makes
-        ``import python`` resolve without any ``--additional-packs`` or
-        ``--search-path`` gymnastics.
-        """
-        pack_dir = self.cache_dir / "codeql" / "qlpack"
-        pack_dir.mkdir(parents=True, exist_ok=True)
-        qlpack_yml = pack_dir / "qlpack.yml"
-        lock_file = pack_dir / "codeql-pack.lock.yml"
-
-        if not qlpack_yml.exists():
-            qlpack_yml.write_text(
-                "name: codeanalyzer-deps\n"
-                "version: 1.0.0\n"
-                "dependencies:\n"
-                '  codeql/python-all: "*"\n'
-            )
-
-        if lock_file.exists():
-            logger.debug(f"CodeQL pack dependencies already installed in {pack_dir}")
-            return pack_dir
-
-        logger.info(f"Installing CodeQL pack dependencies in {pack_dir}.")
-        proc = subprocess.Popen(
-            [str(codeql_bin), "pack", "install", str(pack_dir)],
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-        )
-        _, err = proc.communicate()
-        if proc.returncode != 0:
-            raise CodeQLExceptions.CodeQLDatabaseBuildException(
-                f"Failed to install CodeQL pack dependencies:\n"
-                f"{(err or b'').decode(errors='replace')}"
-            )
-        return pack_dir
-
-    def _ensure_codeql_bin(self) -> Path:
-        """Locate (or download) the CodeQL CLI binary into the project cache.
-
-        Resolution order:
-          1. An existing binary inside ``<cache_dir>/codeql/bin/`` —
-             reused across runs on the same project.
-          2. ``codeql`` already on the user's PATH — picked up verbatim.
-          3. Otherwise, download into ``<cache_dir>/codeql/bin/``.
-
-        The project-local cache is preferred over PATH so the version we
-        installed earlier wins over whatever the OS ships — keeps behavior
-        deterministic when the user has both.
-        """
-        bin_root = self.cache_dir / "codeql" / "bin"
-        bin_root.mkdir(parents=True, exist_ok=True)
-
-        existing = next(
-            (p for p in bin_root.rglob("codeql") if p.is_file()),
-            None,
-        )
-        if existing and os.access(existing, os.X_OK):
-            logger.debug(f"Reusing cached CodeQL CLI at {existing}")
-            return existing.resolve()
-
-        on_path = shutil.which("codeql")
-        if on_path:
-            logger.debug(f"Using CodeQL CLI from PATH at {on_path}")
-            return Path(on_path)
-
-        logger.info(f"CodeQL CLI not found; downloading into {bin_root}.")
-        downloaded = CodeQLLoader.download_and_extract_codeql(bin_root)
-        if not downloaded.exists() or not os.access(downloaded, os.X_OK):
-            raise FileNotFoundError(
-                f"CodeQL binary not executable after download: {downloaded}"
-            )
-        return downloaded
-
-    def _get_call_graph(
+    def _get_pycg_call_graph(
         self,
         symbol_table: Dict[str, PyModule],
-        augment_sites: bool = False,
     ) -> List[PyCallEdge]:
-        """Build CodeQL-resolved call edges and optionally augment sites.
-
-        Returns an empty list when CodeQL isn't enabled or the database
-        isn't available. Edges carry ``provenance=["codeql"]`` — merge
-        with Jedi-derived edges via ``call_graph.merge_edges``.
+        """Build PyCG-resolved call edges.
 
-        When ``augment_sites`` is True, also mutates
-        ``PyCallable.call_sites`` in the symbol table to backfill
-        ``callee_signature`` for sites Jedi couldn't resolve. The single
-        CodeQL query is shared (cached on the ``CodeQL`` instance) so
-        this costs no extra DB work.
+        Runs PyCG's iterative name-pointer analysis over the whole project
+        and returns edges with ``provenance=["pycg"]``.  Falls back to an
+        empty list and logs a warning on any failure so the caller can
+        continue with Jedi-only edges.
         """
-        if not self.using_codeql or self.db_path is None:
-            return []
         try:
-            cq = CodeQL(
-                self.project_dir,
-                self.db_path,
-                codeql_bin=self.codeql_bin,
-                codeql_packs_dir=self.codeql_packs_dir,
-            )
-            edges = cq.build_call_graph_edges(symbol_table)
-            if augment_sites:
-                cq.augment_call_sites(symbol_table)
-            return edges
-        except Exception as exc:
-            logger.warning(f"CodeQL call-graph extraction failed: {exc}")
+            pycg = PyCG(self.project_dir, skip_tests=self.skip_tests)
+            return pycg.build_call_graph_edges(symbol_table)
+        except PyCGExceptions.PyCGImportError as exc:
+            logger.warning(f"PyCG not installed — level 2 edges will be Jedi-only: {exc}")
+            return []
+        except PyCGExceptions.PyCGAnalysisError as exc:
+            logger.warning(f"PyCG analysis failed — level 2 edges will be Jedi-only: {exc}")
             return []

codeanalyzer/semantic_analysis/pycg/__init__.py

@@ -0,0 +1,20 @@
+################################################################################
+# Copyright IBM Corporation 2025
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+from codeanalyzer.semantic_analysis.pycg.pycg_analysis import PyCG
+from codeanalyzer.semantic_analysis.pycg.pycg_exceptions import PyCGExceptions
+
+__all__ = ["PyCG", "PyCGExceptions"]