diff --git a/graphify/extract.py b/graphify/extract.py index f68757a85..8d0c19630 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -2487,6 +2487,56 @@ def _augment_js_reexport_edges( # Header / implementation file-extension pairing for the decl/def class merge. + +def _bind_cross_file_value_refs( + per_file: list[dict], + all_nodes: list[dict], + all_edges: list[dict], +) -> None: + """Bind value references that cross a file boundary. + + An extractor sees one file. It can bind a constant only where that + constant is also declared, and reports the rest as ``raw_value_refs`` + rather than guessing - the same split ``raw_calls`` already uses for + calls. Here every file is available. + + Measured on a 59-file Go project: of fifteen constants each used in + exactly one function, three were used from a sibling file, and those + three stayed unreachable until this pass existed. + + Only for an unambiguous name, the same god-node guard the call + resolver applies: two same-named constants in two packages would + otherwise wire a caller to the wrong one. + """ + by_name: dict[str, list[str]] = {} + for n in all_nodes: + if n.get("value_kind"): + by_name.setdefault(str(n.get("label", "")), []).append(n["id"]) + if not by_name: + return + + have = {(e.get("source"), e.get("target")) for e in all_edges} + for result in per_file: + for rv in result.get("raw_value_refs") or []: + ids = by_name.get(rv.get("name", ""), []) + if len(ids) != 1: + continue + src, tgt = rv.get("caller_nid"), ids[0] + if not src or src == tgt or (src, tgt) in have: + continue + have.add((src, tgt)) + all_edges.append({ + "source": src, + "target": tgt, + "relation": "references", + "context": "value_use", + "confidence": "EXTRACTED", + "source_file": rv.get("source_file", ""), + "source_location": rv.get("source_location", ""), + "weight": 1.0, + }) + + def _merge_swift_extensions( per_file: list[dict], all_nodes: list[dict], @@ -6198,6 +6248,13 @@ def _portable_out_of_root_sf(p: Path) -> str: cn = rc.get("caller_nid") if cn in id_remap: rc["caller_nid"] = id_remap[cn] + # raw_value_refs carry the same kind of id and are consumed by + # _bind_cross_file_value_refs, so they need the same rewrite. + for result in per_file: + for rv in result.get("raw_value_refs") or []: + vn = rv.get("caller_nid") + if vn in id_remap: + rv["caller_nid"] = id_remap[vn] # swift_extensions[].nid is the same kind of id carrier as caller_nid # above (cache.py remaps both), consumed by _merge_swift_extensions far # below. Left stale it matches no node, so whether the extension merge @@ -6270,6 +6327,11 @@ def _portable_out_of_root_sf(p: Path) -> str: cn = rc.get("caller_nid") if cn in sym_remap: rc["caller_nid"] = sym_remap[cn] + for result in per_file: + for rv in result.get("raw_value_refs") or []: + vn = rv.get("caller_nid") + if vn in sym_remap: + rv["caller_nid"] = sym_remap[vn] # Same for swift_extensions[].nid (see the id_remap pass above). for result in per_file: for ext in result.get("swift_extensions", []) or []: @@ -6400,6 +6462,7 @@ def _learn(e: dict) -> None: # (src/) package root before the resolver/import-evidence passes run, so the # graph is identical regardless of scan root (#2072). _repoint_python_package_imports(paths, all_nodes, all_edges, root) + _bind_cross_file_value_refs(per_file, all_nodes, all_edges) _merge_swift_extensions(per_file, all_nodes, all_edges) _merge_csharp_partial_class_nodes(per_file, all_nodes, all_edges, paths, root) _disambiguate_colliding_node_ids(all_nodes, all_edges, all_raw_calls, root) diff --git a/graphify/extractors/go.py b/graphify/extractors/go.py index d8f203b77..ccd4bf1a2 100644 --- a/graphify/extractors/go.py +++ b/graphify/extractors/go.py @@ -83,7 +83,7 @@ def _go_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[st _go_collect_type_refs(c, source, generic, out) def extract_go(path: Path) -> dict: - """Extract functions, methods, type declarations, and imports from a .go file.""" + """Extract functions, methods, types, values, and imports from a .go file.""" try: import tree_sitter_go as tsgo from tree_sitter import Language, Parser @@ -269,9 +269,51 @@ def symbol_nid(plain_nid: str, name: str) -> str: salt = hashlib.sha1(name.encode("utf-8"), usedforsecurity=False).hexdigest()[:6] return _make_id(plain_nid, salt) + # Package-level value names declared in this file. Kept separate from + # label_to_nid so the reference pass below binds only these and not + # every identifier it walks past. + value_nids: dict[str, str] = {} + + def add_value_decl(node, kind: str) -> None: + """Emit a node per name in a const_declaration or var_declaration. + + The dispatch below handled function_declaration, + method_declaration, type_declaration and import_declaration. Go's + grammar also has const_declaration and var_declaration, so a + package-level constant never became a node and nothing could + point at it. + + Measured on a 59-file Go project: of fifteen constants each used + in exactly one function, none was a node, while all fifteen of + those functions were. Asked "which function uses X", the graph + had the answer but no way in. + """ + for spec in node.children: + if spec.type not in ("const_spec", "var_spec"): + continue + line = spec.start_point[0] + 1 + for child in spec.children: + if child.type != "identifier": + continue + name = _read_text(child, source) + if not name or name == "_": + continue + nid = _make_id(pkg_scope, name) + add_node(nid, name, line) + for n in nodes: + if n["id"] == nid: + n["value_kind"] = kind + break + add_edge(file_nid, nid, "contains", line, context=kind) + value_nids[name] = nid + def walk(node) -> None: t = node.type + if t in ("const_declaration", "var_declaration"): + add_value_decl(node, "const" if t == "const_declaration" else "var") + return + if t == "function_declaration": name_node = node.child_by_field_name("name") if name_node: @@ -430,6 +472,8 @@ def walk(node) -> None: label_to_nid[normalised] = n["id"] seen_call_pairs: set[tuple[str, str]] = set() + seen_value_refs: set[tuple[str, str]] = set() + raw_value_refs: list[dict] = [] raw_calls: list[dict] = [] def walk_calls(node, caller_nid: str) -> None: @@ -493,6 +537,40 @@ def walk_calls(node, caller_nid: str) -> None: "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", }) + # Reading a value is a relation this extractor did not record. + # "Where is this constant used" asks for exactly that, and + # without the edge the node stays unreachable even once it + # exists. Restricted to names declared as package-level values, + # so a local identifier does not produce an edge. + if node.type == "identifier": + name = _read_text(node, source) + tgt = value_nids.get(name) + if tgt is None and name and name not in _LANGUAGE_BUILTIN_GLOBALS: + # Not declared in this file. It may be a constant from a + # sibling file or a local variable; only a pass that sees + # every file can tell, which is how cross-file calls are + # already resolved. See _bind_cross_file_value_refs. + raw_value_refs.append({ + "caller_nid": caller_nid, + "name": name, + "source_file": str_path, + "source_location": f"L{node.start_point[0] + 1}", + }) + elif tgt and tgt != caller_nid: + pair = (caller_nid, tgt) + if pair not in seen_value_refs: + seen_value_refs.add(pair) + edges.append({ + "source": caller_nid, + "target": tgt, + "relation": "references", + "context": "value_use", + "confidence": "EXTRACTED", + "source_file": str_path, + "source_location": f"L{node.start_point[0] + 1}", + "weight": 1.0, + }) + for child in node.children: walk_calls(child, caller_nid) @@ -510,5 +588,6 @@ def walk_calls(node, caller_nid: str) -> None: "nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls, + "raw_value_refs": raw_value_refs, "go_imports": dict(go_imported_pkgs), } diff --git a/tests/test_go_value_nodes.py b/tests/test_go_value_nodes.py new file mode 100644 index 000000000..decf9c8f6 --- /dev/null +++ b/tests/test_go_value_nodes.py @@ -0,0 +1,121 @@ +"""Go package-level constants and variables must be nodes with inbound edges. + +The Go extractor dispatched on four node types: function_declaration, +method_declaration, type_declaration and import_declaration. Go's grammar +also has const_declaration and var_declaration, so a package-level +constant never became a node, and reading one was not recorded as a +relation. + +Observed on a 59-file Go codebase, using fifteen constants each read in +exactly one function: none of the fifteen constants was a node, while all +fifteen of the reading functions were. Asked "which function uses X", the +graph held the answer and offered no way in - every query returned "No +matching nodes found". + +Three of those fifteen were read from a sibling file, which a per-file +extractor cannot bind. Those go through raw_value_refs and +_bind_cross_file_value_refs, the same split raw_calls already uses. +""" +import pytest + +from graphify.extract import extract + + +def _extract_go(tmp_path): + return extract(sorted(tmp_path.glob("*.go")), cache_root=tmp_path, parallel=False) + + +def _node_by_label(result, label): + for n in result["nodes"]: + if (n.get("label") or "").strip(".()") == label: + return n + return None + + +def _has_edge(result, src_label, tgt_label, relation): + src = _node_by_label(result, src_label) + tgt = _node_by_label(result, tgt_label) + if src is None or tgt is None: + return False + return any( + e.get("source") == src["id"] + and e.get("target") == tgt["id"] + and e.get("relation") == relation + for e in result["edges"] + ) + + +@pytest.fixture +def same_file(tmp_path): + (tmp_path / "sandbox.go").write_text( + "package runtime\n" + "\n" + "const (\n" + "\tsandboxProviderFD = 3\n" + "\tmaxOutput = 4096\n" + ")\n" + "\n" + "var defaultTimeout = 30\n" + "\n" + "func bubblewrapArguments() []string {\n" + "\t_ = sandboxProviderFD\n" + "\treturn nil\n" + "}\n" + ) + return _extract_go(tmp_path) + + +def test_constants_become_nodes(same_file): + for name in ("sandboxProviderFD", "maxOutput"): + node = _node_by_label(same_file, name) + assert node is not None, f"{name} is not a node" + assert node.get("value_kind") == "const" + + +def test_package_variables_become_nodes(same_file): + node = _node_by_label(same_file, "defaultTimeout") + assert node is not None + assert node.get("value_kind") == "var" + + +def test_reading_a_constant_is_an_edge(same_file): + assert _has_edge(same_file, "bubblewrapArguments", "sandboxProviderFD", "references") + + +def test_unused_constant_has_no_reader(same_file): + # maxOutput is declared and never read. A node, but nothing points at + # it: the pass must not invent an edge for every name it walks past. + assert not _has_edge(same_file, "bubblewrapArguments", "maxOutput", "references") + + +def test_constant_read_from_a_sibling_file(tmp_path): + (tmp_path / "sandbox.go").write_text( + "package runtime\n" + "\n" + "const sandboxProviderFD = 3\n" + ) + (tmp_path / "plan.go").write_text( + "package runtime\n" + "\n" + "func bubblewrapArguments() int {\n" + "\treturn sandboxProviderFD\n" + "}\n" + ) + result = _extract_go(tmp_path) + assert _has_edge(result, "bubblewrapArguments", "sandboxProviderFD", "references") + + +def test_local_variable_does_not_bind_to_a_constant(tmp_path): + # A local shadowing a package constant of another file must not wire + # the reader to it. The cross-file pass binds by name, so the guard + # is that only declared package-level values are candidates. + (tmp_path / "a.go").write_text( + "package runtime\n" + "\n" + "func caller() int {\n" + "\tnotAConstant := 7\n" + "\treturn notAConstant\n" + "}\n" + ) + result = _extract_go(tmp_path) + assert _node_by_label(result, "notAConstant") is None