OSAC-4050: Universal YAML/JSON structural coverage, k8s manifests as the rich layer

graphify/detect.py

@@ -41,8 +41,15 @@ class FileType(str, Enum):
 _MTIME_COARSE_S = 2.0
 _MTIME_SUBSECOND_S = 0.05
 
-CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger'}
-DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.skill', '.txt', '.rst', '.html', '.yaml', '.yml'}
+# .yaml/.yml unconditionally CODE (OSAC-4050) -- matches .json's existing
+# precedent exactly (extract_yaml/extract_json each layer a rich,
+# recognized-schema extractor over a generic structural fallback, so
+# nothing YAML/JSON-shaped is ever invisible to the graph; the user
+# confirmed this universal-coverage direction explicitly, overriding the
+# ticket's original k8s-only scope, having been told plainly that
+# unrecognized content gets low-value/noisy nodes this way).
+CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.yaml', '.yml', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger'}
+DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.skill', '.txt', '.rst', '.html'}
 PAPER_EXTENSIONS = {'.pdf'}
 IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'}
 OFFICE_EXTENSIONS = {'.docx', '.xlsx'}

graphify/extract.py

@@ -54,6 +54,9 @@
 from graphify.extractors.sql import extract_sql  # noqa: F401
 from graphify.extractors.terraform import extract_terraform  # noqa: F401
 from graphify.extractors.verilog import extract_verilog  # noqa: F401
+from graphify.extractors.k8s_manifest import extract_k8s_manifest  # noqa: F401
+from graphify.extractors.yaml_generic import extract_generic_structure as extract_yaml_generic  # noqa: F401
+from graphify.extractors.yaml_dispatch import extract_yaml  # noqa: F401
 from graphify.extractors.zig import extract_zig  # noqa: F401
 from graphify.security import sanitize_metadata
 from graphify.paths import disambiguate_ambiguous_candidates
@@ -4843,6 +4846,14 @@ def add_existing_edge(edge: dict) -> None:
     ".sh": extract_bash,
     ".bash": extract_bash,
     ".json": extract_json,
+    # NOTE: OSAC-4049 (unmerged as of this writing) also dispatches
+    # .yaml/.yml, to extract_github_actions -- when both PRs merge these two
+    # entries will conflict and need combining into one dispatcher that tries
+    # each shape in turn (GH Actions, then k8s manifest, then the generic
+    # structural fallback below), not simply picking one. Flagged in this
+    # PR's description.
+    ".yaml": extract_yaml,
+    ".yml": extract_yaml,
     ".tf": extract_terraform,
     ".tfvars": extract_terraform,
     ".hcl": extract_terraform,
@@ -4870,6 +4881,8 @@ def add_existing_edge(edge: dict) -> None:
 # extract() to tell the user which extra restores the language.
 _EXTRA_FOR_EXTENSION = {
     ".sql": "sql",
+    ".yaml": "yaml",
+    ".yml": "yaml",
     ".tf": "terraform",
     ".tfvars": "terraform",
     ".hcl": "terraform",

graphify/extractors/_yaml_cst.py

@@ -0,0 +1,183 @@
+"""Shared tree-sitter-yaml CST traversal helpers.
+
+Used by both ``graphify/extractors/k8s_manifest.py`` and
+``graphify/extractors/yaml_generic.py`` (OSAC-4050). Originally written
+once inline in ``k8s_manifest.py``, adapted with attribution from the
+tree-sitter-yaml traversal in the unmerged Graphify-Labs/graphify PR #2541;
+factored out here once a second real consumer (the generic structural
+walker) needed the exact same helpers within the same branch. Note:
+``graphify/extractors/github_actions.py`` (OSAC-4049, unmerged as of this
+writing) still carries its own independent copy of an earlier version of
+these same helpers, since its branch predates this module and should not be
+made to depend on this one landing first -- worth deduping further once
+both have merged.
+"""
+from __future__ import annotations
+
+_MAPPING_TYPES = frozenset({"block_mapping", "flow_mapping"})
+_SEQUENCE_TYPES = frozenset({"block_sequence", "flow_sequence"})
+
+
+def descend(node, wanted: frozenset[str]):
+    """Return the first descendant of *node* whose type is in *wanted*.
+
+    YAML wraps every value in `block_node`/`flow_node` before the actual
+    collection, and a document adds another layer, so callers would otherwise
+    repeat the same two-or-three-step unwrap everywhere.
+    """
+    if node is None:
+        return None
+    if node.type in wanted:
+        return node
+    for child in node.children:
+        if not child.is_named:
+            continue
+        if child.type in ("block_node", "flow_node", "document"):
+            found = descend(child, wanted)
+            if found is not None:
+                return found
+        elif child.type in wanted:
+            return child
+    return None
+
+
+def mapping(node):
+    return descend(node, _MAPPING_TYPES)
+
+
+def scalar_text(node) -> str:
+    """Text of the scalar at *node*, with one layer of quotes stripped."""
+    if node is None:
+        return ""
+    text = node.text.decode("utf-8", errors="replace").strip()
+    if len(text) >= 2 and text[0] == text[-1] and text[0] in ("'", '"'):
+        text = text[1:-1]
+    return text.strip()
+
+
+def pairs(node):
+    """Yield `(key, value_node, line)` for each pair of the mapping at *node*.
+
+    *node* may be the mapping itself or any wrapper around it. Pairs whose key
+    is not a plain scalar are skipped rather than stringified, so they never
+    mint a garbage node.
+    """
+    m = mapping(node)
+    if m is None:
+        return
+    for pair in m.children:
+        if pair.type not in ("block_mapping_pair", "flow_pair"):
+            continue
+        key_node = pair.child_by_field_name("key")
+        if key_node is None:
+            continue
+        key = scalar_text(key_node)
+        if not key:
+            continue
+        yield key, pair.child_by_field_name("value"), key_node.start_point[0] + 1
+
+
+def item_value(item):
+    """The value inside a `block_sequence_item`, without the `- ` marker."""
+    if item.type != "block_sequence_item":
+        return item
+    for child in item.children:
+        if child.is_named:
+            return child
+    return item
+
+
+def string_items(node) -> list[tuple[str, int]]:
+    """Scalars reachable from *node* as `(text, line)` -- a bare scalar, a
+    sequence (block or flow), or (defensively) a mapping's keys."""
+    if node is None:
+        return []
+    seq = descend(node, _SEQUENCE_TYPES)
+    if seq is not None:
+        items = []
+        for item in seq.children:
+            if item.type not in ("block_sequence_item", "flow_node"):
+                continue
+            text = scalar_text(item_value(item))
+            if text and "\n" not in text and ":" not in text:
+                items.append((text, item.start_point[0] + 1))
+        return items
+    m = mapping(node)
+    if m is not None:
+        return [(key, line) for key, _value, line in pairs(m)]
+    text = scalar_text(node)
+    return [(text, node.start_point[0] + 1)] if text else []
+
+
+def sequence_items(node):
+    """Yield the item nodes of the sequence at *node*."""
+    seq = descend(node, _SEQUENCE_TYPES)
+    if seq is None:
+        return
+    for item in seq.children:
+        if item.type in ("block_sequence_item", "flow_node"):
+            yield item
+
+
+def all_documents(root):
+    """Yield the raw top-level node of every document in the file (whatever
+    its type -- mapping, sequence, scalar, or ERROR; callers decide what to
+    do with each).
+
+    Root type is `stream` with one `document` child per resource for a
+    multi-document file (confirmed against a real file in this repo,
+    ``osac-operator/config/manager/manager.yaml``, which holds a Namespace
+    and a Deployment separated by `---`), vs a bare `document` (or, for a
+    malformed/templated file, an `ERROR` node) for a single-document file.
+    """
+    if root.type == "stream":
+        docs = [c for c in root.children if c.type == "document"]
+    elif root.type == "document":
+        docs = [root]
+    else:
+        docs = [root]
+    for doc in docs:
+        yield doc
+
+
+# Go/Helm template marker. `has_error` alone is not a reliable signal for
+# real Go-template contamination -- confirmed empirically (OSAC-4050 review):
+# a bare inline template value like `replicas: {{ .Values.x }}` parses with
+# `has_error=False` on the specific document node (the template markers look
+# like valid, if bogus, nested flow-mapping syntax to the YAML grammar, not
+# a parse error), while a concatenated pattern like
+# `image: {{ .Values.x }}:{{ .Values.y }}` sets `has_error=True` on the
+# STREAM root but not on the specific document node -- has_error alone
+# misses real cases in both directions. Any document whose raw text
+# contains this marker is treated as unparseable-for-our-purposes
+# regardless of what has_error reports.
+_TEMPLATE_MARKER = b"{{"
+
+
+def is_unparseable(doc) -> bool:
+    """True if *doc* should not be trusted as real YAML content -- either
+    tree-sitter itself flagged a parse error, or its raw text contains a
+    Go/Helm template marker that has_error alone does not reliably catch
+    (see comment above). Shared by graphify.extractors.k8s_manifest (both
+    the extract_k8s_resources/yaml_dispatch path and the standalone
+    extract_k8s_manifest entry point -- the latter previously lacked this
+    check entirely, a real bypass of the safety net for anyone calling it
+    directly) and graphify.extractors.yaml_dispatch.
+    """
+    return doc.has_error or _TEMPLATE_MARKER in doc.text
+
+
+def all_top_level_mappings(root):
+    """Yield the top-level MAPPING of every document in the file that
+    parses cleanly (see ``is_unparseable``) and is mapping-shaped --
+    documents that aren't mapping-shaped, or are malformed/templated, are
+    silently skipped. For callers that only care about mapping-shaped
+    resources, e.g. a k8s manifest. See ``all_documents`` for the version
+    that yields every document regardless of shape or parseability.
+    """
+    for doc in all_documents(root):
+        if is_unparseable(doc):
+            continue
+        m = mapping(doc)
+        if m is not None:
+            yield m

graphify/extractors/json_config.py

@@ -202,15 +202,20 @@ def walk_object(obj_node, parent_nid: str, parent_key: str | None,
     doc = root
     if doc.type == "document" and doc.child_count > 0:
         doc = doc.children[0]
-    if doc.type == "object":
-        # Only AST-extract recognized config/manifest JSON. Data JSON (fixtures,
-        # datasets, GeoJSON, API dumps) is skipped so it doesn't explode into
-        # orphan key-nodes (#1224); it's left to the LLM semantic pass.
-        if not _is_config_json(path, doc, source):
-            return {"nodes": [], "edges": [], "skipped": "data json (not a config/manifest)"}
+    if doc.type == "object" and _is_config_json(path, doc, source):
         walk_object(doc, file_nid, None, 0, [0])
-    else:
-        # Top-level array or scalar => data JSON, never a config/manifest.
-        return {"nodes": [], "edges": [], "skipped": "data json (non-object root)"}
-
-    return {"nodes": nodes, "edges": edges}
+        return {"nodes": nodes, "edges": edges}
+
+    # Data JSON (fixtures, datasets, GeoJSON, API dumps, or any top-level
+    # array/scalar) doesn't get the rich config-specific dependency/extends/
+    # $ref treatment above (#1224 -- that produced hundreds of orphan
+    # key-nodes when applied indiscriminately). Per OSAC-4050's explicit,
+    # confirmed direction change, it no longer disappears from the graph
+    # either: fall back to a genuinely generic structural walk (no domain
+    # semantics), same design as graphify/extractors/yaml_generic.py.
+    from graphify.extractors.json_generic import extract_generic_structure as _generic_json
+    generic_nodes, generic_edges, truncated = _generic_json(doc, source, str_path, file_nid)
+    result: dict = {"nodes": nodes + generic_nodes, "edges": edges + generic_edges}
+    if truncated:
+        result["truncated"] = f"generic structural walk capped at node limit for {path.name}"
+    return result