diff --git a/c_parser/__init__.py b/c_parser/__init__.py new file mode 100644 index 000000000..191d6c142 --- /dev/null +++ b/c_parser/__init__.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +"""Public C parser skeleton package.""" + +from .models import ( + CArray, + CDiagnostic, + CEnum, + CEnumerator, + CField, + CFile, + CFunction, + CGlobal, + CInclude, + CMacro, + CParameter, + CParseError, + CPointer, + CProject, + CSourceLocation, + CStruct, + CTypeRef, + CTypedef, + CUnion, +) +from .parser import CParser, parse_c_file, parse_c_project + +__all__ = ( + "CArray", + "CDiagnostic", + "CEnum", + "CEnumerator", + "CField", + "CFile", + "CFunction", + "CGlobal", + "CInclude", + "CMacro", + "CParameter", + "CParseError", + "CPointer", + "CParser", + "CProject", + "CSourceLocation", + "CStruct", + "CTypeRef", + "CTypedef", + "CUnion", + "parse_c_file", + "parse_c_project", +) diff --git a/c_parser/__main__.py b/c_parser/__main__.py new file mode 100644 index 000000000..1b9f5c141 --- /dev/null +++ b/c_parser/__main__.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- +"""Run the C parser skeleton CLI.""" + +from .cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/c_parser/cli.py b/c_parser/cli.py new file mode 100644 index 000000000..3d0bc1b8f --- /dev/null +++ b/c_parser/cli.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from .models import CFile, c_model_to_dict +from .parser import CParser + + +_C_SOURCE_SUFFIXES = {".c", ".h"} + + +def _collect_c_extensions(path: Path) -> list[Path]: + return sorted( + p + for p in path.rglob("*") + if p.is_file() and p.suffix.lower() in _C_SOURCE_SUFFIXES + ) + + +def expand_c_paths(paths: list[str]) -> list[Path]: + expanded: list[Path] = [] + for raw in paths: + p = Path(raw) + if p.is_dir(): + expanded.extend(_collect_c_extensions(p)) + else: + expanded.append(p) + return sorted(set(expanded)) + + +def parse_c_report(paths: list[str]) -> dict[str, dict]: + out: dict[str, dict] = {} + parser = CParser() + for p in expand_c_paths(paths): + parsed = parser.visit_file(p, filename=str(p)) + out[str(p)] = c_model_to_dict(parsed) + return out + + +def format_c_report(report: dict[str, dict]) -> str: + lines: list[str] = [] + for fname, parsed in report.items(): + lines.append(f"File: {fname}") + lines.append(f" Language: {parsed.get('language', 'c')}") + lines.append(f" Functions: {len(parsed.get('functions') or [])}") + lines.append(f" Structs: {len(parsed.get('structs') or [])}") + lines.append(f" Unions: {len(parsed.get('unions') or [])}") + lines.append(f" Enums: {len(parsed.get('enums') or [])}") + lines.append(f" Typedefs: {len(parsed.get('typedefs') or [])}") + lines.append(f" Globals: {len(parsed.get('globals') or [])}") + lines.append(f" Macros: {len(parsed.get('macros') or [])}") + lines.append(f" Includes: {len(parsed.get('includes') or [])}") + lines.append(f" Diagnostics: {len(parsed.get('diagnostics') or [])}") + lines.append(f" Parser status: {parsed.get('parser_status', 'skeleton')}") + lines.append("") + return "\n".join(lines).rstrip() + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="C parser skeleton CLI.") + parser.add_argument("paths", nargs="+", help="C source/header file(s) or directory path(s)") + parser.add_argument("--json", action="store_true", help="Print JSON to stdout") + parser.add_argument("--out", type=str, help="Write parser JSON to a file") + args = parser.parse_args(argv) + + payload = parse_c_report(args.paths) + if args.out: + Path(args.out).write_text(json.dumps(payload, indent=2), encoding="utf-8") + return 0 + + if args.json: + print(json.dumps(payload, indent=2)) + else: + print(format_c_report(payload)) + return 0 + + +__all__ = ( + "CFile", + "expand_c_paths", + "format_c_report", + "main", + "parse_c_report", +) diff --git a/c_parser/lexer.py b/c_parser/lexer.py new file mode 100644 index 000000000..8bd64de11 --- /dev/null +++ b/c_parser/lexer.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- +"""C lexer placeholder. + +The real lexer lands after the C parser public API and CLI skeleton are stable. +""" + +__all__: tuple[str, ...] = () diff --git a/c_parser/models.py b/c_parser/models.py new file mode 100644 index 000000000..24f02bc66 --- /dev/null +++ b/c_parser/models.py @@ -0,0 +1,331 @@ +# -*- coding: utf-8 -*- +from __future__ import annotations + +import inspect +import os +import sys +from dataclasses import dataclass, field, fields, is_dataclass +from typing import Any + + +_ANSI = { + "bold": "\033[1m", + "red": "\033[31m", + "blue": "\033[34m", + "cyan": "\033[36m", + "reset": "\033[0m", +} +_TRUE_VALUES = {"1", "true", "yes", "on"} + + +def _env_flag(name: str) -> bool: + return os.getenv(name, "").strip().lower() in _TRUE_VALUES + + +def _apply_color(text: str, *styles: str, enabled: bool) -> str: + if not enabled: + return text + prefix = "".join(_ANSI[style] for style in styles) + return f"{prefix}{text}{_ANSI['reset']}" + + +def _enable_windows_ansi() -> None: # pragma: no cover - Windows-only console setup. + if os.name != "nt": + return + + if sys.modules.get("colorama") is not None: + sys.modules["colorama"].just_fix_windows_console() + return + + import importlib.util + + if importlib.util.find_spec("colorama") is not None: + import importlib + + colorama = importlib.import_module("colorama") + colorama.just_fix_windows_console() + + +def c_model_to_dict(obj: Any) -> Any: + """Convert C parser dataclasses into stable JSON-compatible values.""" + if is_dataclass(obj): + return {f.name: c_model_to_dict(getattr(obj, f.name)) for f in fields(obj)} + if isinstance(obj, list): + return [c_model_to_dict(v) for v in obj] + if isinstance(obj, dict): + return {k: c_model_to_dict(v) for k, v in obj.items()} + if isinstance(obj, set): + return sorted(c_model_to_dict(v) for v in obj) + return obj + + +class CParseError(ValueError): + """C parser error with compiler-style diagnostic rendering support.""" + + default_code = "CPARSE001" + + def __init__( + self, + message: str, + filename: str | None = None, + line_number: int | None = None, + column: int | None = None, + source_line: str | None = None, + *, + code: str | None = None, + ): + self.filename = filename + self.line_number = line_number + self.column = column + self.source_line = source_line + self.base_message = message + self.code = code or self.default_code + frame = inspect.stack()[1] + self.parser_file = frame.filename + self.parser_line_number = frame.lineno + self.parser_function = frame.function + super().__init__(self.format_diagnostic(color=False)) + + def format_diagnostic(self, *, color: bool = False, debug: bool | None = None) -> str: + if color: + _enable_windows_ansi() + if debug is None: + debug = _env_flag("C_PARSER_DEBUG") + + location = self.filename or "" + if self.line_number is not None: + column = self.column if self.column is not None else 1 + location = f"{location}:{self.line_number}:{column}" + + severity = _apply_color("error", "red", "bold", enabled=color) + code = _apply_color(f"[{self.code}]", "cyan", enabled=color) + lines = [f"{_apply_color(location, 'bold', enabled=color)}: {severity}{code}: {self.base_message}"] + + if self.source_line is not None: + line_no = str(self.line_number) if self.line_number is not None else "?" + gutter_width = max(len(line_no), 1) + source = self.source_line.rstrip("\n") + marker_column = max((self.column or 1) - 1, 0) + marker = " " * marker_column + "^" if source.strip() else "" + lines.extend( + [ + f"{' ' * gutter_width} {_apply_color('|', 'blue', enabled=color)}", + f"{_apply_color(line_no.rjust(gutter_width), 'blue', enabled=color)} {_apply_color('|', 'blue', enabled=color)} {source}", + f"{' ' * gutter_width} {_apply_color('|', 'blue', enabled=color)} {_apply_color(marker, 'red', 'bold', enabled=color)}", + ] + ) + + if debug: + lines.append( + _apply_color( + f"note: parser raised at {self.parser_file}:{self.parser_line_number} in {self.parser_function}()", + "cyan", + enabled=color, + ) + ) + + return "\n".join(lines) + + +@dataclass +class CSourceLocation: + filename: str | None = None + line: int | None = None + column: int | None = None + source_line: str | None = None + + @property + def display(self) -> str: + location = self.filename or "" + if self.line is not None: + column = self.column if self.column is not None else 1 + location = f"{location}:{self.line}:{column}" + return location + + +@dataclass +class CDiagnostic: + code: str + message: str + severity: str = "warning" + location: CSourceLocation | None = None + unit_kind: str | None = None + unit_name: str | None = None + + +@dataclass +class CPointer: + qualifiers: list[str] = field(default_factory=list) + + +@dataclass +class CArray: + size: str | None = None + static: bool = False + + +@dataclass +class CTypeRef: + base: str | None = None + qualifiers: list[str] = field(default_factory=list) + storage_class: list[str] = field(default_factory=list) + sign: str | None = None + width: str | None = None + tag_kind: str | None = None + tag_name: str | None = None + typedef_name: str | None = None + pointers: list[CPointer] = field(default_factory=list) + arrays: list[CArray] = field(default_factory=list) + kind: str = "type" + source_text: str = "" + resolved: Any = None + + @property + def pointer_depth(self) -> int: + return len(self.pointers) + + @property + def array_rank(self) -> int: + return len(self.arrays) + + @property + def effective_type_text(self) -> str: + if self.source_text: + return self.source_text + if self.typedef_name: + return self.typedef_name + if self.tag_kind and self.tag_name: + return f"{self.tag_kind} {self.tag_name}" + return self.base or "unknown" + + @property + def is_const_pointer(self) -> bool: + return bool(self.pointers) and "const" in self.qualifiers + + @property + def is_opaque_type(self) -> bool: + return bool(self.tag_kind and self.tag_name and self.resolved is None) + + +@dataclass +class CParameter: + name: str | None = None + type: CTypeRef = field(default_factory=CTypeRef) + source_location: CSourceLocation | None = None + + +@dataclass +class CFunction: + name: str + return_type: CTypeRef = field(default_factory=CTypeRef) + parameters: list[CParameter] = field(default_factory=list) + storage: list[str] = field(default_factory=list) + specifiers: list[str] = field(default_factory=list) + variadic: bool = False + is_definition: bool = False + source_location: CSourceLocation | None = None + + +@dataclass +class CField: + name: str | None = None + type: CTypeRef = field(default_factory=CTypeRef) + source_location: CSourceLocation | None = None + + +@dataclass +class CStruct: + name: str | None = None + fields: list[CField] = field(default_factory=list) + anonymous_id: str | None = None + opaque: bool = False + source_location: CSourceLocation | None = None + + +@dataclass +class CUnion: + name: str | None = None + fields: list[CField] = field(default_factory=list) + anonymous_id: str | None = None + source_location: CSourceLocation | None = None + + +@dataclass +class CEnumerator: + name: str + value: str | None = None + source_location: CSourceLocation | None = None + + +@dataclass +class CEnum: + name: str | None = None + constants: list[CEnumerator] = field(default_factory=list) + anonymous_id: str | None = None + source_location: CSourceLocation | None = None + + +@dataclass +class CTypedef: + name: str + type: CTypeRef = field(default_factory=CTypeRef) + source_location: CSourceLocation | None = None + + +@dataclass +class CGlobal: + name: str + type: CTypeRef = field(default_factory=CTypeRef) + source_location: CSourceLocation | None = None + + +@dataclass +class CMacro: + name: str + value: str | None = None + function_like: bool = False + source_location: CSourceLocation | None = None + + +@dataclass +class CInclude: + target: str + kind: str = "local" + resolved_path: str | None = None + source_location: CSourceLocation | None = None + + +@dataclass +class CFile: + filename: str | None = None + language: str = "c" + parser_status: str = "skeleton" + preprocessing: str = "raw" + functions: list[CFunction] = field(default_factory=list) + structs: list[CStruct] = field(default_factory=list) + unions: list[CUnion] = field(default_factory=list) + enums: list[CEnum] = field(default_factory=list) + typedefs: list[CTypedef] = field(default_factory=list) + globals: list[CGlobal] = field(default_factory=list) + macros: list[CMacro] = field(default_factory=list) + includes: list[CInclude] = field(default_factory=list) + diagnostics: list[CDiagnostic] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return c_model_to_dict(self) + + +@dataclass +class CProject: + files: dict[str, CFile] = field(default_factory=dict) + functions: dict[str, CFunction] = field(default_factory=dict) + structs: dict[str, CStruct] = field(default_factory=dict) + unions: dict[str, CUnion] = field(default_factory=dict) + enums: dict[str, CEnum] = field(default_factory=dict) + typedefs: dict[str, CTypedef] = field(default_factory=dict) + globals: dict[str, CGlobal] = field(default_factory=dict) + macros: dict[str, CMacro] = field(default_factory=dict) + includes: dict[str, CInclude] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return c_model_to_dict(self) diff --git a/c_parser/parser.py b/c_parser/parser.py new file mode 100644 index 000000000..71e9fa5c0 --- /dev/null +++ b/c_parser/parser.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from pathlib import Path + +from .models import CFile, CProject + + +_C_SOURCE_SUFFIXES = {".c", ".h"} + + +def _looks_like_existing_source_path(value: object) -> bool: + if isinstance(value, Path): + return value.is_file() + if not isinstance(value, str) or not value or "\n" in value: + return False + try: + return Path(value).is_file() + except OSError: + return False + + +def _collect_c_paths(path: Path) -> list[Path]: + return sorted( + p + for p in path.rglob("*") + if p.is_file() and p.suffix.lower() in _C_SOURCE_SUFFIXES + ) + + +class CParser: + """C parser skeleton entrypoint. + + This class intentionally returns typed empty models. Grammar parsing lands + in later phases after the public API, CLI, and serialization contracts are + stable. + """ + + def visit_file( + self, + source_or_path: str | Path, + filename: str | None = None, + *, + macro_defines: set[str] | dict[str, int | bool | str] | None = None, + include_dirs: Sequence[str | Path] | None = None, + preprocessing: str = "raw", + encoding: str = "utf-8", + ) -> CFile: + del macro_defines, include_dirs + if _looks_like_existing_source_path(source_or_path): + path = Path(source_or_path) + if filename is None: + filename = str(path) + path.read_text(encoding=encoding) + else: + str(source_or_path) + + return CFile(filename=filename, preprocessing=preprocessing) + + def visit_project( + self, + files: Mapping[str, str] | Sequence[str | Path] | str | Path, + *, + include_dirs: Sequence[str | Path] | None = None, + macro_defines: set[str] | dict[str, int | bool | str] | None = None, + preprocessing: str = "raw", + encoding: str = "utf-8", + ) -> CProject: + if isinstance(files, Mapping): + parsed_files = { + name: self.visit_file( + source, + filename=name, + include_dirs=include_dirs, + macro_defines=macro_defines, + preprocessing=preprocessing, + encoding=encoding, + ) + for name, source in files.items() + } + return CProject(files=parsed_files) + + paths: list[Path] = [] + root: Path | None = None + if isinstance(files, (str, Path)): + path = Path(files) + if path.is_dir(): + root = path + paths = _collect_c_paths(path) + else: + paths = [path] + else: + paths = [Path(p) for p in files] + + parsed_files: dict[str, CFile] = {} + for path in sorted(paths): + key = path.name if root is not None else str(path) + if root is not None: + key = str(path.relative_to(root)) + parsed_files[key] = self.visit_file( + path, + filename=key, + include_dirs=include_dirs, + macro_defines=macro_defines, + preprocessing=preprocessing, + encoding=encoding, + ) + return CProject(files=parsed_files) + + +_DEFAULT_PARSER = CParser() + + +def parse_c_file( + source_or_path: str | Path, + filename: str | None = None, + *, + macro_defines: set[str] | dict[str, int | bool | str] | None = None, + include_dirs: Sequence[str | Path] | None = None, + preprocessing: str = "raw", + encoding: str = "utf-8", +) -> CFile: + return _DEFAULT_PARSER.visit_file( + source_or_path, + filename=filename, + macro_defines=macro_defines, + include_dirs=include_dirs, + preprocessing=preprocessing, + encoding=encoding, + ) + + +def parse_c_project( + files: Mapping[str, str] | Sequence[str | Path] | str | Path, + *, + include_dirs: Sequence[str | Path] | None = None, + macro_defines: set[str] | dict[str, int | bool | str] | None = None, + preprocessing: str = "raw", + encoding: str = "utf-8", +) -> CProject: + return _DEFAULT_PARSER.visit_project( + files, + include_dirs=include_dirs, + macro_defines=macro_defines, + preprocessing=preprocessing, + encoding=encoding, + ) diff --git a/c_parser/preprocessor.py b/c_parser/preprocessor.py new file mode 100644 index 000000000..98fb06f39 --- /dev/null +++ b/c_parser/preprocessor.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +"""C preprocessor metadata placeholder.""" + +__all__: tuple[str, ...] = () diff --git a/c_parser/project.py b/c_parser/project.py new file mode 100644 index 000000000..3a2be400e --- /dev/null +++ b/c_parser/project.py @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +"""C project parsing placeholder.""" + +from .parser import parse_c_project + +__all__ = ("parse_c_project",) diff --git a/c_parser/type_resolver.py b/c_parser/type_resolver.py new file mode 100644 index 000000000..00c71219d --- /dev/null +++ b/c_parser/type_resolver.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +"""C type resolver placeholder.""" + +__all__: tuple[str, ...] = () diff --git a/c_parser/utils.py b/c_parser/utils.py new file mode 100644 index 000000000..dc843db0b --- /dev/null +++ b/c_parser/utils.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +"""C parser utility placeholder.""" + +__all__: tuple[str, ...] = () diff --git a/docs/c_parser/c_parser_architecture.md b/docs/c_parser/c_parser_architecture.md index 5bb3c5501..fc6ecaace 100644 --- a/docs/c_parser/c_parser_architecture.md +++ b/docs/c_parser/c_parser_architecture.md @@ -1,11 +1,41 @@ # C Parser Architecture Plan -Status: planning only. No C parser implementation exists in this branch yet. - -This document records the target architecture for a future C parser frontend in -x2py. The design is based on inspection of the current Fortran parser, -semantic IR conversion layer, `.pyi` parser/printer, CLI, tests, and fixture -workflow. +Status: skeleton implemented. The `c_parser` package, typed skeleton models, +public skeleton entrypoints, and explicit `x2py --language c --parse` CLI path +exist. Real C grammar parsing is still deferred. + +This document records the target architecture for the C parser frontend in +x2py. The initial skeleton now exists, and the remaining sections describe the +architecture it should grow into. The design is based on inspection of the +current Fortran parser, semantic IR conversion layer, `.pyi` parser/printer, +CLI, tests, and fixture workflow. + +## Current Implementation Snapshot + +Implemented now: + +- `c_parser/` package exists and is included in package discovery. +- `c_parser.models` defines JSON-stable skeleton dataclasses and `CParseError`. +- `c_parser.parser` exposes `CParser`, `parse_c_file`, and `parse_c_project`. +- `c_parser.cli` provides C-specific skeleton report formatting. +- `x2py.cli` dispatches `--language c --parse` to the C skeleton path. +- `--language c --semantics`, `--language c --pyi`, and C wrap-readiness are + rejected until semantic conversion exists. +- Focused skeleton CLI/API tests are unskipped while broader roadmap tests + remain skipped. + +Deferred: + +- lexer and lightweight preprocessing behavior +- declaration/declarator parsing +- function, struct, union, enum, typedef, global, macro, and include extraction +- include graph and project type resolution +- C semantic readiness, semantic IR conversion, and `.pyi` output + +Documentation rule: any future C parser implementation change must update all +affected docs under `docs/c_parser/` in the same change. This applies to model, +parser, CLI, test, fixture, semantic, and `.pyi` changes; documentation updates +should not wait for a separate request. ## Inspected Repository Areas @@ -104,10 +134,10 @@ for `.i` files or macro-expanded views, but the x2py C frontend still needs its own typed parser models, source-location handling, diagnostics, and project indexes. Invoking a compiler must not replace the grammar-style parser. -## Target Package Layout +## Package Layout -The future implementation should live in a separate package so it does not -destabilize the Fortran parser: +The implementation lives in a separate package so it does not destabilize the +Fortran parser: ```text c_parser/ @@ -123,59 +153,63 @@ c_parser/ utils.py ``` -Planned responsibilities: +Current and planned responsibilities: - `c_parser/models.py` - - Typed parser models. - - `CParseError` and compiler-style diagnostic rendering. - - JSON-stable dataclasses for files, translation units, declarations, types, - functions, macros/constants, and project indexes. + - Implemented: typed skeleton parser models, `CParseError`, compiler-style + diagnostic rendering, and JSON-stable dataclass serialization. + - Planned: richer source facts for declarations, types, functions, + macros/constants, and project indexes. - `c_parser/lexer.py` - - Tokenization and source-location preservation. + - Placeholder now. + - Planned: tokenization and source-location preservation. - Comment removal that preserves line mapping. - String/character literal awareness. - Line continuation handling for backslash-newline. - `c_parser/preprocessor.py` - - Lightweight preprocessing metadata. + - Placeholder now. + - Planned: lightweight preprocessing metadata. - Include directive collection. - Conditional branch tracking. - Object-like macro collection where safe. - Explicit diagnostics for unsupported macro patterns. - `c_parser/parser.py` - - Grammar-style recursive parser. - - Translation-unit visitor. - - Declaration, declarator, function, struct, union, enum, typedef, and global - variable visitors. - - Shared declaration/declarator parser. - - Module-level convenience wrappers. + - Implemented: skeleton `CParser`, `parse_c_file`, and `parse_c_project`. + - Planned: grammar-style recursive parser, translation-unit visitor, + declaration/declarator/function/composite-type visitors, and shared + declaration/declarator parsing. - `c_parser/project.py` - - File discovery for `.c`, `.h`, and possibly `.i`. + - Placeholder now. + - Planned: file discovery for `.c`, `.h`, and possibly `.i`. - Include graph construction. - Header/source association. - Cross-file type and typedef resolution. - `c_parser/type_resolver.py` - - C primitive type normalization. + - Placeholder now. + - Planned: C primitive type normalization. - Qualifier/storage-class handling. - Typedef chain resolution. - Pointer/array/function-pointer type helpers. - Safe constant expression folding for simple compile-time values. - `c_parser/cli.py` - - C-specific report formatting and serialization helpers. - - Called by `x2py.cli` behind explicit C flags. + - Implemented: skeleton report formatting and serialization helpers called by + `x2py.cli` behind explicit C flags. + - Planned: richer human output once real C facts are populated. - `c_parser/utils.py` - - Top-level splitting helpers for comma, parentheses, brackets, braces, and - declarator fragments. + - Placeholder now. + - Planned: top-level splitting helpers for comma, parentheses, brackets, + braces, and declarator fragments. ## Public API Shape -The public C API should mirror the Fortran style but remain C-specific: +The public C API mirrors the Fortran style but remains C-specific: ```python -parse_c_file(source_or_path, filename=None, macro_defines=None, include_dirs=None, encoding="utf-8") -> CFile -parse_c_project(files, include_dirs=None, macro_defines=None, encoding="utf-8") -> CProject +parse_c_file(source_or_path, filename=None, macro_defines=None, include_dirs=None, preprocessing="raw", encoding="utf-8") -> CFile +parse_c_project(files, include_dirs=None, macro_defines=None, preprocessing="raw", encoding="utf-8") -> CProject ``` -Expected companion class: +Implemented companion class: ```python class CParser: @@ -183,14 +217,13 @@ class CParser: def visit_project(...): ... ``` -The initial implementation should not re-export these from `x2py.__init__` -until the API is useful and tested. During early phases, it may be acceptable to -expose the skeleton only under `c_parser` and integrate the CLI with explicit -flags. +These entrypoints are exposed from `c_parser`, not re-exported from +`x2py.__init__`. Keeping the skeleton API under `c_parser` avoids making the +top-level x2py API promise C behavior before real parsing exists. ## Core Model Families -Proposed parser models: +Implemented skeleton parser models: - `CSourceLocation` - `filename` @@ -215,67 +248,74 @@ Proposed parser models: - `typedef_name` - `pointers` - `arrays` - - `function_pointer` + - `kind` - `source_text` + - `resolved` - `CPointer` - `qualifiers` - - `level` - `CArray` - `size` - - `is_static` - - `qualifiers` + - `static` - `CParameter` - `name` - `type` - `source_location` - - `is_variadic_marker` - `CFunction` - `name` - `return_type` - `parameters` - - `storage_class` - - `qualifiers` - - `is_variadic` + - `storage` + - `specifiers` + - `variadic` - `is_definition` - - `body_span` - `source_location` - `CField` - `name` - `type` - - `bit_width` - `source_location` - `CStruct` - `name` - `fields` - - `is_union` - - `is_anonymous` - - `typedef_names` + - `anonymous_id` + - `opaque` + - `source_location` +- `CUnion` + - `name` + - `fields` + - `anonymous_id` - `source_location` - `CEnum` - `name` - - `enumerators` - - `typedef_names` + - `constants` + - `anonymous_id` - `source_location` - `CEnumerator` - `name` - `value` - - `symbolic_value` - `source_location` - `CTypedef` - `name` - - `target_type` + - `type` + - `source_location` +- `CGlobal` + - `name` + - `type` - `source_location` - `CMacro` - `name` - `value` - - `macro_kind` - - `parameters` - - `is_safe_constant` + - `function_like` + - `source_location` +- `CInclude` + - `target` + - `kind` + - `resolved_path` - `source_location` - `CFile` - `filename` - - `source` - - `encoding` + - `language` + - `parser_status` + - `preprocessing` - `functions` - `structs` - `unions` @@ -285,16 +325,21 @@ Proposed parser models: - `macros` - `includes` - `diagnostics` - - `symbols` - `CProject` - `files` - `functions` - - `types` + - `structs` + - `unions` + - `enums` - `typedefs` + - `globals` - `macros` - - `include_graph` - - `header_source_pairs` - - `diagnostics` + - `includes` + +Future parser phases can add fields such as source spans, bit widths, function +pointer metadata, conditional-region metadata, include graphs, and project +diagnostics when the corresponding behavior lands. Additions should be +documented and tested with stable serialization expectations. ## Grammar-Style Parsing Strategy @@ -418,7 +463,15 @@ Initial non-goal: C project parsing should account for include graphs instead of Fortran `use` graphs. -Planned behavior: +Skeleton behavior: + +- `parse_c_project` accepts mappings, explicit paths, and directories. +- Directory mode currently discovers `.c` and `.h` files only. +- Returned `CProject` objects contain parsed empty `CFile` skeletons. +- Include graphs, cross-file indexes, and type resolution are not populated + yet. + +Planned behavior after project-resolution phases: - Collect `.c`, `.h`, and eventually `.i` files from explicit paths or directories. diff --git a/docs/c_parser/c_parser_cli_workflow.md b/docs/c_parser/c_parser_cli_workflow.md index d443a258f..098f997a9 100644 --- a/docs/c_parser/c_parser_cli_workflow.md +++ b/docs/c_parser/c_parser_cli_workflow.md @@ -1,12 +1,44 @@ # C Parser CLI Workflow Plan -Status: planning only. No C parser CLI implementation exists in this branch -yet. +Status: C parser skeleton implemented. The CLI command shape and stable empty +parse report exist, but no real C declarations are parsed yet. The C parser CLI workflow should be designed before parser implementation so future parser work lands behind a stable command shape, output schema, and diagnostic contract. +## Current Status + +Implemented skeleton commands: + +```bash +python -m x2py path/to/api.h --language c --parse +python -m x2py path/to/api.h --language c --parse --json +python -m x2py path/to/api.h --language c --parse --out report.json +``` + +The skeleton accepts explicit `.c` and `.h` files, plus directories in +explicit C mode. Directory scanning in C mode only collects `.c` and `.h` +files. Auto-detection is deferred, so omitting `--language` keeps the current +Fortran behavior. + +The C parser output differs from Fortran parser output by using C-specific +top-level sections: `functions`, `structs`, `unions`, `enums`, `typedefs`, +`globals`, `macros`, `includes`, and `diagnostics`. During the skeleton phase +all of those lists are empty and `parser_status` is `"skeleton"`. + +Unsupported C stages: + +```bash +python -m x2py path/to/api.h --language c --semantics +python -m x2py path/to/api.h --language c --pyi +python -m x2py path/to/api.h --language c --wrap-readiness +``` + +These commands return clear argparse errors until C semantic IR conversion and +`.pyi` generation are implemented. Fortran-only parse display flags such as +`--show-vars` and `--print-limit` are rejected in C mode. + ## Current CLI Baseline The current `x2py` CLI is Fortran-oriented: @@ -56,7 +88,7 @@ Rationale: - It lets Fortran remain the default during the long C parser stabilization period. -Optional short alias: +Optional short alias, not implemented in the skeleton: ```bash x2py --parse-c @@ -88,6 +120,11 @@ Initial flags: --debug-traceback ``` +Skeleton behavior also accepts `--no-color`. `CParseError` already supports +compiler-style diagnostic formatting and the `C_PARSER_DEBUG` environment +variable, although the skeleton parser does not yet raise syntax diagnostics +for declaration-shaped input. + C-specific flags to add only when needed: ```text @@ -128,8 +165,10 @@ File: include/example.h Unions: 0 Enums: 0 Typedefs: 0 + Globals: 0 Macros: 0 Includes: 0 + Diagnostics: 0 Parser status: skeleton ``` @@ -138,8 +177,10 @@ JSON output: ```json { "include/example.h": { + "filename": "include/example.h", "language": "c", "parser_status": "skeleton", + "preprocessing": "raw", "functions": [], "structs": [], "unions": [], @@ -164,7 +205,9 @@ the concepts differ. Proposed top-level per-file keys: ```text language +filename parser_status +preprocessing functions structs unions @@ -235,7 +278,7 @@ Default CLI behavior: Debug behavior: - `--debug-traceback` re-raises the error -- a C-specific env var such as `C_PARSER_DEBUG=1` may be added +- `C_PARSER_DEBUG=1` re-raises C parser errors - `FORTRAN_PARSER_DEBUG` should not control C behavior - a generic `X2PY_DEBUG=1` may be considered later @@ -247,19 +290,19 @@ Color behavior: ## CLI Test Expectations -Phase 1 should add CLI tests before real parsing: +Phase 1 has CLI tests before real parsing: - Existing Fortran CLI tests still pass unchanged. - `python -m x2py --help` lists `--language`. - `python -m x2py --language c --parse` is accepted. -- `python -m x2py --parse-c` is accepted only if the alias is added. +- `python -m x2py --parse-c` is not implemented in the skeleton. - `--language c --parse --json` emits stable skeleton JSON. - `--language c --parse --out report.json` writes JSON and suppresses stdout. -- `--language c --parse --no-color` affects C diagnostics. -- `--language c --parse --debug-traceback` re-raises `CParseError` once the - error class exists. -- `--show-vars` remains Fortran-specific or is rejected for C until a C - equivalent exists. +- `--language c --parse --no-color` is accepted; real diagnostic color tests + will matter once parser errors can be raised by C syntax. +- `--language c --parse --debug-traceback` is accepted. +- `--show-vars` and `--print-limit` are rejected in C mode until C-specific + display controls exist. - `--semantics` with `--language c` is rejected until C semantic conversion is implemented. - `--pyi` with `--language c` is rejected until C `.pyi` emission is @@ -267,10 +310,15 @@ Phase 1 should add CLI tests before real parsing: ## Integration Order -1. Add CLI language selection behind explicit flags. -2. Keep Fortran as default behavior. -3. Add a skeleton C report provider with no parser logic. -4. Add C-specific docs for command shape and placeholder output. -5. Add CLI tests around discovery, stable command behavior, JSON, output - files, and diagnostics. -6. Only then begin parser package/model work. +Completed skeleton order: + +1. Added CLI language selection behind explicit flags. +2. Kept Fortran as default behavior. +3. Added a C parser package and skeleton C report provider with no grammar + parsing. +4. Added C-specific docs for command shape and placeholder output. +5. Added CLI/API tests around discovery, stable command behavior, JSON, output + files, public entrypoints, and diagnostic formatting. + +Next implementation work should begin with lexer/preprocessor and declaration +model behavior while keeping the explicit `--language c` gate in place. diff --git a/docs/c_parser/c_parser_implementation_checklist.md b/docs/c_parser/c_parser_implementation_checklist.md index 152e82f95..c36c7311e 100644 --- a/docs/c_parser/c_parser_implementation_checklist.md +++ b/docs/c_parser/c_parser_implementation_checklist.md @@ -1,6 +1,8 @@ # C Parser Implementation Checklist -Status: planning checklist. No parser implementation exists in this branch yet. +Status: implementation checklist with Phase 1 skeleton and selected Phase 3 +skeleton work complete. The `c_parser` package and explicit C parse skeleton +exist, but real C grammar parsing is not implemented yet. This checklist is intentionally detailed so future work can proceed one branch, one checklist item, and one tested capability at a time. The C parser initiative @@ -35,6 +37,10 @@ stable. source locations, diagnostics, or project indexes. - [ ] Preserve the semantic IR layer as the source of truth. - [ ] Treat documentation as a first-class deliverable in every phase. +- [ ] Whenever C parser behavior, models, CLI behavior, tests, fixture + workflows, semantic integration, or `.pyi` behavior change, update every + affected file under `docs/c_parser/` in the same change. Do not wait for + a separate documentation request. - [ ] Update this checklist when implementation reality changes. ## Phase 0: Repository Inspection, Branch Setup, And Roadmap @@ -141,11 +147,11 @@ Scope: ### Phase 0 Risks And Open Questions -- [ ] Decide whether the future package should be named `c_parser` or another +- [x] Decide whether the future package should be named `c_parser` or another name before code lands. -- [ ] Decide whether `x2py.__init__` should expose C APIs during skeleton phase +- [x] Decide whether `x2py.__init__` should expose C APIs during skeleton phase or wait until Phase 3 models are useful. -- [ ] Decide whether C debug env var should be `C_PARSER_DEBUG` or generic +- [x] Decide whether C debug env var should be `C_PARSER_DEBUG` or generic `X2PY_DEBUG`. ## Phase 1: CLI And Documentation Skeleton First @@ -164,94 +170,94 @@ Scope: ### Documentation Tasks -- [ ] Update `docs/c_parser/c_parser_cli_workflow.md` with implemented command +- [x] Update `docs/c_parser/c_parser_cli_workflow.md` with implemented command examples. -- [ ] Update `docs/c_parser/c_parser_reference.md` with skeleton CLI behavior. -- [ ] Add a "Current Status" section showing which C features are placeholder +- [x] Update `docs/c_parser/c_parser_reference.md` with skeleton CLI behavior. +- [x] Add a "Current Status" section showing which C features are placeholder only. -- [ ] Document how C parser output differs from Fortran parser output. -- [ ] Document that auto-detection is deferred. -- [ ] Document that `--language c` is required initially. -- [ ] Document unsupported `--semantics --language c` behavior. -- [ ] Document unsupported `--pyi --language c` behavior. -- [ ] Document C parser diagnostics even if only skeleton diagnostics exist. -- [ ] Add examples for parse tree, JSON, and output file behavior. +- [x] Document how C parser output differs from Fortran parser output. +- [x] Document that auto-detection is deferred. +- [x] Document that `--language c` is required initially. +- [x] Document unsupported `--semantics --language c` behavior. +- [x] Document unsupported `--pyi --language c` behavior. +- [x] Document C parser diagnostics even if only skeleton diagnostics exist. +- [x] Add examples for parse tree, JSON, and output file behavior. ### CLI Design Tasks -- [ ] Add `--language {fortran,c}` to `x2py.cli`. -- [ ] Preserve current Fortran behavior when `--language` is omitted. -- [ ] Make `--language fortran` equivalent to current behavior. -- [ ] Add explicit C parse path behind `--language c --parse`. -- [ ] Decide whether to add `--parse-c` alias in this phase. +- [x] Add `--language {fortran,c}` to `x2py.cli`. +- [x] Preserve current Fortran behavior when `--language` is omitted. +- [x] Make `--language fortran` equivalent to current behavior. +- [x] Add explicit C parse path behind `--language c --parse`. +- [x] Decide not to add `--parse-c` alias in this phase. - [ ] If `--parse-c` is added, make it an alias for `--language c --parse`. -- [ ] Reject `--language c` without a supported stage flag. -- [ ] Reject `--language c --semantics` until Phase 10. -- [ ] Reject `--language c --pyi` until Phase 11. -- [ ] Reject Fortran-only flags in C mode if they do not apply. -- [ ] Keep `--json` behavior stable for parse output. -- [ ] Keep `--out` behavior stable for C parse JSON. -- [ ] Keep `--no-color` accepted in C mode. -- [ ] Keep `--debug-traceback` accepted in C mode. -- [ ] Do not change `fortran_parser.cli` unless a compatibility reason is +- [x] Reject `--language c` without a supported stage flag. +- [x] Reject `--language c --semantics` until Phase 10. +- [x] Reject `--language c --pyi` until Phase 11. +- [x] Reject Fortran-only flags in C mode if they do not apply. +- [x] Keep `--json` behavior stable for parse output. +- [x] Keep `--out` behavior stable for C parse JSON. +- [x] Keep `--no-color` accepted in C mode. +- [x] Keep `--debug-traceback` accepted in C mode. +- [x] Do not change `fortran_parser.cli` unless a compatibility reason is documented. ### Skeleton Report Tasks -- [ ] Create a minimal C report provider without real parsing. -- [ ] Ensure skeleton C report can accept `.c` and `.h` paths. -- [ ] Ensure skeleton C report can accept directories only in explicit C mode. -- [ ] Return `language: "c"` in C JSON output. -- [ ] Return `parser_status: "skeleton"` in C JSON output. -- [ ] Return empty `functions` list. -- [ ] Return empty `structs` list. -- [ ] Return empty `unions` list. -- [ ] Return empty `enums` list. -- [ ] Return empty `typedefs` list. -- [ ] Return empty `globals` list. -- [ ] Return empty `macros` list. -- [ ] Return empty `includes` list. -- [ ] Return empty `diagnostics` list unless a skeleton diagnostic is needed. -- [ ] Human tree output should show zero-count C sections and skeleton status. +- [x] Create a minimal C report provider without real parsing. +- [x] Ensure skeleton C report can accept `.c` and `.h` paths. +- [x] Ensure skeleton C report can accept directories only in explicit C mode. +- [x] Return `language: "c"` in C JSON output. +- [x] Return `parser_status: "skeleton"` in C JSON output. +- [x] Return empty `functions` list. +- [x] Return empty `structs` list. +- [x] Return empty `unions` list. +- [x] Return empty `enums` list. +- [x] Return empty `typedefs` list. +- [x] Return empty `globals` list. +- [x] Return empty `macros` list. +- [x] Return empty `includes` list. +- [x] Return empty `diagnostics` list unless a skeleton diagnostic is needed. +- [x] Human tree output should show zero-count C sections and skeleton status. ### CLI Test Tasks -- [ ] Add C CLI tests in a C-specific test file, for example +- [x] Add C CLI tests in a C-specific test file, for example `tests/parser/test_c_cli_skeleton.py` or `tests/c_parser/test_cli.py`. -- [ ] Test existing Fortran CLI behavior still passes. -- [ ] Test `--help` shows `--language`. -- [ ] Test `--language c --parse` accepts a temporary `.h` file. -- [ ] Test `--language c --parse --json` emits valid JSON. -- [ ] Test `--language c --parse --out report.json` writes JSON and suppresses +- [x] Test existing Fortran CLI behavior still passes. +- [x] Test `--help` shows `--language`. +- [x] Test `--language c --parse` accepts a temporary `.h` file. +- [x] Test `--language c --parse --json` emits valid JSON. +- [x] Test `--language c --parse --out report.json` writes JSON and suppresses stdout. -- [ ] Test `--language c --semantics` returns argparse error or clear +- [x] Test `--language c --semantics` returns argparse error or clear unsupported-stage error. -- [ ] Test `--language c --pyi` returns argparse error or clear +- [x] Test `--language c --pyi` returns argparse error or clear unsupported-stage error. -- [ ] Test `--language c --parse --show-vars` is rejected or ignored with +- [x] Test `--language c --parse --show-vars` is rejected or ignored with documented behavior. -- [ ] Test `--parse` without `--language` remains Fortran behavior. +- [x] Test `--parse` without `--language` remains Fortran behavior. - [ ] If `--parse-c` is added, test it maps to C parse mode. -- [ ] Test `--no-color` is accepted in C mode. +- [x] Test `--no-color` is accepted in C mode. - [ ] Test `NO_COLOR=1` is honored once C diagnostics exist. -- [ ] Test `--debug-traceback` is accepted in C mode. +- [x] Test `--debug-traceback` is accepted in C mode. ### Phase 1 Definition Of Done -- [ ] Users can discover C mode from CLI help. -- [ ] Users can run a stable C parse skeleton command. -- [ ] C JSON skeleton output has a documented schema. -- [ ] Fortran CLI behavior is unchanged. -- [ ] Documentation includes the exact command workflow. -- [ ] Tests cover the skeleton command workflow. +- [x] Users can discover C mode from CLI help. +- [x] Users can run a stable C parse skeleton command. +- [x] C JSON skeleton output has a documented schema. +- [x] Fortran CLI behavior is unchanged. +- [x] Documentation includes the exact command workflow. +- [x] Tests cover the skeleton command workflow. ### Phase 1 Risks And Open Questions -- [ ] Decide whether to implement a temporary skeleton module inside +- [x] Decide whether to implement a temporary skeleton module inside `x2py.cli` or create `c_parser/cli.py` early. -- [ ] Decide whether skeleton output should include zero-count sections in +- [x] Decide whether skeleton output should include zero-count sections in human output or omit empty sections like Fortran. -- [ ] Decide whether `--json` should eventually support semantic C output or +- [x] Decide whether `--json` should eventually support semantic C output or stay parse-only. ## Phase 2: Testing Infrastructure @@ -357,7 +363,7 @@ Scope: - [ ] C test directory structure is present. - [ ] C fixture directory structure is present. - [ ] C golden update workflow is documented. -- [ ] Placeholder tests pass against skeleton behavior. +- [x] Placeholder tests pass against skeleton behavior. - [ ] Fortran tests still pass. - [ ] No real parser claims are made without tests. @@ -365,9 +371,9 @@ Scope: - [x] Run the skipped C roadmap suite and confirm it collects without importing `c_parser` at module import time. -- [ ] Run C skeleton CLI tests. -- [ ] Run existing parser CLI tests. -- [ ] Run a small targeted test command, for example +- [x] Run C skeleton CLI tests. +- [x] Run existing parser CLI tests. +- [x] Run a small targeted test command, for example `python -m pytest -q tests/parser/test_cli.py tests/parser/test_c_cli_skeleton.py`. - [ ] Do not update Fortran goldens. @@ -393,104 +399,104 @@ Scope: ### Package Skeleton Tasks -- [ ] Create `c_parser/__init__.py`. -- [ ] Create `c_parser/__main__.py`. -- [ ] Create `c_parser/models.py`. -- [ ] Create `c_parser/parser.py`. -- [ ] Create `c_parser/lexer.py`. -- [ ] Create `c_parser/preprocessor.py`. -- [ ] Create `c_parser/type_resolver.py`. -- [ ] Create `c_parser/project.py`. -- [ ] Create `c_parser/cli.py`. -- [ ] Create `c_parser/utils.py`. -- [ ] Add `c_parser*` to package discovery in `pyproject.toml`. -- [ ] Add `c_parser` to coverage source when implementation begins. -- [ ] Keep imports from `x2py.cli` explicit and isolated. +- [x] Create `c_parser/__init__.py`. +- [x] Create `c_parser/__main__.py`. +- [x] Create `c_parser/models.py`. +- [x] Create `c_parser/parser.py`. +- [x] Create `c_parser/lexer.py`. +- [x] Create `c_parser/preprocessor.py`. +- [x] Create `c_parser/type_resolver.py`. +- [x] Create `c_parser/project.py`. +- [x] Create `c_parser/cli.py`. +- [x] Create `c_parser/utils.py`. +- [x] Add `c_parser*` to package discovery in `pyproject.toml`. +- [x] Add `c_parser` to coverage source when implementation begins. +- [x] Keep imports from `x2py.cli` explicit and isolated. ### Error Model Tasks -- [ ] Implement `CParseError` as a `ValueError` subclass. -- [ ] Include `filename`. -- [ ] Include `line_number`. -- [ ] Include `column`. -- [ ] Include `source_line`. -- [ ] Include `base_message`. -- [ ] Include `code`. -- [ ] Include parser raise location for debug diagnostics. -- [ ] Implement `format_diagnostic(color=False, debug=None)`. -- [ ] Use C diagnostic code prefix such as `CPARSE001`. -- [ ] Add color handling equivalent to `FortranParseError`. -- [ ] Add optional C debug env var. -- [ ] Test C parse error attributes. -- [ ] Test compiler-style diagnostic rendering. +- [x] Implement `CParseError` as a `ValueError` subclass. +- [x] Include `filename`. +- [x] Include `line_number`. +- [x] Include `column`. +- [x] Include `source_line`. +- [x] Include `base_message`. +- [x] Include `code`. +- [x] Include parser raise location for debug diagnostics. +- [x] Implement `format_diagnostic(color=False, debug=None)`. +- [x] Use C diagnostic code prefix such as `CPARSE001`. +- [x] Add color handling equivalent to `FortranParseError`. +- [x] Add optional C debug env var. +- [x] Test C parse error attributes. +- [x] Test compiler-style diagnostic rendering. - [ ] Test color and no-color behavior. -- [ ] Test debug note behavior. +- [x] Test debug note behavior. ### Model Tasks -- [ ] Define `CSourceLocation`. -- [ ] Define `CDiagnostic`. -- [ ] Define `CTypeRef`. -- [ ] Define `CPointer`. -- [ ] Define `CArray`. -- [ ] Define `CParameter`. -- [ ] Define `CFunction`. -- [ ] Define `CField`. -- [ ] Define `CStruct`. -- [ ] Define `CUnion` or use `CStruct(is_union=True)`. -- [ ] Define `CEnum`. -- [ ] Define `CEnumerator`. -- [ ] Define `CTypedef`. -- [ ] Define `CGlobal`. -- [ ] Define `CMacro`. -- [ ] Define `CInclude`. -- [ ] Define `CFile`. -- [ ] Define `CProject`. -- [ ] Add helper properties for pointer depth. -- [ ] Add helper properties for array rank. -- [ ] Add helper properties for effective type text. -- [ ] Add helper properties for `is_const_pointer`. -- [ ] Add helper properties for `is_opaque_type`. -- [ ] Add helper properties for source-location display. +- [x] Define `CSourceLocation`. +- [x] Define `CDiagnostic`. +- [x] Define `CTypeRef`. +- [x] Define `CPointer`. +- [x] Define `CArray`. +- [x] Define `CParameter`. +- [x] Define `CFunction`. +- [x] Define `CField`. +- [x] Define `CStruct`. +- [x] Define `CUnion` or use `CStruct(is_union=True)`. +- [x] Define `CEnum`. +- [x] Define `CEnumerator`. +- [x] Define `CTypedef`. +- [x] Define `CGlobal`. +- [x] Define `CMacro`. +- [x] Define `CInclude`. +- [x] Define `CFile`. +- [x] Define `CProject`. +- [x] Add helper properties for pointer depth. +- [x] Add helper properties for array rank. +- [x] Add helper properties for effective type text. +- [x] Add helper properties for `is_const_pointer`. +- [x] Add helper properties for `is_opaque_type`. +- [x] Add helper properties for source-location display. ### Serialization Tasks -- [ ] Add `_to_dict` or equivalent serialization helper. -- [ ] Avoid cycles in JSON. -- [ ] Keep source locations JSON-serializable. -- [ ] Decide whether sets serialize as sorted lists. -- [ ] Ensure dataclass defaults produce stable JSON. -- [ ] Add tests for empty `CFile` serialization. +- [x] Add `_to_dict` or equivalent serialization helper. +- [x] Avoid cycles in JSON. +- [x] Keep source locations JSON-serializable. +- [x] Decide whether sets serialize as sorted lists. +- [x] Ensure dataclass defaults produce stable JSON. +- [x] Add tests for empty `CFile` serialization. - [ ] Add tests for each model's minimal JSON shape. - [ ] Add tests for source-location serialization. - [ ] Add tests that unknown/unresolved metadata is preserved. ### Public API Skeleton Tasks -- [ ] Implement `CParser` class. -- [ ] Implement `CParser.visit_file` returning skeleton or model-only `CFile`. -- [ ] Implement `CParser.visit_project` returning skeleton/model-only +- [x] Implement `CParser` class. +- [x] Implement `CParser.visit_file` returning skeleton or model-only `CFile`. +- [x] Implement `CParser.visit_project` returning skeleton/model-only `CProject`. -- [ ] Implement module-level `_DEFAULT_PARSER`. -- [ ] Implement `parse_c_file`. -- [ ] Implement `parse_c_project`. -- [ ] Add public API tests for source strings. -- [ ] Add public API tests for file paths. -- [ ] Add public API tests for empty source. -- [ ] Add public API tests for unknown suffix. +- [x] Implement module-level `_DEFAULT_PARSER`. +- [x] Implement `parse_c_file`. +- [x] Implement `parse_c_project`. +- [x] Add public API tests for source strings. +- [x] Add public API tests for file paths. +- [x] Add public API tests for empty source. +- [x] Add public API tests for unknown suffix. ### Phase 3 Definition Of Done -- [ ] `c_parser` imports cleanly. -- [ ] Skeleton public APIs return typed models. -- [ ] JSON serialization is stable and tested. -- [ ] C CLI uses `c_parser` rather than a temporary inline provider. -- [ ] Fortran parser API remains unchanged. -- [ ] Docs describe the new package and API status. +- [x] `c_parser` imports cleanly. +- [x] Skeleton public APIs return typed models. +- [x] JSON serialization is stable and tested. +- [x] C CLI uses `c_parser` rather than a temporary inline provider. +- [x] Fortran parser API remains unchanged. +- [x] Docs describe the new package and API status. ### Phase 3 Risks And Open Questions -- [ ] Decide whether `CUnion` should subclass/share `CStruct`. +- [x] Decide whether `CUnion` should subclass/share `CStruct`. - [ ] Decide how to represent anonymous structs/unions/enums. - [ ] Decide whether macros belong in `CFile.macros` only or also symbols. - [ ] Decide whether `CTypeRef` should be a single recursive model or contain diff --git a/docs/c_parser/c_parser_reference.md b/docs/c_parser/c_parser_reference.md index 18bb27e4b..e032e75e9 100644 --- a/docs/c_parser/c_parser_reference.md +++ b/docs/c_parser/c_parser_reference.md @@ -1,6 +1,7 @@ # C Parser Reference -Status: planning reference. The C parser is not implemented yet. +Status: skeleton reference. The `c_parser` package and explicit C CLI parse +path exist, but no real C declarations are parsed yet. This document is the future home for the C parser user and developer reference. It should evolve into the C equivalent of `fortran_parser.md` as implementation @@ -40,6 +41,27 @@ Possible later source form: Project input should accept explicit files and directories. Directory scanning must be explicit C mode at first to avoid changing Fortran CLI behavior. +## Current Status + +Implemented: + +- `c_parser` package skeleton +- typed empty C parser models +- `CParser`, `parse_c_file`, and `parse_c_project` +- `CParseError` with compiler-style diagnostic formatting +- explicit `x2py --language c --parse` skeleton output +- C JSON skeleton output and `--out` behavior +- rejection of C `--semantics`, `--pyi`, and `--wrap-readiness` + +Placeholder only: + +- function extraction +- declarations and declarators +- structs, unions, enums, and typedef parsing +- include and macro collection +- project include graph and cross-file type resolution +- semantic readiness, semantic IR conversion, and `.pyi` generation + ## Planned Supported C Subset The initial supported subset should focus on stable wrapper-relevant APIs: @@ -110,7 +132,7 @@ Target module-level entrypoints: from c_parser import parse_c_file, parse_c_project ``` -Expected signatures: +Implemented skeleton signatures: ```python parse_c_file( @@ -118,6 +140,7 @@ parse_c_file( filename=None, macro_defines=None, include_dirs=None, + preprocessing="raw", encoding="utf-8", ) @@ -125,14 +148,15 @@ parse_c_project( files, include_dirs=None, macro_defines=None, + preprocessing="raw", encoding="utf-8", ) ``` -These should return typed parser models and dictionaries analogous to the -Fortran parser API. Re-export from `x2py` should wait until the API is tested -and documented. +These return typed parser models analogous to the Fortran parser API. During +the skeleton phase they return empty C file/project models. Re-export from +`x2py` is still deferred; users should import from `c_parser`. The parser itself should stay parse-only. If the C frontend later gains wrappability assessment, that should live in the semantic layer after C parser @@ -146,9 +170,10 @@ Initial explicit mode: ```bash x2py path/to/api.h --language c --parse x2py path/to/api.h --language c --parse --json +x2py path/to/api.h --language c --parse --out report.json ``` -Optional alias: +Optional alias, not implemented in the skeleton: ```bash x2py path/to/api.h --parse-c @@ -163,8 +188,10 @@ Per-file shape: ```text { "": { + "filename": "", "language": "c", - "parser_status": "implemented|partial|skeleton", + "parser_status": "skeleton", + "preprocessing": "raw", "functions": [], "structs": [], "unions": [], @@ -212,9 +239,9 @@ facts to let later semantic work decide what is safe: Those facts should be stored in parser models, but not turned into a parser-side `wrappable` report. -## Planned Error Handling +## Error Handling -The parser should define `CParseError` with: +The parser defines `CParseError` with: - `filename` - `line_number` @@ -223,9 +250,11 @@ The parser should define `CParseError` with: - `base_message` - `code` - internal parser raise location for debug mode -- `format_diagnostic(color=False, debug=False)` +- `format_diagnostic(color=False, debug=None)` The CLI should print compiler-style diagnostics without tracebacks by default. +The skeleton has the error type and formatter, but real syntax diagnostics are +not produced yet because grammar parsing is still deferred. ## Planned Testing Workflow @@ -248,11 +277,13 @@ Test families should mirror the Fortran parser: - error fixture/golden tests - corpus parse-only tests -The first committed C parser tests are skipped roadmap tests under -`tests/parser/c/`. They are intentionally collected but skipped before the -parser exists. Future implementation branches should unskip only the tests for -the capability they implement, then merge those branches back into -`c-parser/main`. +The C test area now contains both unskipped skeleton tests and skipped roadmap +tests under `tests/parser/c/`. The skeleton tests cover public entrypoints, +empty model serialization, CLI discovery, JSON/output-file behavior, and +unsupported C stages. The broader roadmap tests remain skipped until their +matching implementation branches land. Future implementation branches should +unskip only the tests for the capability they implement, then merge those +branches back into `c-parser/main`. Fixture layout should be separate from Fortran: @@ -283,13 +314,16 @@ The C parser documentation lives under: docs/c_parser/ ``` -Current planning documents: +Current documents: - `c_parser_reference.md` - `c_parser_architecture.md` - `c_parser_cli_workflow.md` - `c_parser_implementation_checklist.md` +- `c_parser_main_merge_guard.md` -Future implementation should update these docs in the same change whenever C -parser behavior, public API, CLI output, fixture workflow, semantic conversion, -or `.pyi` output changes. +Documentation update rule: every C parser implementation change must update +all affected files under `docs/c_parser/` in the same change. This includes +changes to parser behavior, public API, models, CLI output, tests, fixture +workflow, semantic conversion, semantic readiness, or `.pyi` output. Do not +wait for a separate documentation request before updating these docs. diff --git a/pyproject.toml b/pyproject.toml index ba2820128..b8ec4d5ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,13 +14,13 @@ dependencies = [ [tool.setuptools.packages.find] where = ["."] -include = ["fortran_parser*", "semantics*", "x2py*",] +include = ["c_parser*", "fortran_parser*", "semantics*", "x2py*",] [project.scripts] x2py = "x2py.cli:main" [tool.coverage.run] -source = ["fortran_parser", "semantics", "x2py"] +source = ["c_parser", "fortran_parser", "semantics", "x2py"] branch = true parallel = true diff --git a/tests/parser/c/test_c_cli_skeleton.py b/tests/parser/c/test_c_cli_skeleton.py new file mode 100644 index 000000000..16709c7b5 --- /dev/null +++ b/tests/parser/c/test_c_cli_skeleton.py @@ -0,0 +1,153 @@ +# -*- coding: utf-8 -*- +"""C parser CLI skeleton coverage.""" + +import json +import subprocess +import sys +from pathlib import Path + + +def test_cli_help_shows_explicit_c_language_mode(): + cmd = [sys.executable, "-m", "x2py", "--help"] + + res = subprocess.run(cmd, capture_output=True, text=True, check=True) + + assert "--language" in res.stdout + assert "c" in res.stdout + + +def test_cli_c_parse_human_tree_output_for_header(tmp_path: Path): + header = tmp_path / "api.h" + header.write_text("int add(int a, int b);\n", encoding="utf-8") + cmd = [sys.executable, "-m", "x2py", str(header), "--language", "c", "--parse"] + + res = subprocess.run(cmd, capture_output=True, text=True, check=True) + + assert f"File: {header}" in res.stdout + assert "Language: c" in res.stdout + assert "Functions: 0" in res.stdout + assert "Parser status: skeleton" in res.stdout + + +def test_cli_c_parse_json_stdout_for_header(tmp_path: Path): + header = tmp_path / "api.h" + header.write_text("int add(int a, int b);\n", encoding="utf-8") + cmd = [sys.executable, "-m", "x2py", str(header), "--language", "c", "--parse", "--json"] + + res = subprocess.run(cmd, capture_output=True, text=True, check=True) + payload = json.loads(res.stdout) + file_payload = payload[str(header)] + + assert file_payload["language"] == "c" + assert file_payload["parser_status"] == "skeleton" + assert file_payload["functions"] == [] + assert file_payload["structs"] == [] + assert file_payload["unions"] == [] + assert file_payload["enums"] == [] + assert file_payload["typedefs"] == [] + assert file_payload["globals"] == [] + assert file_payload["macros"] == [] + assert file_payload["includes"] == [] + assert file_payload["diagnostics"] == [] + + +def test_cli_c_parse_json_out_writes_file_and_suppresses_stdout(tmp_path: Path): + header = tmp_path / "api.h" + output = tmp_path / "report.json" + header.write_text("double scale(double x);\n", encoding="utf-8") + cmd = [ + sys.executable, + "-m", + "x2py", + str(header), + "--language", + "c", + "--parse", + "--json", + "--out", + str(output), + ] + + res = subprocess.run(cmd, capture_output=True, text=True, check=True) + payload = json.loads(output.read_text(encoding="utf-8")) + + assert res.stdout == "" + assert payload[str(header)]["language"] == "c" + assert payload[str(header)]["parser_status"] == "skeleton" + + +def test_cli_c_parse_out_without_json_writes_json_and_suppresses_stdout(tmp_path: Path): + header = tmp_path / "api.h" + output = tmp_path / "report.json" + header.write_text("int run(void);\n", encoding="utf-8") + cmd = [ + sys.executable, + "-m", + "x2py", + str(header), + "--language", + "c", + "--parse", + "--out", + str(output), + ] + + res = subprocess.run(cmd, capture_output=True, text=True, check=True) + payload = json.loads(output.read_text(encoding="utf-8")) + + assert res.stdout == "" + assert payload[str(header)]["parser_status"] == "skeleton" + + +def test_cli_c_semantic_stages_are_rejected_until_implemented(tmp_path: Path): + header = tmp_path / "api.h" + header.write_text("int add(int a, int b);\n", encoding="utf-8") + + for stage in ("--semantics", "--pyi", "--wrap-readiness"): + cmd = [sys.executable, "-m", "x2py", str(header), "--language", "c", stage] + res = subprocess.run(cmd, capture_output=True, text=True) + + assert res.returncode != 0 + assert "not supported" in res.stderr.lower() + + +def test_cli_c_rejects_fortran_only_parse_flags(tmp_path: Path): + header = tmp_path / "api.h" + header.write_text("int add(int a, int b);\n", encoding="utf-8") + cmd = [sys.executable, "-m", "x2py", str(header), "--language", "c", "--parse", "--show-vars"] + + res = subprocess.run(cmd, capture_output=True, text=True) + + assert res.returncode != 0 + assert "show-vars" in res.stderr + assert "Fortran-only" in res.stderr + + +def test_cli_c_no_color_and_debug_traceback_flags_are_accepted(tmp_path: Path): + header = tmp_path / "api.h" + header.write_text("int run(void);\n", encoding="utf-8") + cmd = [ + sys.executable, + "-m", + "x2py", + str(header), + "--language", + "c", + "--parse", + "--no-color", + "--debug-traceback", + ] + + res = subprocess.run(cmd, capture_output=True, text=True, check=True) + + assert "Parser status: skeleton" in res.stdout + + +def test_cli_without_language_keeps_fortran_default_behavior(): + fixture = Path(__file__).resolve().parents[2] / "data" / "fortran" / "general" / "basic_subroutine.f90" + cmd = [sys.executable, "-m", "x2py", str(fixture), "--parse"] + + res = subprocess.run(cmd, capture_output=True, text=True, check=True) + + assert "subroutine add1" in res.stdout + assert "Language: c" not in res.stdout diff --git a/tests/parser/c/test_c_public_api_skeleton.py b/tests/parser/c/test_c_public_api_skeleton.py new file mode 100644 index 000000000..1dac065c9 --- /dev/null +++ b/tests/parser/c/test_c_public_api_skeleton.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +"""C parser public API skeleton coverage.""" + +from pathlib import Path + + +def test_parse_c_file_accepts_inline_source_and_returns_typed_skeleton(): + from c_parser import CFile, parse_c_file + + parsed = parse_c_file("int add(int a, int b);\n", filename="inline.h") + + assert isinstance(parsed, CFile) + assert parsed.filename == "inline.h" + assert parsed.language == "c" + assert parsed.parser_status == "skeleton" + assert parsed.functions == [] + + +def test_parse_c_file_accepts_path_input_and_preserves_filename(tmp_path: Path): + from c_parser import parse_c_file + + header = tmp_path / "api.h" + header.write_text("double scale(double x);\n", encoding="utf-8") + + parsed = parse_c_file(header) + + assert parsed.filename == str(header) + assert parsed.functions == [] + + +def test_parse_c_file_accepts_empty_source_and_unknown_suffix(): + from c_parser import parse_c_file + + parsed = parse_c_file("", filename="empty.src") + + assert parsed.filename == "empty.src" + assert parsed.functions == [] + assert parsed.diagnostics == [] + + +def test_parse_c_project_accepts_mapping_sources(): + from c_parser import CProject, parse_c_project + + project = parse_c_project( + { + "types.h": "typedef int api_int;\n", + "api.h": '#include "types.h"\napi_int answer(void);\n', + } + ) + + assert isinstance(project, CProject) + assert set(project.files) == {"types.h", "api.h"} + assert project.files["api.h"].language == "c" + assert project.functions == {} + + +def test_parse_c_project_accepts_directory_input_with_c_and_h_files(tmp_path: Path): + from c_parser import parse_c_project + + (tmp_path / "api.h").write_text("int add(int a, int b);\n", encoding="utf-8") + (tmp_path / "api.c").write_text('#include "api.h"\n', encoding="utf-8") + (tmp_path / "notes.txt").write_text("ignored\n", encoding="utf-8") + + project = parse_c_project(tmp_path) + + assert set(project.files) == {"api.h", "api.c"} + + +def test_c_file_serialization_is_json_stable(): + from c_parser import parse_c_file + + parsed = parse_c_file("", filename="empty.c") + + assert parsed.to_dict() == { + "filename": "empty.c", + "language": "c", + "parser_status": "skeleton", + "preprocessing": "raw", + "functions": [], + "structs": [], + "unions": [], + "enums": [], + "typedefs": [], + "globals": [], + "macros": [], + "includes": [], + "diagnostics": [], + } + + +def test_public_c_parser_entrypoints_do_not_include_parser_side_readiness(): + import c_parser + + assert hasattr(c_parser, "parse_c_file") + assert hasattr(c_parser, "parse_c_project") + assert not hasattr(c_parser, "assess_c_wrap_readiness") + + +def test_c_parse_error_attributes_and_diagnostic_formatting(): + from c_parser import CParseError + + err = CParseError( + "unexpected token", + filename="bad.h", + line_number=2, + column=5, + source_line="int broken(;", + ) + + assert err.filename == "bad.h" + assert err.line_number == 2 + assert err.column == 5 + assert err.base_message == "unexpected token" + assert err.code == "CPARSE001" + + diagnostic = err.format_diagnostic(color=False, debug=True) + assert "bad.h:2:5: error[CPARSE001]: unexpected token" in diagnostic + assert "2 | int broken(;" in diagnostic + assert "note: parser raised at" in diagnostic diff --git a/x2py/cli.py b/x2py/cli.py index 82dac6185..6799e6f0d 100644 --- a/x2py/cli.py +++ b/x2py/cli.py @@ -7,6 +7,8 @@ from dataclasses import asdict, fields, is_dataclass from pathlib import Path +from c_parser.cli import format_c_report, parse_c_report +from c_parser.models import CParseError from fortran_parser.models import FortranParseError from fortran_parser.parser import FortranParser from fortran_parser.cli import _format_report @@ -242,6 +244,8 @@ def main() -> int: " python -m x2py path/to/src_dir --parse --print-limit 20\n" " Print parser JSON:\n" " python -m x2py path/to/file.f90 --parse --json\n" + " Parse C skeleton JSON:\n" + " python -m x2py path/to/api.h --language c --parse --json\n" " Write parser JSON:\n" " python -m x2py path/to/file.f90 --parse --json --out report.json\n" " Write one JSON file next to each source:\n" @@ -265,7 +269,13 @@ def main() -> int: " pip install rich" ), ) - parser.add_argument("paths", nargs="+", help="Fortran source file(s), .pyi file(s), or directory path(s)") + parser.add_argument("paths", nargs="+", help="Source file(s), .pyi file(s), or directory path(s)") + parser.add_argument( + "--language", + choices=("fortran", "c"), + default="fortran", + help="Frontend language. Defaults to fortran; C currently supports only --parse skeleton output.", + ) parser.add_argument("--parse", action="store_true", help="Run and output parser stage report") parser.add_argument( "--show-vars", @@ -297,6 +307,18 @@ def main() -> int: parser.add_argument("--debug-traceback", action="store_true", help="Re-raise parser errors for debug") args = parser.parse_args() + if args.language == "c": + if not (args.parse or args.semantics or args.pyi or args.wrap_readiness): + parser.error("--language c requires --parse; C semantics and .pyi output are not supported yet") + if args.semantics: + parser.error("--semantics is not supported for --language c yet") + if args.pyi: + parser.error("--pyi is not supported for --language c yet") + if args.wrap_readiness: + parser.error("--wrap-readiness is semantic-layer output and is not supported for --language c yet") + if args.show_vars or args.print_limit is not None or args.vars_limit is not None: + parser.error("--show-vars/--print-limit are Fortran-only and are not supported for --language c") + if args.out is not None and not (args.parse or args.semantics or args.pyi or args.wrap_readiness): parser.error("--out requires a stage flag: choose one of --parse, --semantics, --pyi, or --wrap-readiness") @@ -311,10 +333,19 @@ def main() -> int: parser.error("Select at least one stage flag: --parse, --semantics, --pyi, or --wrap-readiness") try: - parse_payload = _parse_report(args.paths) if args.parse else None + parse_payload = ( + parse_c_report(args.paths) + if args.parse and args.language == "c" + else _parse_report(args.paths) if args.parse else None + ) semantic_payload = _semantic_report(args.paths) if (args.semantics or args.pyi) else None readiness_payload = _wrap_readiness_report(args.paths) if args.wrap_readiness else None _attach_wrap_readiness(semantic_payload, readiness_payload) + except CParseError as exc: + if args.debug_traceback or _env_flag("C_PARSER_DEBUG"): + raise + print(exc.format_diagnostic(color=_diagnostic_color_enabled(disabled=args.no_color), debug=False), file=sys.stderr) + return 1 except FortranParseError as exc: if args.debug_traceback or _env_flag("FORTRAN_PARSER_DEBUG"): raise @@ -378,7 +409,10 @@ def main() -> int: elif args.pyi and not args.json: print_pyi_output(_format_pyi_report(semantic_payload or {})) elif args.parse and not (args.semantics or args.json or args.pyi): - print(_format_report(parse_payload or {}, show_vars=args.show_vars or args.vars_limit is not None, print_limit=print_limit)) + if args.language == "c": + print(format_c_report(parse_payload or {})) + else: + print(_format_report(parse_payload or {}, show_vars=args.show_vars or args.vars_limit is not None, print_limit=print_limit)) else: print(json.dumps(payload, indent=2))