From 48a6fae7d4b6e38351ff0cf89899d173089d9df5 Mon Sep 17 00:00:00 2001 From: said Date: Wed, 20 May 2026 13:09:28 +0100 Subject: [PATCH 1/3] codex: add c parser roadmap docs --- docs/c_parser/c_parser_architecture.md | 525 ++++++++ docs/c_parser/c_parser_cli_workflow.md | 310 +++++ .../c_parser_implementation_checklist.md | 1193 +++++++++++++++++ docs/c_parser/c_parser_reference.md | 261 ++++ 4 files changed, 2289 insertions(+) create mode 100644 docs/c_parser/c_parser_architecture.md create mode 100644 docs/c_parser/c_parser_cli_workflow.md create mode 100644 docs/c_parser/c_parser_implementation_checklist.md create mode 100644 docs/c_parser/c_parser_reference.md diff --git a/docs/c_parser/c_parser_architecture.md b/docs/c_parser/c_parser_architecture.md new file mode 100644 index 000000000..f666f20ff --- /dev/null +++ b/docs/c_parser/c_parser_architecture.md @@ -0,0 +1,525 @@ +# C Parser Architecture Plan + +Status: planning only. No C parser implementation exists in this branch yet. + +This document records the target architecture for a future C parser frontend in +x2py. The design is based on inspection of the current Fortran parser, +semantic IR conversion layer, `.pyi` parser/printer, CLI, tests, and fixture +workflow. + +## Inspected Repository Areas + +- `README.md` +- `fortran_parser.md` +- `parser_implementation_reference.md` +- `Semantic_Multilanguage_Wrapper_Runtime_Architecture.md` +- `docs/pyi_format.md` +- `fortran_parser/lexer.py` +- `fortran_parser/parser.py` +- `fortran_parser/models.py` +- `fortran_parser/type_resolver.py` +- `fortran_parser/utils.py` +- `fortran_parser/cli.py` +- `x2py/cli.py` +- `x2py/__init__.py` +- `semantics/models.py` +- `semantics/fortran2ir.py` +- `semantics/pyi_printer.py` +- `semantics/pyi_parser.py` +- `tests/parser/test_cli.py` +- `tests/parser/test_fortran_fixture_suite.py` +- `tests/parser/test_fortran_error_fixture_suite.py` +- `tests/parser/test_parser_developer_tutorial.py` +- `tests/parser/test_parser_public_entrypoints.py` +- `tests/parser/test_wrap_readiness.py` +- `tests/parser/test_error_handling.py` +- `tests/semantics/test_fortran2ir.py` +- `tests/semantics/test_semantic_conversion_smoke.py` +- `tests/pyi/test_pyi_to_ir.py` +- `tests/pyi/test_pyi_fixture_suite.py` +- `tests/parser/fortran/generate_fortran_parser_goldens.py` +- `tests/parser/fortran/errors/generate_fortran_parser_error_goldens.py` +- `tests/semantics/generate_semantic_fixtures.py` +- `tests/pyi/generate_pyi_fixtures.py` +- `tests/_shared/fixture_outputs.py` + +## Architectural Observations From The Existing Parser + +The Fortran parser is not a compiler frontend. It is a wrapper-oriented +semantic extraction frontend with a bounded language subset. Its important +architectural properties are: + +- A lexer/preprocessor stage normalizes source while preserving original line + numbers and source lines for diagnostics. +- The parser uses recursive source-unit slicing instead of a whole-file scan. +- Each source unit is parsed inside an explicit `_ParserScope`; there is no + ambient global current module/procedure stack. +- Unit visitors are small and grammar-shaped: file, module, submodule, program, + procedure, interface, derived type, and block data each receive only their + own source substring. +- The shared declaration parser is reused for procedure arguments/results, + module variables, program variables, block data variables, and derived type + fields. +- Grammar-region splitting separates header, specification, execution, and + contains regions. Wrapper extraction mostly ignores executable bodies. +- Parser model objects are typed dataclasses with stable JSON-friendly fields. +- Project parsing builds indexes and dependency order from imports/uses. +- Readiness diagnostics are explicit and user-facing, with file-level + `wrappable`, blocker groups, and unit-level blocker records. +- CLI output has a stable human tree, JSON output, readiness output, output + file behavior, no-color support, and debug traceback opt-in. +- Semantic IR conversion is a separate visitor layer. Parser output is a helper + input, not the source of truth. +- `.pyi` generation and parsing operate over semantic IR, not parser internals. +- Fixture testing combines focused unit tests, parser JSON goldens, error + goldens, semantic fixtures, `.pyi` fixtures, and corpus parse-only coverage. + +The C parser should follow these patterns where they map cleanly to C syntax. +It should not be a giant regex parser, a compiler wrapper, or a libclang-only +dependency architecture. + +## Target Package Layout + +The future implementation should live in a separate package so it does not +destabilize the Fortran parser: + +```text +c_parser/ + __init__.py + __main__.py + cli.py + lexer.py + models.py + parser.py + preprocessor.py + project.py + type_resolver.py + utils.py +``` + +Planned responsibilities: + +- `c_parser/models.py` + - Typed parser models. + - `CParseError` and compiler-style diagnostic rendering. + - JSON-stable dataclasses for files, translation units, declarations, types, + functions, macros/constants, project indexes, and readiness reports. +- `c_parser/lexer.py` + - Tokenization and source-location preservation. + - Comment removal that preserves line mapping. + - String/character literal awareness. + - Line continuation handling for backslash-newline. +- `c_parser/preprocessor.py` + - Lightweight preprocessing metadata. + - Include directive collection. + - Conditional branch tracking. + - Object-like macro collection where safe. + - Explicit diagnostics for unsupported macro patterns. +- `c_parser/parser.py` + - Grammar-style recursive parser. + - Translation-unit visitor. + - Declaration, declarator, function, struct, union, enum, typedef, and global + variable visitors. + - Shared declaration/declarator parser. + - Module-level convenience wrappers. +- `c_parser/project.py` + - File discovery for `.c`, `.h`, and possibly `.i`. + - Include graph construction. + - Header/source association. + - Cross-file type and typedef resolution. +- `c_parser/type_resolver.py` + - C primitive type normalization. + - Qualifier/storage-class handling. + - Typedef chain resolution. + - Pointer/array/function-pointer type helpers. + - Safe constant expression folding for simple compile-time values. +- `c_parser/cli.py` + - C-specific report formatting and serialization helpers. + - Called by `x2py.cli` behind explicit C flags. +- `c_parser/utils.py` + - Top-level splitting helpers for comma, parentheses, brackets, braces, and + declarator fragments. + +## Public API Shape + +The public C API should mirror the Fortran style but remain C-specific: + +```python +parse_c_file(source_or_path, filename=None, macro_defines=None, include_dirs=None, encoding="utf-8") -> CFile +parse_c_project(files, include_dirs=None, macro_defines=None, encoding="utf-8") -> CProject +assess_c_wrap_readiness(code, filename=None, include_dirs=None, macro_defines=None) -> dict +``` + +Expected companion class: + +```python +class CParser: + def visit_file(...): ... + def visit_project(...): ... + def visit_wrap_readiness(...): ... +``` + +The initial implementation should not re-export these from `x2py.__init__` +until the API is useful and tested. During early phases, it may be acceptable to +expose the skeleton only under `c_parser` and integrate the CLI with explicit +flags. + +## Core Model Families + +Proposed parser models: + +- `CSourceLocation` + - `filename` + - `line` + - `column` + - `source_line` +- `CDiagnostic` + - `code` + - `message` + - `severity` + - `location` + - `unit_kind` + - `unit_name` +- `CTypeRef` + - `base` + - `qualifiers` + - `storage_class` + - `sign` + - `width` + - `tag_kind` + - `tag_name` + - `typedef_name` + - `pointers` + - `arrays` + - `function_pointer` + - `source_text` +- `CPointer` + - `qualifiers` + - `level` +- `CArray` + - `size` + - `is_static` + - `qualifiers` +- `CParameter` + - `name` + - `type` + - `source_location` + - `is_variadic_marker` +- `CFunction` + - `name` + - `return_type` + - `parameters` + - `storage_class` + - `qualifiers` + - `is_variadic` + - `is_definition` + - `body_span` + - `source_location` +- `CField` + - `name` + - `type` + - `bit_width` + - `source_location` +- `CStruct` + - `name` + - `fields` + - `is_union` + - `is_anonymous` + - `typedef_names` + - `source_location` +- `CEnum` + - `name` + - `enumerators` + - `typedef_names` + - `source_location` +- `CEnumerator` + - `name` + - `value` + - `symbolic_value` + - `source_location` +- `CTypedef` + - `name` + - `target_type` + - `source_location` +- `CMacro` + - `name` + - `value` + - `macro_kind` + - `parameters` + - `is_safe_constant` + - `source_location` +- `CFile` + - `filename` + - `source` + - `encoding` + - `functions` + - `structs` + - `unions` + - `enums` + - `typedefs` + - `globals` + - `macros` + - `includes` + - `diagnostics` + - `symbols` +- `CProject` + - `files` + - `functions` + - `types` + - `typedefs` + - `macros` + - `include_graph` + - `header_source_pairs` + - `diagnostics` + +## Grammar-Style Parsing Strategy + +C does not have Fortran-style modules, but it still has parseable scoped +regions: + +- translation unit +- preprocessor directive lines +- external declarations +- function prototypes +- function definitions +- declaration specifier sequences +- declarators +- parameter declaration lists +- struct/union/enum definitions +- compound statement bodies + +The C parser should parse external declarations by slicing top-level grammar +regions, not by scanning the full file repeatedly. The high-level flow should +be: + +1. Normalize source through a C lexer/preprocessor layer. +2. Produce tokens or logical source records with original source locations. +3. Visit the translation unit. +4. Split direct external declarations while balancing parentheses, brackets, + braces, string literals, and comments. +5. Classify each external declaration: + - include/preprocessor directive + - typedef + - function prototype + - function definition + - struct/union/enum definition + - global variable/static const + - unsupported/macro-dependent declaration +6. Dispatch to a small visitor for that declaration kind. +7. Use a shared declaration-specifier and declarator parser to build type + references for functions, parameters, fields, globals, and typedefs. +8. Ignore executable function bodies except where needed to find the matching + brace and preserve source spans. + +## Declarator-Centered Design + +C type syntax is declarator-centered. The future parser should make declarator +parsing a first-class subsystem, not a pile of ad hoc string splitting. + +Required layered pieces: + +- declaration specifier parser + - storage classes: `extern`, `static`, `typedef`, `register`, `_Thread_local` + - qualifiers: `const`, `restrict`, `volatile`, `_Atomic` + - primitive base: `void`, `char`, `short`, `int`, `long`, `float`, `double`, + `_Bool`, `_Complex` + - signedness: `signed`, `unsigned` + - tags: `struct`, `union`, `enum` + - typedef-name references +- declarator parser + - identifier extraction + - pointer chains + - arrays + - function parameters + - parenthesized declarators + - function pointers + - anonymous abstract declarators where needed +- entity applier + - convert one declaration specifier plus one declarator into a typed model + - reuse this for function returns, parameters, fields, globals, and typedefs + +This is the C equivalent of the Fortran parser's shared declaration backend. + +## Preprocessing Strategy + +The C frontend must be preprocessor-aware without trying to be a full C +preprocessor in v1. + +Initial target: + +- Strip comments safely while preserving line numbers. +- Fold backslash-newline continuations. +- Record `#include` directives as structured include dependencies. +- Record `#define` object-like macros for simple constants. +- Record function-like macros as unsupported or deferred metadata. +- Track `#if`, `#ifdef`, `#ifndef`, `#elif`, `#else`, `#endif` condition sets + similarly to the Fortran duplicate-check branch tracking. +- Allow optional `macro_defines` to select active branches. +- Preserve inactive branch diagnostics when macro selection is not requested. + +Initial non-goal: + +- Do not implement arbitrary macro expansion. +- Do not attempt token-paste/stringify semantics. +- Do not require libclang as the only way to understand headers. + +## Project Parsing Strategy + +C project parsing should account for include graphs instead of Fortran `use` +graphs. + +Planned behavior: + +- Collect `.c`, `.h`, and eventually `.i` files from explicit paths or + directories. +- Parse headers and sources into `CFile` models. +- Build an include graph keyed by normalized path. +- Preserve unresolved includes as diagnostics. +- Associate likely header/source pairs by basename and include relation. +- Resolve typedefs, structs, unions, enums, and constants across parsed files. +- Track duplicate symbols by C namespace: + - ordinary identifiers + - typedef names + - struct/union/enum tags + - labels are not wrapper-relevant and should not enter the public index +- Keep project resolution tolerant enough to parse partial projects, while + readiness diagnostics explain missing dependencies. + +## Readiness Diagnostics + +C readiness should be explicit and actionable. It should not merely report +parse success. + +Planned blocker families: + +- no functions found +- parse errors +- unsupported declarations +- macro-dependent declarations +- unresolved include +- unresolved typedef +- unresolved tag type +- incomplete struct/union used by value +- function pointer parameter or return +- callback parameter requiring manual projection +- variadic function +- K&R style function definition +- array parameter with unknown size relationship +- pointer ownership ambiguity +- pointer mutability ambiguity +- unsupported compiler extension +- unsupported bitfield layout +- unsupported anonymous composite type in public API + +The JSON readiness shape should stay close to Fortran: + +```text +{ + "n_functions": int, + "n_structs": int, + "n_unions": int, + "n_enums": int, + "n_typedefs": int, + "n_macros": int, + "unsupported_constructs": [...], + "unresolved_types": [...], + "macro_dependent_declarations": [...], + "wrappability_blockers": [...], + "unit_blockers": [...], + "why_not_wrappable": [...], + "wrappable": bool +} +``` + +## Semantic IR Mapping + +The semantic layer is the source of truth. The C parser should only help create +or update semantic modules. + +Planned mapping: + +- C file or header group -> `SemanticModule` +- C function -> `SemanticFunction` +- C parameter -> `SemanticArgument` +- C primitive -> `SemanticType` +- C pointer -> constraints and ownership metadata +- C array -> `Shape(...)`, `ORDER_C`, and pointer/extent metadata +- `const` -> read-only ownership/constraint metadata +- `restrict` -> aliasing metadata +- structs/unions -> `SemanticClass` or named opaque semantic type +- enums -> constants or a future semantic enum representation +- typedefs -> semantic aliases or metadata, depending on IR support at the + time of implementation +- macros/constants -> `SemanticArgument` module variables with `Constant` + where safe + +Pointer ownership and lifetime should be conservative. Ambiguous pointers +should be represented with diagnostics and metadata rather than guessed as safe +Python APIs. + +## `.pyi` Integration + +Generated `.pyi` stubs for C should come after parser models, readiness, and +semantic IR conversion are stable. + +Likely stub patterns: + +- plain scalar functions: + - `def f(x: Int32) -> Float64: ...` +- pointer arguments: + - use semantic constraints such as `Pointer`, `Writable`, `Const`, `Shape` + only after the IR supports them cleanly +- arrays: + - `Float64[Shape("n"), ORDER_C]` +- opaque handles: + - classes or named semantic types with ownership constraints +- structs: + - `class struct_name: ...` when field layout is useful and stable +- callbacks: + - defer until function pointer semantics are explicit +- constants: + - `Final[...]` + +The existing `.pyi` parser already supports `Final`, `private`, `native_call`, +imports, classes, functions, shapes, and native projection entries. C-specific +work should extend the semantic model intentionally before changing `.pyi` +syntax. + +## Isolation Policy + +All C parser work must be isolated from project `main` until the C frontend is +mature and stable. + +Long-lived integration branch: + +```text +c-parser/main +``` + +Roadmap branch: + +```text +c-parser/phase-0-roadmap +``` + +Future feature branches should be created from `c-parser/main` and merged only +back into `c-parser/main`, for example: + +```text +c-parser/phase-1-cli-and-docs +c-parser/phase-2-testing +c-parser/phase-3-models +c-parser/phase-4-lexer +c-parser/phase-5-declarations +c-parser/phase-6-functions +c-parser/phase-7-structs-enums +c-parser/phase-8-project-resolution +c-parser/phase-9-readiness +c-parser/phase-10-semantics +c-parser/phase-11-pyi +c-parser/phase-12-corpus-stabilization +``` + +No C parser branch should merge directly into project `main` until the +frontend has stable parser behavior, docs, CLI, tests, readiness diagnostics, +semantic conversion, and `.pyi` expectations. diff --git a/docs/c_parser/c_parser_cli_workflow.md b/docs/c_parser/c_parser_cli_workflow.md new file mode 100644 index 000000000..824028154 --- /dev/null +++ b/docs/c_parser/c_parser_cli_workflow.md @@ -0,0 +1,310 @@ +# C Parser CLI Workflow Plan + +Status: planning only. No C parser CLI implementation exists in this branch +yet. + +The C parser CLI workflow should be designed before parser implementation so +future parser work lands behind a stable command shape, output schema, and +diagnostic contract. + +## Current CLI Baseline + +The current `x2py` CLI is Fortran-oriented: + +```bash +python -m x2py --parse +python -m x2py --parse --json +python -m x2py --parse --wrap-readiness +python -m x2py --semantics +python -m x2py --pyi +``` + +Important current behaviors to preserve: + +- Stage flags are explicit: `--parse`, `--semantics`, `--pyi`. +- `--out` writes stage output and suppresses stdout. +- `--wrap-readiness` requires `--parse`. +- `--json` currently applies to parse output. +- Parse diagnostics are compiler-style and go to stderr. +- Python tracebacks are hidden by default. +- `--debug-traceback` or parser debug env vars re-raise parse errors. +- Diagnostics use ANSI color by default unless `--no-color` or `NO_COLOR=1` + disables it. +- Human parse output is a stable tree. +- JSON parse output is a stable per-file object keyed by input path. + +The C CLI must integrate without breaking any of these Fortran behaviors. + +## Command Shape Recommendation + +Strong initial preference: + +```bash +x2py --language c --parse +``` + +Rationale: + +- It is explicit. +- It scales to future language frontends. +- It avoids surprising users by auto-detecting mixed-language directories too + early. +- It lets Fortran remain the default during the long C parser stabilization + period. + +Optional short alias: + +```bash +x2py --parse-c +``` + +The short alias is convenient, but should be secondary. If added, it should be +implemented as a strict alias for `--language c --parse`. + +Auto-detection should be later: + +```bash +x2py --parse +``` + +Auto-detection should wait until C parser behavior is mature enough to handle +mixed source trees predictably. Until then, `--parse` without `--language` +should keep existing Fortran behavior. + +## Planned Flags + +Initial flags: + +```text +--language {fortran,c} +--parse +--json +--wrap-readiness +--out [PATH] +--no-color +--debug-traceback +``` + +C-specific flags to add only when needed: + +```text +--include-dir PATH +--define NAME[=VALUE] +--undef NAME +--show-macros +--show-includes +--print-limit N +``` + +Potential later flags: + +```text +--auto-language +--project +--header-mode +--source-mode +--preprocessed +``` + +## Early Skeleton Behavior + +Phase 1 may implement CLI structure before a real parser exists. Skeleton +behavior should be intentionally stable: + +```bash +x2py include/example.h --language c --parse +``` + +Human output: + +```text +File: include/example.h + Language: c + Functions: 0 + Structs: 0 + Unions: 0 + Enums: 0 + Typedefs: 0 + Macros: 0 + Includes: 0 + Parser status: skeleton +``` + +JSON output: + +```json +{ + "include/example.h": { + "language": "c", + "parser_status": "skeleton", + "functions": [], + "structs": [], + "unions": [], + "enums": [], + "typedefs": [], + "globals": [], + "macros": [], + "includes": [], + "diagnostics": [], + "wrap_readiness": { + "n_functions": 0, + "n_structs": 0, + "n_unions": 0, + "n_enums": 0, + "n_typedefs": 0, + "n_macros": 0, + "wrappability_blockers": [ + { + "code": "parser_skeleton", + "message": "The C parser frontend is present but not implemented yet.", + "items": [] + } + ], + "unit_blockers": [], + "why_not_wrappable": [ + "The C parser frontend is present but not implemented yet." + ], + "wrappable": false + } + } +} +``` + +Readiness output: + +```text +File: include/example.h + Wrappable: no + Why not wrappable: + - The C parser frontend is present but not implemented yet. +``` + +The skeleton should not claim C files are wrappable. + +## JSON Parse Schema + +The C parse JSON should be per-file and should not reuse Fortran key names when +the concepts differ. Proposed top-level per-file keys: + +```text +language +parser_status +functions +structs +unions +enums +typedefs +globals +macros +includes +diagnostics +wrap_readiness +``` + +Every model should include source-location metadata once implementation begins: + +```text +source_location: { + filename: str | null, + line: int | null, + column: int | null, + source_line: str | null +} +``` + +## Human Tree Output + +The tree should mirror the Fortran parser style: compact by default, expanded +with explicit flags. + +Initial mature output shape: + +```text +File: src/api.c + Includes: 2 + - api.h + - stddef.h + Functions: 2 + - int add(int a, int b) + - void scale(double *x, size_t n) + Structs: 1 + - struct vector (fields=2) + Typedefs: 1 + - vector_t -> struct vector + Macros: 1 + - MAX_DIM = 16 +``` + +With readiness: + +```text +File: src/api.c + Wrappable: no + Why not wrappable: + - Some functions use pointer arguments with unknown ownership. + * scale:x is a non-const pointer without ownership metadata +``` + +## Diagnostics Behavior + +C parser errors should use a `CParseError` model with the same user experience +as `FortranParseError`: + +```text +src/api.h:12:5: error[CPARSE001]: Unsupported declaration. + | +12 | __attribute__((vector_size(16))) float v; + | ^ +``` + +Default CLI behavior: + +- print the formatted diagnostic to stderr +- exit with status code `1` +- do not show Python traceback +- colorize when color is enabled + +Debug behavior: + +- `--debug-traceback` re-raises the error +- a C-specific env var such as `C_PARSER_DEBUG=1` may be added +- `FORTRAN_PARSER_DEBUG` should not control C behavior +- a generic `X2PY_DEBUG=1` may be considered later + +Color behavior: + +- `--no-color` disables ANSI diagnostics +- `NO_COLOR=1` disables ANSI diagnostics +- Windows color behavior can follow the existing `colorama` pattern + +## CLI Test Expectations + +Phase 1 should add CLI tests before real parsing: + +- Existing Fortran CLI tests still pass unchanged. +- `python -m x2py --help` lists `--language`. +- `python -m x2py --language c --parse` is accepted. +- `python -m x2py --parse-c` is accepted only if the alias is added. +- `--language c --parse --json` emits stable skeleton JSON. +- `--language c --parse --wrap-readiness` emits stable skeleton readiness. +- `--language c --parse --out report.json` writes JSON and suppresses stdout. +- `--language c --parse --no-color` affects C diagnostics. +- `--language c --parse --debug-traceback` re-raises `CParseError` once the + error class exists. +- `--wrap-readiness` still requires `--parse`. +- `--show-vars` remains Fortran-specific or is rejected for C until a C + equivalent exists. +- `--semantics` with `--language c` is rejected until C semantic conversion is + implemented. +- `--pyi` with `--language c` is rejected until C `.pyi` emission is + implemented. + +## Integration Order + +1. Add CLI language selection behind explicit flags. +2. Keep Fortran as default behavior. +3. Add a skeleton C report provider with no parser logic. +4. Add C-specific docs for command shape and placeholder output. +5. Add CLI tests around discovery, stable command behavior, JSON, readiness, + output files, and diagnostics. +6. Only then begin parser package/model work. diff --git a/docs/c_parser/c_parser_implementation_checklist.md b/docs/c_parser/c_parser_implementation_checklist.md new file mode 100644 index 000000000..bb5001185 --- /dev/null +++ b/docs/c_parser/c_parser_implementation_checklist.md @@ -0,0 +1,1193 @@ +# C Parser Implementation Checklist + +Status: planning checklist. No parser implementation exists in this branch yet. + +This checklist is intentionally detailed so future work can proceed one branch, +one checklist item, and one tested capability at a time. The C parser initiative +must remain isolated from project `main` until the frontend is mature and +stable. + +## Global Rules + +- [ ] Keep all C parser work on `c-parser/main` and child branches until the + frontend is stable. +- [ ] Do not merge C parser work directly into project `main`. +- [ ] Keep the Fortran parser behavior unchanged unless a future task + explicitly requires shared infrastructure changes. +- [ ] Put C parser implementation in a separate `c_parser` package. +- [ ] Keep C parser tests separated from existing Fortran tests. +- [ ] Gate all integration through explicit C flags or C-specific APIs. +- [ ] Do not implement a giant regex parser. +- [ ] Do not implement a whole-file scanner as the core architecture. +- [ ] Do not make libclang the only parser architecture. +- [ ] Preserve the semantic IR layer as the source of truth. +- [ ] Treat documentation as a first-class deliverable in every phase. +- [ ] Update this checklist when implementation reality changes. + +## Phase 0: Repository Inspection, Branch Setup, And Roadmap + +Branch target: + +- `c-parser/main` +- `c-parser/phase-0-roadmap` + +Scope: + +- Planning only. +- Documentation only. +- No parser package. +- No parser logic. +- No CLI code changes. +- No fixture generation. + +### Branch And Isolation Tasks + +- [x] Start from project `main`. +- [x] Create long-lived integration branch `c-parser/main`. +- [x] Create planning branch `c-parser/phase-0-roadmap` from + `c-parser/main`. +- [ ] Merge `c-parser/phase-0-roadmap` back into `c-parser/main`. +- [ ] Confirm `main` has no C parser planning commits unless intentionally + merged later after stabilization. +- [ ] Record the branch strategy in C parser docs. +- [ ] Use `codex: ...` prefix for planning commit message. + +### Repository Inspection Tasks + +- [x] Inspect `README.md`. +- [x] Inspect `fortran_parser.md`. +- [x] Inspect `parser_implementation_reference.md`. +- [x] Inspect `Semantic_Multilanguage_Wrapper_Runtime_Architecture.md`. +- [x] Inspect `docs/pyi_format.md`. +- [x] Inspect `fortran_parser/lexer.py`. +- [x] Inspect `fortran_parser/parser.py`. +- [x] Inspect `fortran_parser/models.py`. +- [x] Inspect `fortran_parser/type_resolver.py`. +- [x] Inspect `fortran_parser/utils.py`. +- [x] Inspect `fortran_parser/cli.py`. +- [x] Inspect `x2py/cli.py`. +- [x] Inspect `x2py/__init__.py`. +- [x] Inspect `semantics/models.py`. +- [x] Inspect `semantics/fortran2ir.py`. +- [x] Inspect `semantics/pyi_printer.py`. +- [x] Inspect `semantics/pyi_parser.py`. +- [x] Inspect parser CLI tests. +- [x] Inspect parser public entrypoint tests. +- [x] Inspect parser developer tutorial tests. +- [x] Inspect wrap-readiness tests. +- [x] Inspect parser error handling tests. +- [x] Inspect parser fixture/golden suite. +- [x] Inspect parser golden regeneration script. +- [x] Inspect parser error golden regeneration script. +- [x] Inspect semantic conversion tests. +- [x] Inspect semantic fixture generator. +- [x] Inspect `.pyi` tests. +- [x] Inspect `.pyi` fixture generator. +- [x] Avoid spending analysis on ignored Fortran source fixtures except file + layout and fixture workflow. +- [x] Avoid spending analysis on generated JSON fixtures. + +### Planning Document Tasks + +- [x] Create `docs/c_parser/`. +- [x] Create `docs/c_parser/c_parser_reference.md`. +- [x] Create `docs/c_parser/c_parser_architecture.md`. +- [x] Create `docs/c_parser/c_parser_cli_workflow.md`. +- [x] Create `docs/c_parser/c_parser_implementation_checklist.md`. +- [x] Document the branch isolation strategy. +- [x] Document Fortran parser architecture observations. +- [x] Document proposed C parser package layout. +- [x] Document public API shape. +- [x] Document CLI command shape. +- [x] Document early skeleton CLI behavior. +- [x] Document JSON schema direction. +- [x] Document readiness diagnostics direction. +- [x] Document semantic IR mapping direction. +- [x] Document `.pyi` integration direction. +- [x] Document v1 non-goals. + +### Phase 0 Definition Of Done + +- [x] Planning docs exist under `docs/c_parser/`. +- [x] Docs clearly say no parser implementation exists yet. +- [x] Docs make CLI and documentation Phase 1 deliverables. +- [x] Docs preserve the grammar-style parser requirement. +- [x] Docs preserve project-main isolation. +- [ ] Planning branch is committed. +- [ ] Planning branch is merged into `c-parser/main`. + +### Phase 0 Test Expectations + +- [ ] No test changes are required in Phase 0. +- [ ] Run a documentation-safe sanity command such as `git status`. +- [ ] Do not regenerate fixtures. +- [ ] Do not run parser golden update scripts. + +### Phase 0 Risks And Open Questions + +- [ ] Decide whether the future package should be named `c_parser` or another + name before code lands. +- [ ] Decide whether `x2py.__init__` should expose C APIs during skeleton phase + or wait until Phase 3 models are useful. +- [ ] Decide whether C debug env var should be `C_PARSER_DEBUG` or generic + `X2PY_DEBUG`. + +## Phase 1: CLI And Documentation Skeleton First + +Branch target: + +- `c-parser/phase-1-cli-and-docs` + +Scope: + +- User-visible command shape. +- Documentation skeleton. +- Placeholder C parser reports. +- CLI tests. +- No real parser logic. + +### Documentation Tasks + +- [ ] Update `docs/c_parser/c_parser_cli_workflow.md` with implemented command + examples. +- [ ] Update `docs/c_parser/c_parser_reference.md` with skeleton CLI behavior. +- [ ] Add a "Current Status" section showing which C features are placeholder + only. +- [ ] Document how C parser output differs from Fortran parser output. +- [ ] Document that auto-detection is deferred. +- [ ] Document that `--language c` is required initially. +- [ ] Document unsupported `--semantics --language c` behavior. +- [ ] Document unsupported `--pyi --language c` behavior. +- [ ] Document C parser diagnostics even if only skeleton diagnostics exist. +- [ ] Add examples for parse tree, JSON, readiness, and output file behavior. + +### CLI Design Tasks + +- [ ] Add `--language {fortran,c}` to `x2py.cli`. +- [ ] Preserve current Fortran behavior when `--language` is omitted. +- [ ] Make `--language fortran` equivalent to current behavior. +- [ ] Add explicit C parse path behind `--language c --parse`. +- [ ] Decide whether to add `--parse-c` alias in this phase. +- [ ] If `--parse-c` is added, make it an alias for `--language c --parse`. +- [ ] Reject `--language c` without a supported stage flag. +- [ ] Reject `--language c --semantics` until Phase 10. +- [ ] Reject `--language c --pyi` until Phase 11. +- [ ] Reject Fortran-only flags in C mode if they do not apply. +- [ ] Keep `--wrap-readiness` requiring `--parse`. +- [ ] Keep `--json` behavior stable for parse output. +- [ ] Keep `--out` behavior stable for C parse JSON. +- [ ] Keep `--no-color` accepted in C mode. +- [ ] Keep `--debug-traceback` accepted in C mode. +- [ ] Do not change `fortran_parser.cli` unless a compatibility reason is + documented. + +### Skeleton Report Tasks + +- [ ] Create a minimal C report provider without real parsing. +- [ ] Ensure skeleton C report can accept `.c` and `.h` paths. +- [ ] Ensure skeleton C report can accept directories only in explicit C mode. +- [ ] Return `language: "c"` in C JSON output. +- [ ] Return `parser_status: "skeleton"` in C JSON output. +- [ ] Return empty `functions` list. +- [ ] Return empty `structs` list. +- [ ] Return empty `unions` list. +- [ ] Return empty `enums` list. +- [ ] Return empty `typedefs` list. +- [ ] Return empty `globals` list. +- [ ] Return empty `macros` list. +- [ ] Return empty `includes` list. +- [ ] Return empty `diagnostics` list unless a skeleton diagnostic is needed. +- [ ] Return readiness with `wrappable: false`. +- [ ] Include a `parser_skeleton` blocker. +- [ ] Human tree output should show zero-count C sections and skeleton status. +- [ ] Readiness output should show the skeleton blocker. + +### CLI Test Tasks + +- [ ] Add C CLI tests in a C-specific test file, for example + `tests/parser/test_c_cli_skeleton.py` or `tests/c_parser/test_cli.py`. +- [ ] Test existing Fortran CLI behavior still passes. +- [ ] Test `--help` shows `--language`. +- [ ] Test `--language c --parse` accepts a temporary `.h` file. +- [ ] Test `--language c --parse --json` emits valid JSON. +- [ ] Test `--language c --parse --wrap-readiness` emits stable readiness. +- [ ] Test `--language c --parse --out report.json` writes JSON and suppresses + stdout. +- [ ] Test `--language c --semantics` returns argparse error or clear + unsupported-stage error. +- [ ] Test `--language c --pyi` returns argparse error or clear + unsupported-stage error. +- [ ] Test `--language c --parse --show-vars` is rejected or ignored with + documented behavior. +- [ ] Test `--parse` without `--language` remains Fortran behavior. +- [ ] If `--parse-c` is added, test it maps to C parse mode. +- [ ] Test `--no-color` is accepted in C mode. +- [ ] Test `NO_COLOR=1` is honored once C diagnostics exist. +- [ ] Test `--debug-traceback` is accepted in C mode. + +### Phase 1 Definition Of Done + +- [ ] Users can discover C mode from CLI help. +- [ ] Users can run a stable C parse skeleton command. +- [ ] C JSON skeleton output has a documented schema. +- [ ] C readiness skeleton output is stable and not falsely wrappable. +- [ ] Fortran CLI behavior is unchanged. +- [ ] Documentation includes the exact command workflow. +- [ ] Tests cover the skeleton command workflow. + +### Phase 1 Risks And Open Questions + +- [ ] Decide whether to implement a temporary skeleton module inside + `x2py.cli` or create `c_parser/cli.py` early. +- [ ] Decide whether skeleton output should include zero-count sections in + human output or omit empty sections like Fortran. +- [ ] Decide whether `--json` should eventually support semantic C output or + stay parse-only. + +## Phase 2: Testing Infrastructure + +Branch target: + +- `c-parser/phase-2-testing` + +Scope: + +- Test directories, fixtures, golden scripts, and update workflow. +- Minimal placeholder fixtures are allowed. +- No real parser logic unless needed to support skeleton output already added. + +### Test Layout Tasks + +- [ ] Create a dedicated C parser test area. +- [ ] Choose between `tests/c_parser/` and `tests/parser/c/` for focused C + parser tests. +- [ ] Create `tests/data/c/general/`. +- [ ] Create `tests/data/c/errors/parser/`. +- [ ] Create `tests/data/c/corpus/`. +- [ ] Create `tests/data/c/scientific/`. +- [ ] Create `tests/parser/c/fixtures/general/`. +- [ ] Create `tests/parser/c/fixtures/errors/`. +- [ ] Keep C fixture data separate from Fortran fixture data. +- [ ] Add README files explaining each C fixture directory. +- [ ] Add small placeholder `.h` and `.c` fixture files only if tests need them. + +### Golden Workflow Tasks + +- [ ] Create `tests/parser/c/generate_c_parser_goldens.py`. +- [ ] Mirror the Fortran parser golden generator structure. +- [ ] Serialize only dataclass/JSON-stable C parse models. +- [ ] Strip parent/back-reference fields if future models need them. +- [ ] Support updating all fixtures. +- [ ] Support updating selected fixtures. +- [ ] Add an environment variable update flow, for example + `C_PARSER_UPDATE_GOLDENS=1`. +- [ ] Document whether C uses `C_PARSER_UPDATE_GOLDENS` or a generic + `X2PY_UPDATE_GOLDENS`. +- [ ] Create a C error golden generator. +- [ ] Store expected error type, message fragments, diagnostic fragments, and + parser entrypoint metadata. + +### Focused Test Buckets + +- [ ] Add lexer test file. +- [ ] Add preprocessor test file. +- [ ] Add declaration-specifier test file. +- [ ] Add declarator test file. +- [ ] Add function parser test file. +- [ ] Add struct/union/enum parser test file. +- [ ] Add typedef parser test file. +- [ ] Add macro/constant parser test file. +- [ ] Add project/include parser test file. +- [ ] Add readiness test file. +- [ ] Add public entrypoint test file. +- [ ] Add developer tutorial test file once internal helpers exist. +- [ ] Add CLI test file. +- [ ] Add fixture/golden test file. +- [ ] Add error fixture/golden test file. +- [ ] Add semantic conversion tests in Phase 10. +- [ ] Add `.pyi` tests in Phase 11. + +### Phase 2 Definition Of Done + +- [ ] C test directory structure is present. +- [ ] C fixture directory structure is present. +- [ ] C golden update workflow is documented. +- [ ] Placeholder tests pass against skeleton behavior. +- [ ] Fortran tests still pass. +- [ ] No real parser claims are made without tests. + +### Phase 2 Test Expectations + +- [ ] Run C skeleton CLI tests. +- [ ] Run existing parser CLI tests. +- [ ] Run a small targeted test command, for example + `python -m pytest -q tests/parser/test_cli.py tests/parser/test_c_cli_skeleton.py`. +- [ ] Do not update Fortran goldens. + +### Phase 2 Risks And Open Questions + +- [ ] Decide how much C fixture data is appropriate before parser behavior + exists. +- [ ] Decide whether C parser tests should live alongside Fortran parser tests + or under a new top-level C test package. + +## Phase 3: Parser Package Skeleton, Models, And Serialization Contracts + +Branch target: + +- `c-parser/phase-3-models` + +Scope: + +- Create the `c_parser` package. +- Define typed models. +- Define serialization helper contracts. +- Keep actual grammar parsing minimal or placeholder. + +### Package Skeleton Tasks + +- [ ] Create `c_parser/__init__.py`. +- [ ] Create `c_parser/__main__.py`. +- [ ] Create `c_parser/models.py`. +- [ ] Create `c_parser/parser.py`. +- [ ] Create `c_parser/lexer.py`. +- [ ] Create `c_parser/preprocessor.py`. +- [ ] Create `c_parser/type_resolver.py`. +- [ ] Create `c_parser/project.py`. +- [ ] Create `c_parser/cli.py`. +- [ ] Create `c_parser/utils.py`. +- [ ] Add `c_parser*` to package discovery in `pyproject.toml`. +- [ ] Add `c_parser` to coverage source when implementation begins. +- [ ] Keep imports from `x2py.cli` explicit and isolated. + +### Error Model Tasks + +- [ ] Implement `CParseError` as a `ValueError` subclass. +- [ ] Include `filename`. +- [ ] Include `line_number`. +- [ ] Include `column`. +- [ ] Include `source_line`. +- [ ] Include `base_message`. +- [ ] Include `code`. +- [ ] Include parser raise location for debug diagnostics. +- [ ] Implement `format_diagnostic(color=False, debug=None)`. +- [ ] Use C diagnostic code prefix such as `CPARSE001`. +- [ ] Add color handling equivalent to `FortranParseError`. +- [ ] Add optional C debug env var. +- [ ] Test C parse error attributes. +- [ ] Test compiler-style diagnostic rendering. +- [ ] Test color and no-color behavior. +- [ ] Test debug note behavior. + +### Model Tasks + +- [ ] Define `CSourceLocation`. +- [ ] Define `CDiagnostic`. +- [ ] Define `CTypeRef`. +- [ ] Define `CPointer`. +- [ ] Define `CArray`. +- [ ] Define `CParameter`. +- [ ] Define `CFunction`. +- [ ] Define `CField`. +- [ ] Define `CStruct`. +- [ ] Define `CUnion` or use `CStruct(is_union=True)`. +- [ ] Define `CEnum`. +- [ ] Define `CEnumerator`. +- [ ] Define `CTypedef`. +- [ ] Define `CGlobal`. +- [ ] Define `CMacro`. +- [ ] Define `CInclude`. +- [ ] Define `CFile`. +- [ ] Define `CProject`. +- [ ] Define `CWrapReadinessReport` only if a dataclass helps; otherwise use + stable dictionaries like Fortran readiness. +- [ ] Add helper properties for pointer depth. +- [ ] Add helper properties for array rank. +- [ ] Add helper properties for effective type text. +- [ ] Add helper properties for `is_const_pointer`. +- [ ] Add helper properties for `is_opaque_type`. +- [ ] Add helper properties for source-location display. + +### Serialization Tasks + +- [ ] Add `_to_dict` or equivalent serialization helper. +- [ ] Avoid cycles in JSON. +- [ ] Keep source locations JSON-serializable. +- [ ] Decide whether sets serialize as sorted lists. +- [ ] Ensure dataclass defaults produce stable JSON. +- [ ] Add tests for empty `CFile` serialization. +- [ ] Add tests for each model's minimal JSON shape. +- [ ] Add tests for source-location serialization. +- [ ] Add tests that unknown/unresolved metadata is preserved. + +### Public API Skeleton Tasks + +- [ ] Implement `CParser` class. +- [ ] Implement `CParser.visit_file` returning skeleton or model-only `CFile`. +- [ ] Implement `CParser.visit_project` returning skeleton/model-only + `CProject`. +- [ ] Implement `CParser.visit_wrap_readiness`. +- [ ] Implement module-level `_DEFAULT_PARSER`. +- [ ] Implement `parse_c_file`. +- [ ] Implement `parse_c_project`. +- [ ] Implement `assess_c_wrap_readiness`. +- [ ] Add public API tests for source strings. +- [ ] Add public API tests for file paths. +- [ ] Add public API tests for empty source. +- [ ] Add public API tests for unknown suffix. + +### Phase 3 Definition Of Done + +- [ ] `c_parser` imports cleanly. +- [ ] Skeleton public APIs return typed models. +- [ ] JSON serialization is stable and tested. +- [ ] C CLI uses `c_parser` rather than a temporary inline provider. +- [ ] Fortran parser API remains unchanged. +- [ ] Docs describe the new package and API status. + +### Phase 3 Risks And Open Questions + +- [ ] Decide whether `CUnion` should subclass/share `CStruct`. +- [ ] Decide how to represent anonymous structs/unions/enums. +- [ ] Decide whether macros belong in `CFile.macros` only or also symbols. +- [ ] Decide whether `CTypeRef` should be a single recursive model or contain + normalized pointer/array/function layers. + +## Phase 4: Lexer And Lightweight Preprocessor + +Branch target: + +- `c-parser/phase-4-lexer` + +Scope: + +- Token/source normalization. +- Comments, continuations, directives, includes, simple macro metadata. +- No full macro expansion. + +### Lexer Tasks + +- [ ] Preserve original line numbers for all logical records. +- [ ] Preserve original source lines for diagnostics. +- [ ] Remove block comments `/* ... */` without losing line accounting. +- [ ] Remove line comments `// ...`. +- [ ] Avoid stripping comment markers inside string literals. +- [ ] Avoid stripping comment markers inside character literals. +- [ ] Handle escaped quotes inside literals. +- [ ] Fold backslash-newline continuations. +- [ ] Preserve preprocessor directive line locations. +- [ ] Produce token records or logical line records with filename, line, column, + and text. +- [ ] Track braces, parentheses, and brackets. +- [ ] Add top-level split helpers aware of nesting and literals. +- [ ] Add tests for comment stripping. +- [ ] Add tests for multiline block comments. +- [ ] Add tests for string literal comment markers. +- [ ] Add tests for char literal escapes. +- [ ] Add tests for backslash-newline continuations. +- [ ] Add tests for line/column preservation. + +### Preprocessor Metadata Tasks + +- [ ] Recognize `#include "local.h"`. +- [ ] Recognize `#include `. +- [ ] Store include spelling and include kind. +- [ ] Resolve local includes relative to current file when possible. +- [ ] Preserve unresolved includes as diagnostics, not hard errors by default. +- [ ] Recognize object-like `#define NAME value`. +- [ ] Recognize function-like `#define NAME(...) body`. +- [ ] Store function-like macros as unsupported/deferred metadata. +- [ ] Recognize `#undef`. +- [ ] Track `#ifdef`. +- [ ] Track `#ifndef`. +- [ ] Track `#if`. +- [ ] Track `#elif`. +- [ ] Track `#else`. +- [ ] Track `#endif`. +- [ ] Add branch condition sets to parsed external declarations. +- [ ] Support optional `macro_defines` for active-branch selection. +- [ ] Implement a tiny safe evaluator for simple `defined(NAME)`, `&&`, `||`, + `!`, `0`, and `1`. +- [ ] Do not implement general macro expansion. +- [ ] Do not expand token-paste or stringify macros. +- [ ] Add tests for include collection. +- [ ] Add tests for object-like macro collection. +- [ ] Add tests for function-like macro diagnostics. +- [ ] Add tests for conditional branch tracking. +- [ ] Add tests for selected active branches. +- [ ] Add tests for duplicate declarations in mutually exclusive branches. + +### Phase 4 Definition Of Done + +- [ ] Lexer/preprocessor preserves source locations. +- [ ] Includes and macros are collected as metadata. +- [ ] Conditional branch tracking exists. +- [ ] No arbitrary macro expansion is attempted. +- [ ] Tests cover comments, continuations, directives, and branch selection. + +### Phase 4 Risks And Open Questions + +- [ ] Decide whether to tokenize fully now or keep logical records until + declarator parsing requires tokens. +- [ ] Decide whether system headers are recorded only or optionally searched. +- [ ] Decide whether `#pragma` should become diagnostics or metadata. + +## Phase 5: Declarations And Declarators + +Branch target: + +- `c-parser/phase-5-declarations` + +Scope: + +- Shared C declaration parser. +- Declarator model and type construction. +- No broad function/project behavior beyond declarations. + +### Declaration Specifier Tasks + +- [ ] Parse storage class `typedef`. +- [ ] Parse storage class `extern`. +- [ ] Parse storage class `static`. +- [ ] Parse storage class `register`. +- [ ] Parse storage class `_Thread_local`. +- [ ] Parse qualifier `const`. +- [ ] Parse qualifier `restrict`. +- [ ] Parse qualifier `volatile`. +- [ ] Parse qualifier `_Atomic` as basic metadata. +- [ ] Parse `void`. +- [ ] Parse `char`. +- [ ] Parse `signed char`. +- [ ] Parse `unsigned char`. +- [ ] Parse `short`. +- [ ] Parse `short int`. +- [ ] Parse `unsigned short`. +- [ ] Parse `int`. +- [ ] Parse `unsigned`. +- [ ] Parse `unsigned int`. +- [ ] Parse `long`. +- [ ] Parse `long int`. +- [ ] Parse `unsigned long`. +- [ ] Parse `long long`. +- [ ] Parse `unsigned long long`. +- [ ] Parse `float`. +- [ ] Parse `double`. +- [ ] Parse `long double`. +- [ ] Parse `_Bool`. +- [ ] Parse `_Complex` as deferred or supported with explicit tests. +- [ ] Parse `struct name`. +- [ ] Parse `union name`. +- [ ] Parse `enum name`. +- [ ] Parse typedef-name references. +- [ ] Preserve original declaration specifier text. +- [ ] Diagnose unknown specifier sequences. + +### Declarator Tasks + +- [ ] Parse identifier declarators. +- [ ] Parse pointer declarators. +- [ ] Parse pointer qualifiers. +- [ ] Parse array declarators. +- [ ] Parse multidimensional array declarators. +- [ ] Parse static array parameter qualifiers, for example `int a[static 4]`. +- [ ] Parse parenthesized declarators. +- [ ] Parse function declarators. +- [ ] Parse function pointer declarators. +- [ ] Parse abstract declarators where needed for unnamed parameters. +- [ ] Parse multiple declarators in one declaration. +- [ ] Keep declarator entity order stable. +- [ ] Preserve original declarator source text. +- [ ] Add source locations for each declared entity. +- [ ] Reject or diagnose unsupported declarator forms explicitly. + +### Shared Declaration Backend Tasks + +- [ ] Implement a helper analogous to `_helper_parse_declaration_line`. +- [ ] Feed procedure parameters through the same declaration backend. +- [ ] Feed function return types through the same declaration backend. +- [ ] Feed struct/union fields through the same declaration backend. +- [ ] Feed typedefs through the same declaration backend. +- [ ] Feed global variables/constants through the same declaration backend. +- [ ] Apply declaration specifiers to declarator-derived type layers. +- [ ] Normalize C type spelling into `CTypeRef`. +- [ ] Preserve typedef references before project resolution. +- [ ] Add tests for each declaration role. +- [ ] Add tests for declarations with multiple variables. +- [ ] Add tests for declarations with initializers. +- [ ] Add tests that local executable statements are not parsed as declarations. + +### Phase 5 Definition Of Done + +- [ ] Shared declaration/declarator parser exists. +- [ ] It is used by all declaration roles available so far. +- [ ] Primitive, pointer, array, typedef-name, and tag references have tests. +- [ ] Unsupported declaration-shaped input raises `CParseError` or structured + diagnostics. + +### Phase 5 Risks And Open Questions + +- [ ] Function pointer parsing is complex; decide which forms are supported in + extraction and which only produce diagnostics. +- [ ] Typedef-name recognition may require project/type context; decide how to + represent unresolved names before Phase 8. + +## Phase 6: Function Parsing + +Branch target: + +- `c-parser/phase-6-functions` + +Scope: + +- Function prototypes and definitions. +- Signature extraction. +- Function body skipping by balanced brace slicing. + +### Function Prototype Tasks + +- [ ] Classify top-level declarations ending with `;` as possible prototypes. +- [ ] Parse return type through declaration/declarator backend. +- [ ] Parse function name. +- [ ] Parse ordered parameter list. +- [ ] Preserve parameter names. +- [ ] Preserve unnamed parameter types when legal. +- [ ] Parse `void` parameter list as zero parameters. +- [ ] Parse variadic marker `...`. +- [ ] Mark `is_variadic`. +- [ ] Parse pointer parameters. +- [ ] Parse array parameters. +- [ ] Parse function pointer parameters. +- [ ] Parse `const` parameters. +- [ ] Parse `restrict` parameters. +- [ ] Parse `volatile` parameters. +- [ ] Parse storage class `extern`. +- [ ] Parse storage class `static`. +- [ ] Add source locations. +- [ ] Add tests for simple prototypes. +- [ ] Add tests for no-argument prototypes. +- [ ] Add tests for `void` arguments. +- [ ] Add tests for pointer and array parameters. +- [ ] Add tests for const pointer variants. +- [ ] Add tests for variadic prototypes. +- [ ] Add tests for function pointer parameters. + +### Function Definition Tasks + +- [ ] Classify top-level declarator followed by `{` as function definition. +- [ ] Parse signature from the definition header. +- [ ] Preserve `is_definition=True`. +- [ ] Preserve body source span. +- [ ] Skip body contents for wrapper metadata. +- [ ] Balance braces while respecting strings, chars, and comments. +- [ ] Ignore local declarations for exported signatures in v1. +- [ ] Reject or diagnose K&R style function definitions initially. +- [ ] Add tests for simple definitions. +- [ ] Add tests for nested braces in function body. +- [ ] Add tests for strings containing braces. +- [ ] Add tests for K&R unsupported diagnostics. + +### Function Deduplication Tasks + +- [ ] Merge matching prototype and definition in the same file. +- [ ] Prefer definition metadata where useful. +- [ ] Preserve both source locations if helpful. +- [ ] Detect conflicting declarations. +- [ ] Detect duplicate definitions. +- [ ] Allow same function under mutually exclusive preprocessor branches. +- [ ] Add tests for prototype plus definition. +- [ ] Add tests for conflicting prototypes. +- [ ] Add tests for duplicate definitions. + +### Phase 6 Definition Of Done + +- [ ] Basic C function signatures parse from `.h` and `.c`. +- [ ] Function bodies are skipped safely. +- [ ] Variadic and function pointer cases are represented and diagnosed. +- [ ] CLI human and JSON output show functions. +- [ ] Readiness reports no-functions only when appropriate. + +### Phase 6 Risks And Open Questions + +- [ ] Decide whether inline functions in headers are definitions or prototypes + for wrapper purposes. +- [ ] Decide how to handle attributes in function declarations before full + extension support exists. + +## Phase 7: Structs, Unions, Enums, And Typedefs + +Branch target: + +- `c-parser/phase-7-structs-enums` + +Scope: + +- C composite types and typedef aliases. + +### Struct Tasks + +- [ ] Parse named `struct name { ... };`. +- [ ] Parse forward declaration `struct name;`. +- [ ] Parse anonymous `struct { ... }`. +- [ ] Parse typedef anonymous struct `typedef struct { ... } name;`. +- [ ] Parse typedef named struct `typedef struct tag name;`. +- [ ] Parse fields with shared declaration backend. +- [ ] Parse pointer fields. +- [ ] Parse array fields. +- [ ] Parse nested anonymous structs as unsupported or metadata. +- [ ] Parse bitfields as metadata with readiness limitations. +- [ ] Preserve field order. +- [ ] Preserve source locations. +- [ ] Mark incomplete structs. +- [ ] Add tests for named structs. +- [ ] Add tests for forward declarations. +- [ ] Add tests for typedef structs. +- [ ] Add tests for pointer fields. +- [ ] Add tests for array fields. +- [ ] Add tests for bitfield diagnostics. + +### Union Tasks + +- [ ] Parse named unions. +- [ ] Parse forward union declarations. +- [ ] Parse anonymous unions. +- [ ] Parse typedef unions. +- [ ] Parse union fields with shared declaration backend. +- [ ] Mark union fields distinctly from struct fields. +- [ ] Add readiness diagnostics for by-value unions if unsafe. +- [ ] Add tests for named unions. +- [ ] Add tests for typedef unions. +- [ ] Add tests for union readiness. + +### Enum Tasks + +- [ ] Parse named enums. +- [ ] Parse anonymous enums. +- [ ] Parse typedef enums. +- [ ] Parse enumerator names. +- [ ] Parse explicit enumerator values. +- [ ] Preserve symbolic enumerator values. +- [ ] Safely fold simple integer expressions. +- [ ] Preserve expression text when folding is unsafe. +- [ ] Add tests for plain enums. +- [ ] Add tests for explicit values. +- [ ] Add tests for expression values. +- [ ] Add tests for typedef enums. + +### Typedef Tasks + +- [ ] Parse primitive typedefs. +- [ ] Parse pointer typedefs. +- [ ] Parse array typedefs. +- [ ] Parse function pointer typedefs. +- [ ] Parse struct/union/enum typedefs. +- [ ] Preserve alias chains before resolution. +- [ ] Detect duplicate typedefs in same scope. +- [ ] Add tests for typedef chains. +- [ ] Add tests for opaque handle typedefs. +- [ ] Add tests for function pointer typedef diagnostics. + +### Phase 7 Definition Of Done + +- [ ] C composite and typedef models are populated from basic fixtures. +- [ ] Shared declaration backend handles fields and typedefs. +- [ ] Incomplete/anonymous/bitfield cases are represented or diagnosed. +- [ ] JSON goldens cover composite type schema. +- [ ] Docs list supported and unsupported composite forms. + +### Phase 7 Risks And Open Questions + +- [ ] Decide when anonymous structs should become generated internal names. +- [ ] Decide how much enum expression folding is safe without compiler + semantics. +- [ ] Decide how unions map to semantic IR, if at all in v1. + +## Phase 8: Include And Project Resolution + +Branch target: + +- `c-parser/phase-8-project-resolution` + +Scope: + +- Multi-file project parsing. +- Include graph. +- Type and typedef resolution. + +### File Discovery Tasks + +- [ ] Discover `.c` files in C mode. +- [ ] Discover `.h` files in C mode. +- [ ] Decide whether `.i` is included now or later. +- [ ] Keep Fortran directory scanning unchanged. +- [ ] Support explicit file lists. +- [ ] Support directory recursion only in explicit C mode. +- [ ] Preserve deterministic file ordering. +- [ ] Add tests for file discovery. + +### Include Resolution Tasks + +- [ ] Resolve quoted includes relative to current file. +- [ ] Resolve quoted includes through `include_dirs`. +- [ ] Record unresolved quoted includes. +- [ ] Record system includes without requiring local resolution by default. +- [ ] Build `include_graph`. +- [ ] Detect include cycles without crashing. +- [ ] Preserve include spelling and resolved path separately. +- [ ] Add tests for local includes. +- [ ] Add tests for include dirs. +- [ ] Add tests for missing includes. +- [ ] Add tests for include cycles. + +### Project Index Tasks + +- [ ] Index functions by name and file. +- [ ] Index typedefs by name. +- [ ] Index struct tags by tag namespace. +- [ ] Index union tags by tag namespace. +- [ ] Index enum tags by tag namespace. +- [ ] Index enum constants in ordinary identifier namespace. +- [ ] Index macros/constants separately. +- [ ] Detect duplicate definitions. +- [ ] Distinguish compatible redeclarations from conflicts. +- [ ] Add tests for duplicate handling. + +### Type Resolution Tasks + +- [ ] Resolve typedef chains. +- [ ] Detect typedef cycles. +- [ ] Resolve struct tag references. +- [ ] Resolve union tag references. +- [ ] Resolve enum tag references. +- [ ] Resolve opaque pointer typedefs. +- [ ] Preserve unresolved references for readiness diagnostics. +- [ ] Do not lose original spelling during resolution. +- [ ] Add tests for cross-file typedef resolution. +- [ ] Add tests for cross-file struct resolution. +- [ ] Add tests for opaque handles. +- [ ] Add tests for unresolved references. + +### Header/Source Pairing Tasks + +- [ ] Pair `foo.c` with `foo.h` by basename. +- [ ] Pair source with headers it includes. +- [ ] Preserve many-to-many relationships. +- [ ] Use pairings for reporting, not for hidden behavior. +- [ ] Add tests for header/source pairing. + +### Phase 8 Definition Of Done + +- [ ] `parse_c_project` returns a populated `CProject`. +- [ ] Include graph is stable and serialized. +- [ ] Cross-file typedef/tag resolution works for basic projects. +- [ ] Missing project context becomes readiness diagnostics. +- [ ] Tests cover directory and file-list parsing. + +### Phase 8 Risks And Open Questions + +- [ ] Decide whether project parsing should parse all included system headers + when found. +- [ ] Decide how to handle generated headers. +- [ ] Decide whether include graph should be path-keyed, module-keyed, or both. + +## Phase 9: Wrap-Readiness Diagnostics + +Branch target: + +- `c-parser/phase-9-readiness` + +Scope: + +- Actionable readiness diagnostics. +- File-level and unit-level blockers. + +### Readiness Schema Tasks + +- [ ] Define stable C readiness dictionary keys. +- [ ] Include counts for functions, structs, unions, enums, typedefs, macros, + includes, and diagnostics. +- [ ] Include `unsupported_constructs`. +- [ ] Include `unresolved_includes`. +- [ ] Include `unresolved_typedefs`. +- [ ] Include `unresolved_tags`. +- [ ] Include `macro_dependent_declarations`. +- [ ] Include `unsupported_extensions`. +- [ ] Include `ambiguous_pointer_ownership`. +- [ ] Include `ambiguous_array_extents`. +- [ ] Include `callback_functions`. +- [ ] Include `variadic_functions`. +- [ ] Include `incomplete_public_types`. +- [ ] Include `wrappability_blockers`. +- [ ] Include `unit_blockers`. +- [ ] Include `why_not_wrappable`. +- [ ] Include file-level `wrappable`. + +### Diagnostic Tasks + +- [ ] Report no functions found. +- [ ] Report unresolved includes. +- [ ] Report unresolved typedefs. +- [ ] Report unresolved struct tags. +- [ ] Report unresolved union tags. +- [ ] Report unresolved enum tags. +- [ ] Report incomplete structs used by value. +- [ ] Report incomplete unions used by value. +- [ ] Report variadic functions. +- [ ] Report K&R functions. +- [ ] Report function pointer return types. +- [ ] Report function pointer parameters. +- [ ] Report callback typedef parameters. +- [ ] Report macro-dependent declarations. +- [ ] Report unsupported attributes. +- [ ] Report unsupported compiler extensions. +- [ ] Report unsupported bitfields. +- [ ] Report pointer ownership ambiguity. +- [ ] Report non-const pointer mutability ambiguity. +- [ ] Report arrays with unknown extent relationships. +- [ ] Report opaque pointers as warning or non-blocker if policy allows. + +### Unit Blocker Tasks + +- [ ] Assign function blockers to function units. +- [ ] Assign struct field blockers to struct units. +- [ ] Assign union field blockers to union units. +- [ ] Assign enum value blockers to enum units. +- [ ] Assign typedef blockers to typedef units. +- [ ] Assign include/macro/global blockers to file units when no narrower owner + exists. +- [ ] Use qualified names where available. +- [ ] Avoid per-unit ready flags; keep readiness file-level like Fortran. +- [ ] Add tests for every blocker family. +- [ ] Add CLI readiness formatting tests. + +### Phase 9 Definition Of Done + +- [ ] Readiness output is stable in JSON and human CLI. +- [ ] Diagnostics identify exactly which units block wrapping. +- [ ] Common unsupported C constructs produce actionable messages. +- [ ] Docs list readiness codes and examples. +- [ ] Tests cover readiness families. + +### Phase 9 Risks And Open Questions + +- [ ] Decide which pointer cases are blockers versus warnings. +- [ ] Decide how readiness should treat opaque handles. +- [ ] Decide whether callbacks are always blockers in v1. + +## Phase 10: Semantic IR Conversion + +Branch target: + +- `c-parser/phase-10-semantics` + +Scope: + +- Convert C parser models into language-independent semantic IR. + +### Converter Structure Tasks + +- [ ] Create `semantics/c2ir.py`. +- [ ] Implement `CToIRConverter`. +- [ ] Mirror the visitor style of `FortranToIRConverter`. +- [ ] Add compatibility helpers such as `c_file_to_semantic_modules`. +- [ ] Add `c_function_to_semantic_function`. +- [ ] Add `c_struct_to_semantic_class` where appropriate. +- [ ] Add `c_project_to_semantic_modules` if project context is needed. +- [ ] Keep conversion separate from parser internals. + +### Type Mapping Tasks + +- [ ] Map `void` return to `None`. +- [ ] Map `_Bool` to `Bool`. +- [ ] Map `char` to an explicit semantic type policy. +- [ ] Map signed integer widths. +- [ ] Map unsigned integer widths. +- [ ] Map `float` to `Float32`. +- [ ] Map `double` to `Float64`. +- [ ] Map `long double` to a documented type or unsupported diagnostic. +- [ ] Map pointers to constraints/metadata. +- [ ] Map arrays to `Shape` and `ORDER_C`. +- [ ] Map `const` to read-only/ownership metadata. +- [ ] Map `restrict` to aliasing metadata. +- [ ] Map structs to semantic classes or named semantic types. +- [ ] Map unions conservatively. +- [ ] Map enums/constants. +- [ ] Preserve unresolved semantic types as errors, not `Unknown` output. + +### Function Projection Tasks + +- [ ] Convert C functions to `SemanticFunction`. +- [ ] Preserve native function name. +- [ ] Preserve parameter order. +- [ ] Mark pointer mutability. +- [ ] Represent array pointer plus size patterns only when known. +- [ ] Add projection metadata only where native and Python signatures diverge. +- [ ] Treat out parameters conservatively until ownership/intent policy exists. +- [ ] Reject or defer variadic functions. +- [ ] Reject or defer callbacks. +- [ ] Add semantic tests for scalar functions. +- [ ] Add semantic tests for pointer input. +- [ ] Add semantic tests for const pointer input. +- [ ] Add semantic tests for arrays with explicit size parameter. +- [ ] Add semantic tests for structs/opaque handles. + +### Phase 10 Definition Of Done + +- [ ] C semantic conversion works for the supported parser subset. +- [ ] Unsupported C semantic mappings fail explicitly. +- [ ] `--language c --semantics` can be enabled with tests. +- [ ] Semantic fixture workflow exists for C if stable enough. +- [ ] Docs explain C to semantic IR mapping. + +### Phase 10 Risks And Open Questions + +- [ ] Current semantic IR may need richer pointer/ownership constraints. +- [ ] Unsigned integer semantic type names may need standardization. +- [ ] Struct/union/enum representation may require semantic model extensions. + +## Phase 11: `.pyi` Generation And Parsing Integration + +Branch target: + +- `c-parser/phase-11-pyi` + +Scope: + +- Emit/edit semantic interface stubs for C APIs. +- Extend `.pyi` syntax only through semantic IR needs. + +### Generation Tasks + +- [ ] Enable `--language c --pyi` only after semantic conversion is stable. +- [ ] Generate stubs from C semantic modules. +- [ ] Emit scalar functions. +- [ ] Emit pointer constraints when semantic model supports them. +- [ ] Emit arrays with `ORDER_C`. +- [ ] Emit constants as `Final[...]`. +- [ ] Emit opaque handles as classes or semantic type annotations. +- [ ] Emit structs as classes only when field semantics are intended. +- [ ] Avoid emitting `Unknown`. +- [ ] Emit imports/includes only if represented in semantic IR. +- [ ] Add tests for scalar function stubs. +- [ ] Add tests for constant stubs. +- [ ] Add tests for opaque handle stubs. +- [ ] Add tests for array stubs. + +### Parser Integration Tasks + +- [ ] Confirm existing `.pyi` parser accepts generated C stubs. +- [ ] Extend `.pyi` parser only if semantic IR requires new constructs. +- [ ] Add round-trip tests for C-generated stubs. +- [ ] Add edited-stub tests for C APIs. +- [ ] Add tests that unsupported C stubs fail clearly. +- [ ] Add fixture tests under `tests/pyi/fixtures/c/` if C stubs are stable. + +### Native Projection Tasks + +- [ ] Represent pointer/size hidden relationships where known. +- [ ] Represent returned output buffers only with explicit projection metadata. +- [ ] Represent ownership/lifetime metadata if supported by IR. +- [ ] Defer callback projection until function pointer semantics are designed. +- [ ] Defer arbitrary ABI details. + +### Phase 11 Definition Of Done + +- [ ] C `.pyi` output works for supported semantic subset. +- [ ] Generated stubs parse back into semantic IR. +- [ ] C `.pyi` tests are separate from Fortran `.pyi` tests. +- [ ] Docs describe generated C stub shape and limitations. + +### Phase 11 Risks And Open Questions + +- [ ] Existing `.pyi` syntax may not be expressive enough for ownership and + lifetimes. +- [ ] Opaque handles might require new conventions. +- [ ] C callbacks likely need a dedicated semantic design. + +## Phase 12: Corpus Testing, Stabilization, And Regression Hardening + +Branch target: + +- `c-parser/phase-12-corpus-stabilization` + +Scope: + +- Harden parser against realistic C APIs. +- Stabilize docs, tests, schema, and CLI. +- Prepare eventual mature integration. + +### Corpus Tasks + +- [ ] Add small real-world C header corpus. +- [ ] Add scientific C API fixtures. +- [ ] Add source/header project fixtures. +- [ ] Add macro-heavy unsupported fixtures. +- [ ] Add callback fixtures. +- [ ] Add variadic function fixtures. +- [ ] Add opaque handle fixtures. +- [ ] Add array-size pattern fixtures. +- [ ] Add typedef-chain fixtures. +- [ ] Add parse-only corpus tests. +- [ ] Add selected parser JSON goldens for representative corpus files. +- [ ] Keep corpus license provenance documented. + +### Regression Hardening Tasks + +- [ ] Run full parser tests. +- [ ] Run semantic tests. +- [ ] Run `.pyi` tests. +- [ ] Run C corpus parse-only tests. +- [ ] Run CLI tests. +- [ ] Run golden fixture tests. +- [ ] Confirm Fortran tests still pass. +- [ ] Audit JSON schema stability. +- [ ] Audit error diagnostic stability. +- [ ] Audit docs for implemented behavior. +- [ ] Remove stale skeleton wording where implementation has matured. +- [ ] Add developer tutorial for C parser internals. +- [ ] Add public API reference examples. + +### Stabilization Tasks + +- [ ] Decide criteria for merging `c-parser/main` into project `main`. +- [ ] Require green CI for Fortran and C suites. +- [ ] Require docs updated for implemented subset. +- [ ] Require fixture/golden workflow documented. +- [ ] Require semantic and `.pyi` behavior documented. +- [ ] Require explicit non-goals still documented. +- [ ] Require migration notes for users. +- [ ] Consider adding CI guard requiring C parser docs updates for C parser + changes, mirroring the Fortran parser reference policy. + +### Phase 12 Definition Of Done + +- [ ] C parser handles a representative stable subset. +- [ ] CLI, JSON, readiness, semantic IR, and `.pyi` workflows are tested. +- [ ] Fixture/golden workflows are stable. +- [ ] Corpus tests catch regressions. +- [ ] Fortran behavior remains stable. +- [ ] `c-parser/main` is mature enough to consider a planned merge to project + `main`. + +### Phase 12 Risks And Open Questions + +- [ ] Scope creep toward compiler-grade parsing. +- [ ] Macro-heavy APIs may exceed lightweight preprocessing. +- [ ] Ownership/lifetime semantics may need broader IR work. +- [ ] C extensions may need fixture-driven prioritization. + +## Cross-Phase Non-Goals For V1 + +- [ ] Do not support full compiler-grade C parsing. +- [ ] Do not support full C preprocessor compatibility. +- [ ] Do not support arbitrary macro expansion. +- [ ] Do not support token-paste/stringify expansion. +- [ ] Do not support all compiler extensions. +- [ ] Do not support arbitrary GCC extensions. +- [ ] Do not support arbitrary MSVC extensions. +- [ ] Do not parse C++. +- [ ] Do not generate a full ABI model. +- [ ] Do not infer pointer ownership silently. +- [ ] Do not claim callbacks are safely wrappable before a callback design + exists. +- [ ] Do not modify Fortran parser behavior as part of C parser work unless a + shared change is explicitly planned, tested, and documented. diff --git a/docs/c_parser/c_parser_reference.md b/docs/c_parser/c_parser_reference.md new file mode 100644 index 000000000..8d5baca03 --- /dev/null +++ b/docs/c_parser/c_parser_reference.md @@ -0,0 +1,261 @@ +# C Parser Reference + +Status: planning reference. The C parser is not implemented yet. + +This document is the future home for the C parser user and developer reference. +It should evolve into the C equivalent of `fortran_parser.md` as implementation +lands on `c-parser/main`. + +## Purpose + +The C parser frontend will be a wrapper-oriented source extraction system for +x2py. It should extract enough stable semantic information from C sources and +headers to help create or update the semantic interface layer. + +It is not intended to be: + +- a compiler-grade C frontend +- a full C preprocessor +- a replacement for semantic `.pyi` interfaces +- a libclang-only wrapper +- a C++ parser +- a complete ABI generator + +## Planned Source Coverage + +Initial source forms: + +- `.c` +- `.h` + +Possible later source form: + +- `.i` preprocessed C input + +Project input should accept explicit files and directories. Directory scanning +must be explicit C mode at first to avoid changing Fortran CLI behavior. + +## Planned Supported C Subset + +The initial supported subset should focus on stable wrapper-relevant APIs: + +- function prototypes +- function definitions with extractable signatures +- primitive C scalar types +- pointers +- arrays in parameters and fields +- `const`, `restrict`, and `volatile` qualifiers +- `static` and `extern` storage classes where wrapper-relevant +- `struct` definitions +- `union` definitions +- `enum` definitions and enumerators +- `typedef` declarations +- simple global constants +- simple object-like numeric and string macros +- include dependency tracking +- cross-file typedef and tag resolution within parsed project files + +## Initial Unsupported Subset + +The initial C parser should explicitly report or defer: + +- full compiler-grade C parsing +- full preprocessor compatibility +- arbitrary macro expansion +- token pasting and stringification +- macro-generated declarations +- complex conditional compilation evaluation +- all compiler extensions +- arbitrary GCC extensions +- arbitrary MSVC extensions +- C++ parsing +- K&R style function definitions +- full ABI generation +- guaranteed struct layout computation +- full bitfield ABI interpretation +- inline assembly +- `_Generic` semantic evaluation +- complex `_Atomic` behavior +- arbitrary attributes before fixture-driven support exists + +## Planned Public API + +Target module-level entrypoints: + +```python +from c_parser import parse_c_file, parse_c_project, assess_c_wrap_readiness +``` + +Expected signatures: + +```python +parse_c_file( + source_or_path, + filename=None, + macro_defines=None, + include_dirs=None, + encoding="utf-8", +) + +parse_c_project( + files, + include_dirs=None, + macro_defines=None, + encoding="utf-8", +) + +assess_c_wrap_readiness( + code, + filename=None, + include_dirs=None, + macro_defines=None, +) +``` + +These should return typed parser models and dictionaries analogous to the +Fortran parser API. Re-export from `x2py` should wait until the API is tested +and documented. + +## Planned CLI Usage + +Initial explicit mode: + +```bash +x2py path/to/api.h --language c --parse +x2py path/to/api.h --language c --parse --json +x2py path/to/api.h --language c --parse --wrap-readiness +``` + +Optional alias: + +```bash +x2py path/to/api.h --parse-c +``` + +Auto-detection should come later, after the frontend is stable. + +## Planned JSON Output + +Per-file shape: + +```text +{ + "": { + "language": "c", + "parser_status": "implemented|partial|skeleton", + "functions": [], + "structs": [], + "unions": [], + "enums": [], + "typedefs": [], + "globals": [], + "macros": [], + "includes": [], + "diagnostics": [], + "wrap_readiness": {} + } +} +``` + +JSON compatibility rules: + +- prefer additive schema changes +- include source locations once parser models exist +- preserve unknown or unresolved information rather than dropping it silently +- keep model fields stable enough for golden fixture testing +- document every intentional schema break + +## Planned Readiness Diagnostics + +Readiness should answer whether the parsed C API is safe enough for x2py to +wrap automatically or semi-automatically. + +Expected readiness categories: + +- unsupported constructs +- parse errors +- unresolved includes +- unresolved typedefs +- unresolved struct/union/enum tags +- incomplete public types +- macro-dependent declarations +- variadic functions +- function pointers and callbacks +- pointer ownership ambiguity +- pointer mutability ambiguity +- array extent ambiguity +- unsupported compiler extensions +- no functions found + +The top-level readiness report should keep a file-level `wrappable` boolean +and unit-scoped blockers, following the Fortran parser pattern. + +## Planned Error Handling + +The parser should define `CParseError` with: + +- `filename` +- `line_number` +- `column` +- `source_line` +- `base_message` +- `code` +- internal parser raise location for debug mode +- `format_diagnostic(color=False, debug=False)` + +The CLI should print compiler-style diagnostics without tracebacks by default. + +## Planned Testing Workflow + +Test families should mirror the Fortran parser: + +- focused lexer tests +- declaration-specifier tests +- declarator parser tests +- function prototype tests +- function definition tests +- struct/union/enum tests +- typedef tests +- macro/constant tests +- include/project tests +- readiness tests +- CLI tests +- semantic conversion tests +- `.pyi` generation/parser tests +- fixture/golden parser tests +- error fixture/golden tests +- corpus parse-only tests + +Fixture layout should be separate from Fortran: + +```text +tests/data/c/ + general/ + errors/parser/ + corpus/ + scientific/ + +tests/parser/c/ + fixtures/ + errors/ + generate_c_parser_goldens.py +``` + +## Planned Documentation Set + +The C parser documentation lives under: + +```text +docs/c_parser/ +``` + +Current planning documents: + +- `c_parser_reference.md` +- `c_parser_architecture.md` +- `c_parser_cli_workflow.md` +- `c_parser_implementation_checklist.md` + +Future implementation should update these docs in the same change whenever C +parser behavior, public API, CLI output, fixture workflow, semantic conversion, +or `.pyi` output changes. From 923a99217d3fb312bdde47492477a08a5d4fc285 Mon Sep 17 00:00:00 2001 From: said Date: Wed, 20 May 2026 13:10:37 +0100 Subject: [PATCH 2/3] codex: mark c parser roadmap setup complete --- .../c_parser_implementation_checklist.md | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/c_parser/c_parser_implementation_checklist.md b/docs/c_parser/c_parser_implementation_checklist.md index bb5001185..caa026977 100644 --- a/docs/c_parser/c_parser_implementation_checklist.md +++ b/docs/c_parser/c_parser_implementation_checklist.md @@ -46,11 +46,11 @@ Scope: - [x] Create long-lived integration branch `c-parser/main`. - [x] Create planning branch `c-parser/phase-0-roadmap` from `c-parser/main`. -- [ ] Merge `c-parser/phase-0-roadmap` back into `c-parser/main`. -- [ ] Confirm `main` has no C parser planning commits unless intentionally +- [x] Merge `c-parser/phase-0-roadmap` back into `c-parser/main`. +- [x] Confirm `main` has no C parser planning commits unless intentionally merged later after stabilization. -- [ ] Record the branch strategy in C parser docs. -- [ ] Use `codex: ...` prefix for planning commit message. +- [x] Record the branch strategy in C parser docs. +- [x] Use `codex: ...` prefix for planning commit message. ### Repository Inspection Tasks @@ -113,15 +113,15 @@ Scope: - [x] Docs make CLI and documentation Phase 1 deliverables. - [x] Docs preserve the grammar-style parser requirement. - [x] Docs preserve project-main isolation. -- [ ] Planning branch is committed. -- [ ] Planning branch is merged into `c-parser/main`. +- [x] Planning branch is committed. +- [x] Planning branch is merged into `c-parser/main`. ### Phase 0 Test Expectations -- [ ] No test changes are required in Phase 0. -- [ ] Run a documentation-safe sanity command such as `git status`. -- [ ] Do not regenerate fixtures. -- [ ] Do not run parser golden update scripts. +- [x] No test changes are required in Phase 0. +- [x] Run a documentation-safe sanity command such as `git status`. +- [x] Do not regenerate fixtures. +- [x] Do not run parser golden update scripts. ### Phase 0 Risks And Open Questions From b612cb0db5947cb1909c8f584e9ab3846679ad31 Mon Sep 17 00:00:00 2001 From: said Date: Wed, 20 May 2026 13:29:14 +0100 Subject: [PATCH 3/3] codex: clarify c parser preprocessing and callbacks --- docs/c_parser/c_parser_architecture.md | 69 ++++++++++++++++++- .../c_parser_implementation_checklist.md | 63 ++++++++++++++--- docs/c_parser/c_parser_reference.md | 36 ++++++++++ 3 files changed, 157 insertions(+), 11 deletions(-) diff --git a/docs/c_parser/c_parser_architecture.md b/docs/c_parser/c_parser_architecture.md index f666f20ff..1c59eea18 100644 --- a/docs/c_parser/c_parser_architecture.md +++ b/docs/c_parser/c_parser_architecture.md @@ -345,7 +345,14 @@ This is the C equivalent of the Fortran parser's shared declaration backend. The C frontend must be preprocessor-aware without trying to be a full C preprocessor in v1. -Initial target: +The practical rule is: x2py should not own full preprocessor correctness. +Macro-heavy APIs are still in scope, but the supported path for those APIs is +compiler-assisted preprocessing. The parser should support both raw-source +mode and a later preprocessed-input mode, and should store the facts it learns +from either mode in C parser models before any semantic IR conversion is +attempted. + +Raw-source mode target: - Strip comments safely while preserving line numbers. - Fold backslash-newline continuations. @@ -356,11 +363,30 @@ Initial target: similarly to the Fortran duplicate-check branch tracking. - Allow optional `macro_defines` to select active branches. - Preserve inactive branch diagnostics when macro selection is not requested. +- Parse ordinary declarations visible without macro expansion. +- Mark declaration regions that depend on unresolved macros. + +Compiler-assisted preprocessing target: + +- Accept a preprocessed stream from `.i` files or a configured compiler command + such as `cc -E` or `clang -E`. +- Preserve `#line` marker information so diagnostics can map preprocessed + declarations back to original files. +- Store both the original input path and the preprocessed origin metadata on + parsed models. +- Mark declarations discovered only after preprocessing with + `origin="preprocessed"` or equivalent model metadata. +- Record the macro definitions/include directories/preprocessor command that + produced the parse, because different `-D` and `-I` settings can expose + different public APIs. +- Treat function-like macros as metadata in raw mode, but allow their expanded + declarations to be parsed when they appear in compiler-preprocessed input. Initial non-goal: - Do not implement arbitrary macro expansion. - Do not attempt token-paste/stringify semantics. +- Do not implement recursive compiler-compatible macro semantics inside x2py. - Do not require libclang as the only way to understand headers. ## Project Parsing Strategy @@ -401,7 +427,7 @@ Planned blocker families: - unresolved tag type - incomplete struct/union used by value - function pointer parameter or return -- callback parameter requiring manual projection +- callback/function-pointer API without user-supplied `.pyi` policy - variadic function - K&R style function definition - array parameter with unknown size relationship @@ -431,6 +457,20 @@ The JSON readiness shape should stay close to Fortran: } ``` +## Parser-First Model Policy + +For the current planning horizon, the C parser should store C-specific facts in +`c_parser/models.py`. This includes preprocessing origin, macro dependencies, +function pointer signatures, callback-like parameters, pointer qualifiers, +ownership ambiguity, include dependencies, typedef resolution state, and +readiness diagnostics. + +Semantic IR conversion is deliberately later work. When that phase starts, the +IR model may need extensions for C pointer ownership, unsigned integer types, +callbacks, opaque handles, function pointer policies, and preprocessing-origin +metadata. The parser should not wait for those IR decisions before preserving +the source facts it can extract. + ## Semantic IR Mapping The semantic layer is the source of truth. The C parser should only help create @@ -476,7 +516,8 @@ Likely stub patterns: - structs: - `class struct_name: ...` when field layout is useful and stable - callbacks: - - defer until function pointer semantics are explicit + - generated stubs should include callback declarations only after user policy + fields are designed and supported - constants: - `Final[...]` @@ -485,6 +526,28 @@ imports, classes, functions, shapes, and native projection entries. C-specific work should extend the semantic model intentionally before changing `.pyi` syntax. +For function pointers and callbacks, parser extraction and wrap-readiness are +separate decisions. The parser should extract the function pointer type into C +models whenever possible. A callback-bearing API should become wrap-ready only +when the user supplies enough `.pyi` policy for wrapper generation. The policy +needs to identify: + +- the callback signature, including argument and return semantic types +- whether native calls Python, Python passes a callback to native, or both +- whether the callback is used only during the call or stored by native code +- which context/userdata parameter, often `void *`, is paired with it +- whether `NULL` is allowed for the callback and/or context +- the calling convention when it is not the platform default +- whether invocation is synchronous, asynchronous, same-thread, or arbitrary + native-thread +- who owns callback and context memory +- how and when a stored callback is released +- what should happen if the Python callback raises + +Until those fields exist and are supplied, readiness should report an explicit +callback policy blocker rather than pretending the function is safely +wrappable. + ## Isolation Policy All C parser work must be isolated from project `main` until the C frontend is diff --git a/docs/c_parser/c_parser_implementation_checklist.md b/docs/c_parser/c_parser_implementation_checklist.md index caa026977..c0561690c 100644 --- a/docs/c_parser/c_parser_implementation_checklist.md +++ b/docs/c_parser/c_parser_implementation_checklist.md @@ -463,7 +463,9 @@ Scope: - Token/source normalization. - Comments, continuations, directives, includes, simple macro metadata. -- No full macro expansion. +- No internal full macro expansion. +- Raw-source parsing first; compiler-assisted preprocessing path planned for + macro-heavy APIs. ### Lexer Tasks @@ -508,8 +510,15 @@ Scope: - [ ] Support optional `macro_defines` for active-branch selection. - [ ] Implement a tiny safe evaluator for simple `defined(NAME)`, `&&`, `||`, `!`, `0`, and `1`. +- [ ] Mark declarations that depend on unresolved macros. +- [ ] Store macro-dependency metadata in C parser models. +- [ ] Store preprocessing mode metadata in `CFile`. +- [ ] Store preprocessor configuration metadata such as macro defines and + include dirs. - [ ] Do not implement general macro expansion. - [ ] Do not expand token-paste or stringify macros. +- [ ] Do not attempt recursive compiler-compatible macro expansion inside + x2py. - [ ] Add tests for include collection. - [ ] Add tests for object-like macro collection. - [ ] Add tests for function-like macro diagnostics. @@ -517,12 +526,28 @@ Scope: - [ ] Add tests for selected active branches. - [ ] Add tests for duplicate declarations in mutually exclusive branches. +### Compiler-Assisted Preprocessing Tasks + +- [ ] Design a preprocessed-input mode for `.i` files. +- [ ] Design an optional compiler invocation mode for `cc -E` or `clang -E`. +- [ ] Preserve `#line` markers from compiler-preprocessed input. +- [ ] Map diagnostics from preprocessed declarations back to original files. +- [ ] Mark preprocessed declarations with origin metadata. +- [ ] Store the preprocessor command/configuration in `CFile` or `CProject`. +- [ ] Store original and preprocessed source paths when both exist. +- [ ] Add tests for parsing a simple `.i` file. +- [ ] Add tests for `#line` source mapping. +- [ ] Add tests that macro-generated declarations are parseable only when they + appear in preprocessed input. + ### Phase 4 Definition Of Done - [ ] Lexer/preprocessor preserves source locations. - [ ] Includes and macros are collected as metadata. - [ ] Conditional branch tracking exists. - [ ] No arbitrary macro expansion is attempted. +- [ ] Compiler-assisted preprocessing has a documented design path for + macro-heavy APIs. - [ ] Tests cover comments, continuations, directives, and branch selection. ### Phase 4 Risks And Open Questions @@ -531,6 +556,8 @@ Scope: declarator parsing requires tokens. - [ ] Decide whether system headers are recorded only or optionally searched. - [ ] Decide whether `#pragma` should become diagnostics or metadata. +- [ ] Decide whether compiler invocation belongs in Phase 4 or a later + project-resolution phase. ## Phase 5: Declarations And Declarators @@ -910,6 +937,7 @@ Scope: - [ ] Include `unresolved_typedefs`. - [ ] Include `unresolved_tags`. - [ ] Include `macro_dependent_declarations`. +- [ ] Include `preprocessing_origin`. - [ ] Include `unsupported_extensions`. - [ ] Include `ambiguous_pointer_ownership`. - [ ] Include `ambiguous_array_extents`. @@ -936,6 +964,7 @@ Scope: - [ ] Report function pointer return types. - [ ] Report function pointer parameters. - [ ] Report callback typedef parameters. +- [ ] Report missing callback `.pyi` policy. - [ ] Report macro-dependent declarations. - [ ] Report unsupported attributes. - [ ] Report unsupported compiler extensions. @@ -971,7 +1000,8 @@ Scope: - [ ] Decide which pointer cases are blockers versus warnings. - [ ] Decide how readiness should treat opaque handles. -- [ ] Decide whether callbacks are always blockers in v1. +- [ ] Decide the exact readiness code names for callback APIs that are parsed + but missing user-supplied `.pyi` policy. ## Phase 10: Semantic IR Conversion @@ -1023,7 +1053,10 @@ Scope: - [ ] Add projection metadata only where native and Python signatures diverge. - [ ] Treat out parameters conservatively until ownership/intent policy exists. - [ ] Reject or defer variadic functions. -- [ ] Reject or defer callbacks. +- [ ] Preserve callback/function-pointer facts from C parser models even if + semantic conversion defers wrapper generation. +- [ ] Defer callback conversion unless `.pyi` policy supplies the required + callback facts. - [ ] Add semantic tests for scalar functions. - [ ] Add semantic tests for pointer input. - [ ] Add semantic tests for const pointer input. @@ -1086,7 +1119,19 @@ Scope: - [ ] Represent pointer/size hidden relationships where known. - [ ] Represent returned output buffers only with explicit projection metadata. - [ ] Represent ownership/lifetime metadata if supported by IR. -- [ ] Defer callback projection until function pointer semantics are designed. +- [ ] Define `.pyi` policy fields for callback signatures. +- [ ] Define `.pyi` policy fields for callback direction. +- [ ] Define `.pyi` policy fields for call-only versus stored callback + lifetime. +- [ ] Define `.pyi` policy fields for context/userdata pairing. +- [ ] Define `.pyi` policy fields for callback nullability. +- [ ] Define `.pyi` policy fields for non-default calling conventions. +- [ ] Define `.pyi` policy fields for threading and async invocation. +- [ ] Define `.pyi` policy fields for ownership of callback/context memory. +- [ ] Define `.pyi` policy fields for release/unregistration APIs. +- [ ] Define `.pyi` policy fields for Python exception/error handling. +- [ ] Defer callback projection until those policy fields are supplied by the + user. - [ ] Defer arbitrary ABI details. ### Phase 11 Definition Of Done @@ -1101,7 +1146,8 @@ Scope: - [ ] Existing `.pyi` syntax may not be expressive enough for ownership and lifetimes. - [ ] Opaque handles might require new conventions. -- [ ] C callbacks likely need a dedicated semantic design. +- [ ] C callbacks likely need dedicated `.pyi` policy syntax and later semantic + IR extensions. ## Phase 12: Corpus Testing, Stabilization, And Regression Hardening @@ -1171,7 +1217,8 @@ Scope: ### Phase 12 Risks And Open Questions - [ ] Scope creep toward compiler-grade parsing. -- [ ] Macro-heavy APIs may exceed lightweight preprocessing. +- [ ] Macro-heavy APIs require clear raw-source versus compiler-preprocessed + mode behavior. - [ ] Ownership/lifetime semantics may need broader IR work. - [ ] C extensions may need fixture-driven prioritization. @@ -1187,7 +1234,7 @@ Scope: - [ ] Do not parse C++. - [ ] Do not generate a full ABI model. - [ ] Do not infer pointer ownership silently. -- [ ] Do not claim callbacks are safely wrappable before a callback design - exists. +- [ ] Do not claim callbacks are safely wrappable before the required `.pyi` + callback policy is supplied. - [ ] Do not modify Fortran parser behavior as part of C parser work unless a shared change is explicitly planned, tested, and documented. diff --git a/docs/c_parser/c_parser_reference.md b/docs/c_parser/c_parser_reference.md index 8d5baca03..b7772dafb 100644 --- a/docs/c_parser/c_parser_reference.md +++ b/docs/c_parser/c_parser_reference.md @@ -78,6 +78,25 @@ The initial C parser should explicitly report or defer: - complex `_Atomic` behavior - arbitrary attributes before fixture-driven support exists +## Planned Preprocessing Policy + +The C parser should have clear preprocessing modes instead of trying to become +a full C preprocessor. + +Raw-source mode should parse declarations that are visible without arbitrary +macro expansion, collect includes, collect simple object-like constants, track +conditional branches, and preserve macro-dependent declarations as parser model +metadata. + +Compiler-assisted preprocessing should be the practical path for macro-heavy +APIs. x2py may later accept `.i` files or invoke a configured compiler +preprocessor such as `cc -E` or `clang -E`. In that mode, the parser should +preserve `#line` mapping, record the preprocessor command/configuration, and +mark declarations that came from preprocessed input. + +This means macro-heavy APIs are not out of scope. The boundary is that x2py v1 +should not implement recursive, compiler-compatible macro expansion internally. + ## Planned Public API Target module-level entrypoints: @@ -190,6 +209,23 @@ Expected readiness categories: The top-level readiness report should keep a file-level `wrappable` boolean and unit-scoped blockers, following the Fortran parser pattern. +Function pointers and callbacks should be parsed into C models when possible. +They should not be marked wrap-ready until the user supplies enough `.pyi` +policy to explain how wrapper generation should handle them. + +The required user policy should make these facts explicit: + +- callback signature +- callback direction: native-to-Python, Python-to-native, or both +- lifetime: call-only, stored by native, or released by a specific API +- associated context/userdata parameter +- nullability rules +- non-default calling convention +- threading or async behavior +- ownership of callback and context memory +- release/unregistration API +- exception/error policy for Python callback failures + ## Planned Error Handling The parser should define `CParseError` with: