From 7e81f5258830d80d83c2e970f1464328aa2e7330 Mon Sep 17 00:00:00 2001 From: said Date: Sat, 23 May 2026 00:03:49 +0100 Subject: [PATCH] codex: add C function definition start and end locations --- c_parser/lexer.py | 41 ++++++++++++++----- c_parser/models.py | 2 + c_parser/parser.py | 10 +++++ docs/c_parser/c_parser_architecture.md | 32 +++++++++------ docs/c_parser/c_parser_cli_workflow.md | 14 +++++-- .../c_parser_implementation_checklist.md | 18 ++++---- docs/c_parser/c_parser_reference.md | 23 +++++++---- tests/parser/c/test_c_corpus.py | 2 +- tests/parser/c/test_c_functions.py | 10 ++--- 9 files changed, 103 insertions(+), 49 deletions(-) diff --git a/c_parser/lexer.py b/c_parser/lexer.py index 2b071f08b..6a3ecca7e 100644 --- a/c_parser/lexer.py +++ b/c_parser/lexer.py @@ -45,7 +45,9 @@ class CTopLevelSegment: original_start_line: int = 1 original_end_line: int = 1 original_start_column: int = 1 + original_end_column: int = 1 original_source_line: str | None = None + original_end_source_line: str | None = None _TWO_CHAR_OPERATORS = { @@ -192,6 +194,10 @@ def split_top_level_c_source( state = "normal" quote = "" escaped = False + block_header: str | None = None + block_start_line = 1 + block_start_column = 1 + block_source_line: str | None = None while i < len(stripped): char = stripped[i] @@ -241,23 +247,32 @@ def split_top_level_c_source( if start_index is not None: header = stripped[start_index:i].strip() if header: - segments.append( - CTopLevelSegment( - text=header, - terminator="block", - filename=filename, - original_start_line=start_line, - original_end_line=line, - original_start_column=start_column, - original_source_line=_source_line(source_lines, start_line), - ) - ) + block_header = header + block_start_line = start_line + block_start_column = start_column + block_source_line = _source_line(source_lines, start_line) brace_depth = 1 start_index = None elif char == "{" and brace_depth: brace_depth += 1 elif char == "}" and brace_depth: brace_depth -= 1 + if brace_depth == 0 and block_header: + segments.append( + CTopLevelSegment( + text=block_header, + terminator="block", + filename=filename, + original_start_line=block_start_line, + original_end_line=line, + original_start_column=block_start_column, + original_end_column=column, + original_source_line=block_source_line, + original_end_source_line=_source_line(source_lines, line), + ) + ) + block_header = None + block_source_line = None elif ( char == ";" and paren_depth == 0 @@ -275,7 +290,9 @@ def split_top_level_c_source( original_start_line=start_line, original_end_line=line, original_start_column=start_column, + original_end_column=column, original_source_line=_source_line(source_lines, start_line), + original_end_source_line=_source_line(source_lines, line), ) ) start_index = None @@ -294,7 +311,9 @@ def split_top_level_c_source( original_start_line=start_line, original_end_line=line, original_start_column=start_column, + original_end_column=column, original_source_line=_source_line(source_lines, start_line), + original_end_source_line=_source_line(source_lines, line), ) ) diff --git a/c_parser/models.py b/c_parser/models.py index a7913c44b..61fd6250d 100644 --- a/c_parser/models.py +++ b/c_parser/models.py @@ -225,6 +225,8 @@ class CFunction: is_definition: bool = False prototype_style: str | None = None source_location: CSourceLocation | None = None + start: CSourceLocation | None = None + end: CSourceLocation | None = None @dataclass diff --git a/c_parser/parser.py b/c_parser/parser.py index b871e1d86..faa22ca09 100644 --- a/c_parser/parser.py +++ b/c_parser/parser.py @@ -86,6 +86,14 @@ def _source_location(self, segment: CTopLevelSegment) -> CSourceLocation: source_line=segment.original_source_line, ) + def _end_location(self, segment: CTopLevelSegment) -> CSourceLocation: + return CSourceLocation( + filename=segment.filename, + line=segment.original_end_line, + column=segment.original_end_column, + source_line=segment.original_end_source_line, + ) + def _last_identifier(self, text: str) -> re.Match[str] | None: bracket_depth = 0 allowed_spans: list[tuple[int, int]] = [] @@ -414,6 +422,8 @@ def _parse_function(self, segment: CTopLevelSegment) -> CFunction | None: is_definition=segment.terminator == "block", prototype_style=self._prototype_style(parameters_text), source_location=self._source_location(segment), + start=self._source_location(segment), + end=self._end_location(segment) if segment.terminator == "block" else None, ) def _parse_declaration(self, segment: CTopLevelSegment) -> tuple[list[CTypedef], list[CGlobal]]: diff --git a/docs/c_parser/c_parser_architecture.md b/docs/c_parser/c_parser_architecture.md index 889acceb9..04b842d1f 100644 --- a/docs/c_parser/c_parser_architecture.md +++ b/docs/c_parser/c_parser_architecture.md @@ -4,7 +4,7 @@ Status: partial parser plus raw directive metadata implemented. The `c_parser` package, typed parser models, public entrypoints, explicit `x2py --language c --parse` CLI path, raw include/macro/undef metadata collection, top-level source splitting, and a first simple -declaration/function subset exist. +declaration/function subset with function-definition start/end locations exist. This document records the target architecture for the C parser frontend in x2py. The initial skeleton has grown into a partial parser, and the remaining @@ -25,14 +25,16 @@ Implemented now: public entrypoints and small path helpers. - `c_parser.lexer` strips comments safely, folds backslash-newline logical records, exposes lightweight token records, and provides top-level splitting - helpers that track braces, parentheses, brackets, and literals. + helpers that track braces, parentheses, brackets, literals, and + function-definition end locations. - `c_parser.preprocessor` records raw `#include` directives, simple object-like macros, `#undef` directives, and unsupported function-like macro diagnostics without expanding macros. - `c_parser.parser` parses simple globals, typedefs, function prototypes, and function-definition signatures while skipping bodies. Function models include - `prototype_style`, and K&R-style function definitions raise focused - diagnostics. + `prototype_style`; definitions preserve direct `start` and `end` locations + from the signature start through the closing brace; and K&R-style function + definitions raise focused diagnostics. - `c_parser.cli` provides C-specific partial report formatting. - `x2py.cli` dispatches `--language c --parse` to the C parser path. - `--language c --semantics`, `--language c --pyi`, and C wrap-readiness are @@ -185,8 +187,8 @@ Current and planned responsibilities: - `c_parser/lexer.py` - Implemented: safe comment removal that preserves line mapping, logical record folding for backslash-newline, string/character literal awareness, - lightweight tokens with source locations, top-level splitting, and - delimiter splitting aware of nesting and literals. + lightweight tokens with source locations, top-level splitting with block + end locations, and delimiter splitting aware of nesting and literals. - Planned: richer token helpers as recursive declarator parsing requires them. - `c_parser/preprocessor.py` @@ -201,8 +203,9 @@ Current and planned responsibilities: declaration-specifier handling, and simple pointer/array declarator extraction. Helper methods live on `CParser` rather than as broad module-level functions. Current function models record prototype-style - versus unspecified empty parameter lists, and K&R-style definitions are - rejected with `CParseError`. + versus unspecified empty parameter lists, function definitions preserve + start/end locations, and K&R-style definitions are rejected with + `CParseError`. - Planned: recursive declarator/function/composite-type visitors and a richer shared declaration/declarator backend. - `c_parser/project.py` @@ -298,7 +301,10 @@ Implemented parser models: - `specifiers` - `variadic` - `is_definition` + - `prototype_style` - `source_location` + - `start` + - `end` - `CField` - `name` - `type` @@ -366,10 +372,10 @@ Implemented parser models: - `macros` - `includes` -Future parser phases can add fields such as source spans, bit widths, function -pointer metadata, conditional-region metadata, include graphs, and project -diagnostics when the corresponding behavior lands. Additions should be -documented and tested with stable serialization expectations. +Future parser phases can add fields such as bit widths, function pointer +metadata, conditional-region metadata, include graphs, and project diagnostics +when the corresponding behavior lands. Additions should be documented and +tested with stable serialization expectations. ## Grammar-Style Parsing Strategy @@ -408,7 +414,7 @@ be: 7. Use a shared declaration-specifier and declarator parser to build type references for functions, parameters, fields, globals, and typedefs. 8. Ignore executable function bodies except where needed to find the matching - brace and preserve source spans. + brace and preserve function start/end locations. ## Declarator-Centered Design diff --git a/docs/c_parser/c_parser_cli_workflow.md b/docs/c_parser/c_parser_cli_workflow.md index af364c631..6b8866531 100644 --- a/docs/c_parser/c_parser_cli_workflow.md +++ b/docs/c_parser/c_parser_cli_workflow.md @@ -4,7 +4,7 @@ Status: C parser partial subset plus raw directive metadata implemented. The CLI command shape exists and parse reports can include raw includes, simple macros, `#undef` provenance, metadata diagnostics, simple globals, typedefs, function prototypes, prototype-style metadata, and function-definition -signatures. +signatures with start/end locations. The C parser CLI workflow should be designed before parser implementation so future parser work lands behind a stable command shape, output schema, and @@ -33,6 +33,9 @@ while composite type sections remain empty. Raw `includes`, `macros`, and metadata `diagnostics` can also be populated. The parser reports `parser_status: "partial"`. C parse diagnostics, currently including unsupported K&R-style function definitions, honor `--no-color` and `NO_COLOR=1`. +Function definitions do not store executable body text; they preserve a +direct `start` location and `end` location from the signature start through the +closing brace. Unsupported C stages: @@ -205,7 +208,9 @@ JSON output for a file without raw directives: "specifiers": [], "variadic": false, "is_definition": false, - "prototype_style": "prototype" + "prototype_style": "prototype", + "start": {"filename": "include/example.h", "line": 1, "...": "..."}, + "end": null } ], "structs": [], @@ -345,7 +350,8 @@ The active CLI/parser tests cover the current partial subset: - raw comment stripping, line-continuation folding, top-level splitting, include collection, simple macro collection, function-like macro diagnostics, conditional non-selection, simple declarations, globals, typedefs, and - function signatures are covered by focused C tests. + function signatures with definition start/end locations are covered by focused C + tests. - `--show-vars` and `--print-limit` are rejected in C mode until C-specific display controls exist. - `--semantics` with `--language c` is rejected until C semantic conversion is @@ -368,6 +374,8 @@ Completed order: macros. 7. Added top-level splitting and a first partial grammar subset for simple globals, typedefs, function prototypes, and function-definition headers. +8. Added function-definition start/end locations while continuing to skip + executable bodies. Next implementation work should continue with richer declarator support, preprocessed-input line mapping, composite types, and project resolution while diff --git a/docs/c_parser/c_parser_implementation_checklist.md b/docs/c_parser/c_parser_implementation_checklist.md index 47da7165b..671ac7f0d 100644 --- a/docs/c_parser/c_parser_implementation_checklist.md +++ b/docs/c_parser/c_parser_implementation_checklist.md @@ -4,8 +4,9 @@ Status: implementation checklist with Phase 1 skeleton, selected Phase 2 fixture scaffolding, selected Phase 3 model/error work, Phase 4 raw lexer/directive metadata, and a first Phase 5/6 partial declaration/function subset complete. The `c_parser` package and explicit C -parse path exist, and simple globals, typedefs, function prototypes, and -function-definition signatures are now parsed. +parse path exist, and simple globals, typedefs, function prototypes, +function-definition signatures, and function-definition start/end locations are now +parsed. This checklist is intentionally detailed so future work can proceed one branch, one checklist item, and one tested capability at a time. The C parser initiative @@ -15,10 +16,11 @@ stable. ## Progress Snapshot - Last updated: 2026-05-22 -- Checklist progress: 443/848 checked (52.2%). +- Checklist progress: 445/848 checked (52.5%). - Current parser status: partial C parser with raw directive metadata, top-level source splitting, simple declarations/globals/typedefs, prototype-style - metadata, K&R diagnostics, and simple function signatures. + metadata, K&R diagnostics, simple function signatures, and start/end + locations for function definitions. ## Global Rules @@ -792,10 +794,10 @@ Scope: - [x] Classify top-level declarator followed by `{` as function definition. - [x] Parse signature from the definition header. - [x] Preserve `is_definition=True`. -- [ ] Preserve body source span. -- [ ] Add `CSourceSpan` or equivalent start/end model before preserving body - spans. -- [x] Skip body contents for wrapper metadata. +- [x] Preserve function-definition start/end locations. +- [x] Add direct `start`/`end` model fields before preserving definition + ranges. +- [x] Skip function body contents for wrapper metadata. - [x] Balance braces while respecting strings, chars, and comments. - [x] Ignore local declarations for exported signatures in v1. - [x] Reject or diagnose K&R style function definitions initially. diff --git a/docs/c_parser/c_parser_reference.md b/docs/c_parser/c_parser_reference.md index b1c032d38..290a130e6 100644 --- a/docs/c_parser/c_parser_reference.md +++ b/docs/c_parser/c_parser_reference.md @@ -3,7 +3,8 @@ Status: partial parser reference with raw directive metadata. The `c_parser` package and explicit C CLI parse path exist, raw includes/simple macros are recorded, and a first grammar-shaped subset parses simple declarations, -typedefs, globals, function prototypes, and function-definition headers. +typedefs, globals, function prototypes, and function-definition headers with +start/end locations. This document is the future home for the C parser user and developer reference. It should evolve into the C equivalent of `fortran_parser.md` as implementation @@ -67,6 +68,8 @@ Implemented: - simple function prototype extraction - prototype-style metadata distinguishing `int f(void)` from `int f()` - simple function-definition signature extraction with body skipping +- start/end locations for function definitions, from the signature start + through the closing brace - unsupported K&R function-definition diagnostics - C fixture inputs under `tests/data/c/general/` plus C fixture directory scaffolding for errors, corpus, and scientific APIs @@ -201,8 +204,9 @@ current partial phase can populate `functions`, `typedefs`, `globals`, `structs`, `unions`, and `enums` remain empty until their dedicated parser phase lands. Functions include `prototype_style`, currently `"prototype"` for typed or explicit `void` parameter lists and `"unspecified"` for empty -parameter lists such as `int f()`. Re-export from `x2py` is still deferred; -users should import from `c_parser`. +parameter lists such as `int f()`. Function definitions do not store +executable body text; they include direct `start` and `end` locations. +Re-export from `x2py` is still deferred; users should import from `c_parser`. `macro_defines` is reserved for future compiler-assisted preprocessing configuration. It must not mean that raw mode evaluates C preprocessor @@ -252,7 +256,9 @@ Per-file shape: "variadic": false, "is_definition": false, "prototype_style": "prototype", - "source_location": {"filename": "", "line": 1, "...": "..."} + "source_location": {"filename": "", "line": 1, "...": "..."}, + "start": {"filename": "", "line": 1, "...": "..."}, + "end": null } ], "structs": [], @@ -350,10 +356,11 @@ public entrypoints, empty model serialization, CLI discovery, JSON/output-file behavior, unsupported C stages, comment stripping, line-continuation folding, top-level splitting, include collection, simple macro collection, macro-shaped declaration deferral, raw conditional branch non-selection, simple declarations, -globals, typedefs, and simple function prototypes/definitions. The broader -roadmap tests remain skipped until their matching implementation branches land. -Future implementation branches should unskip only the tests for the capability -they implement, then merge those branches back into `c-parser/main`. +globals, typedefs, and simple function prototypes/definitions, including +function-definition start/end locations. The broader roadmap tests remain skipped +until their matching implementation branches land. Future implementation +branches should unskip only the tests for the capability they implement, then +merge those branches back into `c-parser/main`. Fixture layout should be separate from Fortran: diff --git a/tests/parser/c/test_c_corpus.py b/tests/parser/c/test_c_corpus.py index f26afb9a2..b5cf78fa3 100644 --- a/tests/parser/c/test_c_corpus.py +++ b/tests/parser/c/test_c_corpus.py @@ -72,7 +72,7 @@ def test_cjson_source_file_parse_skips_function_bodies_safely(): parsed = parse_c_file(_CJSON_DIR / "cJSON.c") assert any(fn.name == "cJSON_Parse" for fn in parsed.functions) - assert all(fn.body is None for fn in parsed.functions) + assert not any(hasattr(fn, "body") for fn in parsed.functions) def test_cjson_project_parse_links_header_and_source(): diff --git a/tests/parser/c/test_c_functions.py b/tests/parser/c/test_c_functions.py index 2460eec8e..ff38b11b9 100644 --- a/tests/parser/c/test_c_functions.py +++ b/tests/parser/c/test_c_functions.py @@ -18,8 +18,7 @@ def test_function_prototypes_preserve_return_type_parameter_order_and_names(): assert [param.name for param in fn.parameters] == ["n", "x", "y"] -@pytest.mark.skip(reason="function body source spans are not modeled yet.") -def test_function_definitions_skip_bodies_but_preserve_source_span(): +def test_function_definitions_skip_bodies_but_preserve_start_and_end_locations(): from c_parser import parse_c_file parsed = parse_c_file( @@ -34,9 +33,10 @@ def test_function_definitions_skip_bodies_but_preserve_source_span(): fn = parsed.functions[0] assert fn.name == "add" - assert fn.body is None - assert fn.source_span.start.line == 2 - assert fn.source_span.end.line == 5 + assert fn.start is not None + assert fn.end is not None + assert fn.start.line == 2 + assert fn.end.line == 5 def test_void_parameter_list_and_empty_parameter_list_are_distinguished():