Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 30 additions & 11 deletions c_parser/lexer.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,9 @@ class CTopLevelSegment:
original_start_line: int = 1
original_end_line: int = 1
original_start_column: int = 1
original_end_column: int = 1
original_source_line: str | None = None
original_end_source_line: str | None = None


_TWO_CHAR_OPERATORS = {
Expand Down Expand Up @@ -192,6 +194,10 @@ def split_top_level_c_source(
state = "normal"
quote = ""
escaped = False
block_header: str | None = None
block_start_line = 1
block_start_column = 1
block_source_line: str | None = None

while i < len(stripped):
char = stripped[i]
Expand Down Expand Up @@ -241,23 +247,32 @@ def split_top_level_c_source(
if start_index is not None:
header = stripped[start_index:i].strip()
if header:
segments.append(
CTopLevelSegment(
text=header,
terminator="block",
filename=filename,
original_start_line=start_line,
original_end_line=line,
original_start_column=start_column,
original_source_line=_source_line(source_lines, start_line),
)
)
block_header = header
block_start_line = start_line
block_start_column = start_column
block_source_line = _source_line(source_lines, start_line)
brace_depth = 1
start_index = None
elif char == "{" and brace_depth:
brace_depth += 1
elif char == "}" and brace_depth:
brace_depth -= 1
if brace_depth == 0 and block_header:
segments.append(
CTopLevelSegment(
text=block_header,
terminator="block",
filename=filename,
original_start_line=block_start_line,
original_end_line=line,
original_start_column=block_start_column,
original_end_column=column,
original_source_line=block_source_line,
original_end_source_line=_source_line(source_lines, line),
)
)
block_header = None
block_source_line = None
elif (
char == ";"
and paren_depth == 0
Expand All @@ -275,7 +290,9 @@ def split_top_level_c_source(
original_start_line=start_line,
original_end_line=line,
original_start_column=start_column,
original_end_column=column,
original_source_line=_source_line(source_lines, start_line),
original_end_source_line=_source_line(source_lines, line),
)
)
start_index = None
Expand All @@ -294,7 +311,9 @@ def split_top_level_c_source(
original_start_line=start_line,
original_end_line=line,
original_start_column=start_column,
original_end_column=column,
original_source_line=_source_line(source_lines, start_line),
original_end_source_line=_source_line(source_lines, line),
)
)

Expand Down
2 changes: 2 additions & 0 deletions c_parser/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,8 @@ class CFunction:
is_definition: bool = False
prototype_style: str | None = None
source_location: CSourceLocation | None = None
start: CSourceLocation | None = None
end: CSourceLocation | None = None


@dataclass
Expand Down
10 changes: 10 additions & 0 deletions c_parser/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,14 @@ def _source_location(self, segment: CTopLevelSegment) -> CSourceLocation:
source_line=segment.original_source_line,
)

def _end_location(self, segment: CTopLevelSegment) -> CSourceLocation:
return CSourceLocation(
filename=segment.filename,
line=segment.original_end_line,
column=segment.original_end_column,
source_line=segment.original_end_source_line,
)

def _last_identifier(self, text: str) -> re.Match[str] | None:
bracket_depth = 0
allowed_spans: list[tuple[int, int]] = []
Expand Down Expand Up @@ -414,6 +422,8 @@ def _parse_function(self, segment: CTopLevelSegment) -> CFunction | None:
is_definition=segment.terminator == "block",
prototype_style=self._prototype_style(parameters_text),
source_location=self._source_location(segment),
start=self._source_location(segment),
end=self._end_location(segment) if segment.terminator == "block" else None,
)

def _parse_declaration(self, segment: CTopLevelSegment) -> tuple[list[CTypedef], list[CGlobal]]:
Expand Down
32 changes: 19 additions & 13 deletions docs/c_parser/c_parser_architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ Status: partial parser plus raw directive metadata implemented. The `c_parser`
package, typed parser models, public entrypoints, explicit
`x2py --language c --parse` CLI path, raw include/macro/undef metadata
collection, top-level source splitting, and a first simple
declaration/function subset exist.
declaration/function subset with function-definition start/end locations exist.

This document records the target architecture for the C parser frontend in
x2py. The initial skeleton has grown into a partial parser, and the remaining
Expand All @@ -25,14 +25,16 @@ Implemented now:
public entrypoints and small path helpers.
- `c_parser.lexer` strips comments safely, folds backslash-newline logical
records, exposes lightweight token records, and provides top-level splitting
helpers that track braces, parentheses, brackets, and literals.
helpers that track braces, parentheses, brackets, literals, and
function-definition end locations.
- `c_parser.preprocessor` records raw `#include` directives, simple object-like
macros, `#undef` directives, and unsupported function-like macro diagnostics
without expanding macros.
- `c_parser.parser` parses simple globals, typedefs, function prototypes, and
function-definition signatures while skipping bodies. Function models include
`prototype_style`, and K&R-style function definitions raise focused
diagnostics.
`prototype_style`; definitions preserve direct `start` and `end` locations
from the signature start through the closing brace; and K&R-style function
definitions raise focused diagnostics.
- `c_parser.cli` provides C-specific partial report formatting.
- `x2py.cli` dispatches `--language c --parse` to the C parser path.
- `--language c --semantics`, `--language c --pyi`, and C wrap-readiness are
Expand Down Expand Up @@ -185,8 +187,8 @@ Current and planned responsibilities:
- `c_parser/lexer.py`
- Implemented: safe comment removal that preserves line mapping, logical
record folding for backslash-newline, string/character literal awareness,
lightweight tokens with source locations, top-level splitting, and
delimiter splitting aware of nesting and literals.
lightweight tokens with source locations, top-level splitting with block
end locations, and delimiter splitting aware of nesting and literals.
- Planned: richer token helpers as recursive declarator parsing requires
them.
- `c_parser/preprocessor.py`
Expand All @@ -201,8 +203,9 @@ Current and planned responsibilities:
declaration-specifier handling, and simple pointer/array declarator
extraction. Helper methods live on `CParser` rather than as broad
module-level functions. Current function models record prototype-style
versus unspecified empty parameter lists, and K&R-style definitions are
rejected with `CParseError`.
versus unspecified empty parameter lists, function definitions preserve
start/end locations, and K&R-style definitions are rejected with
`CParseError`.
- Planned: recursive declarator/function/composite-type visitors and a
richer shared declaration/declarator backend.
- `c_parser/project.py`
Expand Down Expand Up @@ -298,7 +301,10 @@ Implemented parser models:
- `specifiers`
- `variadic`
- `is_definition`
- `prototype_style`
- `source_location`
- `start`
- `end`
- `CField`
- `name`
- `type`
Expand Down Expand Up @@ -366,10 +372,10 @@ Implemented parser models:
- `macros`
- `includes`

Future parser phases can add fields such as source spans, bit widths, function
pointer metadata, conditional-region metadata, include graphs, and project
diagnostics when the corresponding behavior lands. Additions should be
documented and tested with stable serialization expectations.
Future parser phases can add fields such as bit widths, function pointer
metadata, conditional-region metadata, include graphs, and project diagnostics
when the corresponding behavior lands. Additions should be documented and
tested with stable serialization expectations.

## Grammar-Style Parsing Strategy

Expand Down Expand Up @@ -408,7 +414,7 @@ be:
7. Use a shared declaration-specifier and declarator parser to build type
references for functions, parameters, fields, globals, and typedefs.
8. Ignore executable function bodies except where needed to find the matching
brace and preserve source spans.
brace and preserve function start/end locations.

## Declarator-Centered Design

Expand Down
14 changes: 11 additions & 3 deletions docs/c_parser/c_parser_cli_workflow.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ Status: C parser partial subset plus raw directive metadata implemented. The
CLI command shape exists and parse reports can include raw includes, simple
macros, `#undef` provenance, metadata diagnostics, simple globals, typedefs,
function prototypes, prototype-style metadata, and function-definition
signatures.
signatures with start/end locations.

The C parser CLI workflow should be designed before parser implementation so
future parser work lands behind a stable command shape, output schema, and
Expand Down Expand Up @@ -33,6 +33,9 @@ while composite type sections remain empty. Raw `includes`, `macros`, and
metadata `diagnostics` can also be populated. The parser reports
`parser_status: "partial"`. C parse diagnostics, currently including
unsupported K&R-style function definitions, honor `--no-color` and `NO_COLOR=1`.
Function definitions do not store executable body text; they preserve a
direct `start` location and `end` location from the signature start through the
closing brace.

Unsupported C stages:

Expand Down Expand Up @@ -205,7 +208,9 @@ JSON output for a file without raw directives:
"specifiers": [],
"variadic": false,
"is_definition": false,
"prototype_style": "prototype"
"prototype_style": "prototype",
"start": {"filename": "include/example.h", "line": 1, "...": "..."},
"end": null
}
],
"structs": [],
Expand Down Expand Up @@ -345,7 +350,8 @@ The active CLI/parser tests cover the current partial subset:
- raw comment stripping, line-continuation folding, top-level splitting,
include collection, simple macro collection, function-like macro diagnostics,
conditional non-selection, simple declarations, globals, typedefs, and
function signatures are covered by focused C tests.
function signatures with definition start/end locations are covered by focused C
tests.
- `--show-vars` and `--print-limit` are rejected in C mode until C-specific
display controls exist.
- `--semantics` with `--language c` is rejected until C semantic conversion is
Expand All @@ -368,6 +374,8 @@ Completed order:
macros.
7. Added top-level splitting and a first partial grammar subset for simple
globals, typedefs, function prototypes, and function-definition headers.
8. Added function-definition start/end locations while continuing to skip
executable bodies.

Next implementation work should continue with richer declarator support,
preprocessed-input line mapping, composite types, and project resolution while
Expand Down
18 changes: 10 additions & 8 deletions docs/c_parser/c_parser_implementation_checklist.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@ Status: implementation checklist with Phase 1 skeleton, selected Phase 2
fixture scaffolding, selected Phase 3 model/error work, Phase 4 raw
lexer/directive metadata, and a first Phase 5/6 partial
declaration/function subset complete. The `c_parser` package and explicit C
parse path exist, and simple globals, typedefs, function prototypes, and
function-definition signatures are now parsed.
parse path exist, and simple globals, typedefs, function prototypes,
function-definition signatures, and function-definition start/end locations are now
parsed.

This checklist is intentionally detailed so future work can proceed one branch,
one checklist item, and one tested capability at a time. The C parser initiative
Expand All @@ -15,10 +16,11 @@ stable.
## Progress Snapshot

- Last updated: 2026-05-22
- Checklist progress: 443/848 checked (52.2%).
- Checklist progress: 445/848 checked (52.5%).
- Current parser status: partial C parser with raw directive metadata, top-level
source splitting, simple declarations/globals/typedefs, prototype-style
metadata, K&R diagnostics, and simple function signatures.
metadata, K&R diagnostics, simple function signatures, and start/end
locations for function definitions.

## Global Rules

Expand Down Expand Up @@ -792,10 +794,10 @@ Scope:
- [x] Classify top-level declarator followed by `{` as function definition.
- [x] Parse signature from the definition header.
- [x] Preserve `is_definition=True`.
- [ ] Preserve body source span.
- [ ] Add `CSourceSpan` or equivalent start/end model before preserving body
spans.
- [x] Skip body contents for wrapper metadata.
- [x] Preserve function-definition start/end locations.
- [x] Add direct `start`/`end` model fields before preserving definition
ranges.
- [x] Skip function body contents for wrapper metadata.
- [x] Balance braces while respecting strings, chars, and comments.
- [x] Ignore local declarations for exported signatures in v1.
- [x] Reject or diagnose K&R style function definitions initially.
Expand Down
23 changes: 15 additions & 8 deletions docs/c_parser/c_parser_reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
Status: partial parser reference with raw directive metadata. The `c_parser`
package and explicit C CLI parse path exist, raw includes/simple macros are
recorded, and a first grammar-shaped subset parses simple declarations,
typedefs, globals, function prototypes, and function-definition headers.
typedefs, globals, function prototypes, and function-definition headers with
start/end locations.

This document is the future home for the C parser user and developer reference.
It should evolve into the C equivalent of `fortran_parser.md` as implementation
Expand Down Expand Up @@ -67,6 +68,8 @@ Implemented:
- simple function prototype extraction
- prototype-style metadata distinguishing `int f(void)` from `int f()`
- simple function-definition signature extraction with body skipping
- start/end locations for function definitions, from the signature start
through the closing brace
- unsupported K&R function-definition diagnostics
- C fixture inputs under `tests/data/c/general/` plus C fixture directory
scaffolding for errors, corpus, and scientific APIs
Expand Down Expand Up @@ -201,8 +204,9 @@ current partial phase can populate `functions`, `typedefs`, `globals`,
`structs`, `unions`, and `enums` remain empty until their dedicated parser
phase lands. Functions include `prototype_style`, currently `"prototype"` for
typed or explicit `void` parameter lists and `"unspecified"` for empty
parameter lists such as `int f()`. Re-export from `x2py` is still deferred;
users should import from `c_parser`.
parameter lists such as `int f()`. Function definitions do not store
executable body text; they include direct `start` and `end` locations.
Re-export from `x2py` is still deferred; users should import from `c_parser`.

`macro_defines` is reserved for future compiler-assisted preprocessing
configuration. It must not mean that raw mode evaluates C preprocessor
Expand Down Expand Up @@ -252,7 +256,9 @@ Per-file shape:
"variadic": false,
"is_definition": false,
"prototype_style": "prototype",
"source_location": {"filename": "<path>", "line": 1, "...": "..."}
"source_location": {"filename": "<path>", "line": 1, "...": "..."},
"start": {"filename": "<path>", "line": 1, "...": "..."},
"end": null
}
],
"structs": [],
Expand Down Expand Up @@ -350,10 +356,11 @@ public entrypoints, empty model serialization, CLI discovery, JSON/output-file
behavior, unsupported C stages, comment stripping, line-continuation folding,
top-level splitting, include collection, simple macro collection, macro-shaped
declaration deferral, raw conditional branch non-selection, simple declarations,
globals, typedefs, and simple function prototypes/definitions. The broader
roadmap tests remain skipped until their matching implementation branches land.
Future implementation branches should unskip only the tests for the capability
they implement, then merge those branches back into `c-parser/main`.
globals, typedefs, and simple function prototypes/definitions, including
function-definition start/end locations. The broader roadmap tests remain skipped
until their matching implementation branches land. Future implementation
branches should unskip only the tests for the capability they implement, then
merge those branches back into `c-parser/main`.

Fixture layout should be separate from Fortran:

Expand Down
2 changes: 1 addition & 1 deletion tests/parser/c/test_c_corpus.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ def test_cjson_source_file_parse_skips_function_bodies_safely():
parsed = parse_c_file(_CJSON_DIR / "cJSON.c")

assert any(fn.name == "cJSON_Parse" for fn in parsed.functions)
assert all(fn.body is None for fn in parsed.functions)
assert not any(hasattr(fn, "body") for fn in parsed.functions)


def test_cjson_project_parse_links_header_and_source():
Expand Down
10 changes: 5 additions & 5 deletions tests/parser/c/test_c_functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,7 @@ def test_function_prototypes_preserve_return_type_parameter_order_and_names():
assert [param.name for param in fn.parameters] == ["n", "x", "y"]


@pytest.mark.skip(reason="function body source spans are not modeled yet.")
def test_function_definitions_skip_bodies_but_preserve_source_span():
def test_function_definitions_skip_bodies_but_preserve_start_and_end_locations():
from c_parser import parse_c_file

parsed = parse_c_file(
Expand All @@ -34,9 +33,10 @@ def test_function_definitions_skip_bodies_but_preserve_source_span():

fn = parsed.functions[0]
assert fn.name == "add"
assert fn.body is None
assert fn.source_span.start.line == 2
assert fn.source_span.end.line == 5
assert fn.start is not None
assert fn.end is not None
assert fn.start.line == 2
assert fn.end.line == 5


def test_void_parameter_list_and_empty_parameter_list_are_distinguished():
Expand Down
Loading