Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 52 additions & 9 deletions c_parser/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -245,47 +245,69 @@ def _macro_dependencies(
self,
source: str,
filename: str | None,
macro_names: set[str],
function_like_macro_names: set[str],
object_like_macro_names: set[str] | None = None,
) -> list[CMacroDependency]:
dependencies: list[CMacroDependency] = []
if not macro_names:
if not function_like_macro_names and not object_like_macro_names:
return dependencies

for segment in split_top_level_c_source(source, filename=filename):
dependency = self._segment_macro_dependency(segment, macro_names)
dependency = self._segment_macro_dependency(
segment,
function_like_macro_names,
object_like_macro_names,
)
if dependency is not None:
dependencies.append(dependency)
return dependencies

def _segment_macro_dependency(
self,
segment: CTopLevelSegment,
macro_names: set[str],
function_like_macro_names: set[str],
object_like_macro_names: set[str] | None = None,
) -> CMacroDependency | None:
text = segment.text.strip()
if not text:
return None
declaration_text, initializer = top_level_partition(text, "=")
scan_text = declaration_text if initializer is not None else text
for macro_name in sorted(macro_names):
for macro_name in sorted(function_like_macro_names):
if re.search(rf"\b{re.escape(macro_name)}\s*\(", scan_text):
return CMacroDependency(
name=macro_name,
context="declaration",
source_text=text,
source_location=self._source_location(segment),
)
for macro_name in sorted(object_like_macro_names or set()):
prefix_words = _STORAGE_CLASSES | _TYPE_QUALIFIERS | _FUNCTION_SPECIFIERS
prefix_pattern = "|".join(re.escape(word) for word in sorted(prefix_words))
if re.match(
rf"^(?:(?:{prefix_pattern})\s+)*{re.escape(macro_name)}\b",
scan_text,
):
return CMacroDependency(
name=macro_name,
context="declaration",
source_text=text,
source_location=self._source_location(segment),
)
return None

def _macro_dependent_declaration_diagnostic(
self,
segment: CTopLevelSegment,
dependency: CMacroDependency,
*,
function_like: bool,
) -> CDiagnostic:
macro_kind = "function-like" if function_like else "object-like"
return CDiagnostic(
code="C_MACRO_DEPENDENT_DECLARATION",
message=(
f"Declaration depends on function-like macro {dependency.name!r}; "
f"Declaration depends on {macro_kind} macro {dependency.name!r}; "
"provide preprocessed input to parse it."
),
severity="warning",
Expand Down Expand Up @@ -1160,6 +1182,8 @@ def _raise_for_unsupported_old_style_definitions(

for index, line in enumerate(stripped_lines):
text = line.strip()
if text.startswith("#"):
continue
parameter_bounds = self._find_parameter_list(text)
if parameter_bounds is None:
continue
Expand All @@ -1168,6 +1192,8 @@ def _raise_for_unsupported_old_style_definitions(
name_match = self._last_identifier(before_parameters)
if name_match is None:
continue
if name_match.group(0) in {"if", "for", "while", "switch"}:
continue
return_spec = before_parameters[: name_match.start()].strip()
if not return_spec or "(" in return_spec or ")" in return_spec:
continue
Expand Down Expand Up @@ -1722,6 +1748,7 @@ def _parse_translation_unit(
filename: str | None,
*,
function_like_macros: set[str] | None = None,
object_like_macros: set[str] | None = None,
) -> tuple[
list[CFunction],
list[CStruct],
Expand All @@ -1741,12 +1768,21 @@ def _parse_translation_unit(
variables: list[CVariable] = []
diagnostics: list[CDiagnostic] = []

macro_names = function_like_macros or set()
function_like_names = function_like_macros or set()
object_like_names = object_like_macros or set()
for segment in split_top_level_c_source(source, filename=filename):
macro_dependency = self._segment_macro_dependency(segment, macro_names)
macro_dependency = self._segment_macro_dependency(
segment,
function_like_names,
object_like_names,
)
if macro_dependency is not None:
diagnostics.append(
self._macro_dependent_declaration_diagnostic(segment, macro_dependency)
self._macro_dependent_declaration_diagnostic(
segment,
macro_dependency,
function_like=macro_dependency.name in function_like_names,
)
)
continue
tag_definition = self._parse_tag_definition(segment)
Expand Down Expand Up @@ -2017,16 +2053,23 @@ def visit_file(
parsed.macros = metadata.macros
parsed.raw_directives = metadata.raw_directives
function_like_macro_names = {macro.name for macro in metadata.macros if macro.function_like}
object_like_macro_names = {
macro.name
for macro in metadata.macros
if macro.directive == "define" and not macro.function_like
}
parsed.macro_dependencies = self._macro_dependencies(
source,
filename,
function_like_macro_names,
object_like_macro_names,
)
parsed.diagnostics = metadata.diagnostics
functions, structs, unions, enums, typedefs, variables, parser_diagnostics = self._parse_translation_unit(
source,
filename,
function_like_macros=function_like_macro_names,
object_like_macros=object_like_macro_names,
)
parsed.functions = functions
parsed.structs = structs
Expand Down
27 changes: 17 additions & 10 deletions docs/c_parser/c_parser_architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,10 @@ Implemented now:
models expose `result_type` and named `parameters`; their derived
`CFunctionType` is the nameless callable signature. Array and function
parameter declarations preserve their written `declared_type` and expose
pointer-adjusted effective `type` values. Selected unsupported
declaration forms, including attributes, alignment specifiers,
pointer-adjusted effective `type` values. Declarations prefixed by
unexpanded object-like macros are deferred as macro dependencies rather than
misreported as invalid type sequences. Selected unsupported declaration
forms, including attributes, alignment specifiers,
`_Atomic(type)`, nested aggregate member definitions, and static assertions,
are reported as diagnostics with
explicit `unit_kind` values. A declarator must be fully consumed before a
Expand All @@ -74,11 +76,13 @@ Implemented now:
- `--language c --semantics`, `--language c --pyi`, and C wrap-readiness are
rejected until semantic conversion exists.
- Focused partial CLI/API, declaration/function, diagnostic color, project
include/index, and raw lexer/directive tests are unskipped while broader
roadmap tests remain skipped.
- `tests/data/c/` contains C fixture scaffolding and general fixtures modeled
after the Fortran general fixture themes, with additional C-specific API
shapes.
include/index, raw lexer/directive, project golden, error golden, and JSON
schema tests are active while corpus, semantic, and `.pyi` roadmap tests
remain skipped.
- `tests/data/c/` contains general fixtures modeled after the Fortran general
fixture themes, additional C-specific API shapes, fatal diagnostic inputs,
and real-world cJSON/jsmn/tinyexpr/linmath/NanoSVG/stb inputs whose partial
project parse reports are covered by regression goldens.

Deferred:

Expand Down Expand Up @@ -131,6 +135,8 @@ should not wait for a separate request.
- `tests/pyi/test_pyi_fixture_suite.py`
- `tests/parser/fortran/generate_fortran_parser_goldens.py`
- `tests/parser/fortran/errors/generate_fortran_parser_error_goldens.py`
- `tests/parser/c/generate_c_parser_goldens.py`
- `tests/parser/c/errors/generate_c_parser_error_goldens.py`
- `tests/semantics/generate_semantic_fixtures.py`
- `tests/pyi/generate_pyi_fixtures.py`
- `tests/_shared/fixture_outputs.py`
Expand Down Expand Up @@ -246,7 +252,8 @@ Current and planned responsibilities:
locations, K&R-style definitions are rejected with `CParseError`, and
invalid primitive-specifier combinations are rejected with `CPARSE003`.
Array and function parameters preserve `declared_type` while effective
`type` uses C parameter adjustment.
`type` uses C parameter adjustment. Raw declarations beginning with an
object-like macro name are retained as macro-dependent diagnostics.
- Planned: symbol resolution and additional declaration-specifier and
extension coverage.
- `c_parser/project.py`
Expand Down Expand Up @@ -512,8 +519,8 @@ Raw-source mode target:
provenance.
- Parse ordinary declarations only when they are visible without macro
expansion.
- Mark macro-shaped declaration regions as unsupported/deferred rather than
treating them as parsed declarations.
- Mark function-like wrappers and object-like declaration-prefix regions as
unsupported/deferred rather than treating them as parsed declarations.
- Do not select active branches from `#if`/`#ifdef` in raw mode.

Compiler-assisted preprocessing target:
Expand Down
9 changes: 6 additions & 3 deletions docs/c_parser/c_parser_cli_workflow.md
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,8 @@ Potential later flags:
`--define` and `--undef` should belong to compiler-assisted preprocessing, not
to raw parser-side macro evaluation. Raw C mode records directives and parses
ordinary visible declarations only; it does not select `#if` branches or expand
macros.
macros. A declaration prefixed by an unexpanded object-like macro is deferred
as macro-dependent rather than treated as an invalid type sequence.

## Current Partial Behavior

Expand Down Expand Up @@ -270,8 +271,9 @@ For raw directives, the same JSON shape is used, but `includes`, `macros`,
`raw_directives`, `macro_dependencies`, and `diagnostics` may contain populated
model dictionaries. Function-like macros are recorded as macro metadata and
also produce a non-fatal `C_UNSUPPORTED_FUNCTION_LIKE_MACRO` diagnostic.
Macro-shaped declarations are marked through `macro_dependencies` without
being parsed as expanded declarations. Local quoted includes are resolved
Function-like declaration wrappers and object-like declaration prefixes are
marked through `macro_dependencies` without being parsed as expanded
declarations. Local quoted includes are resolved
relative to the current file or configured include dirs when possible;
unresolved local includes produce `C_UNRESOLVED_INCLUDE` diagnostics instead
of hard failures.
Expand Down Expand Up @@ -418,6 +420,7 @@ The active CLI/parser tests cover the current partial subset:
- `--language c --parse --debug-traceback` is accepted.
- raw comment stripping, line-continuation folding, top-level splitting,
include collection, simple macro collection, function-like macro diagnostics,
object-like macro declaration-prefix deferral,
conditional non-selection, simple declarations, variables, typedefs,
parenthesized declarators, function pointer typedefs/parameters, recursive
declarator combinations, concrete declaration objects, aggregate
Expand Down
71 changes: 49 additions & 22 deletions docs/c_parser/c_parser_implementation_checklist.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# C Parser Implementation Checklist

Status: implementation checklist with Phase 1 skeleton, selected Phase 2
fixture scaffolding, selected Phase 3 model/error work, Phase 4 raw
Status: implementation checklist with Phase 1 skeleton, Phase 2 parser/error
golden workflow, selected Phase 3 model/error work, Phase 4 raw
lexer/directive metadata, a first Phase 5/6 partial declaration/function
subset including top-level redeclaration handling, and selected Phase 8 project
include/index work complete. The
Expand All @@ -19,7 +19,8 @@ Aggregate members carry their own source locations, and flexible array
members are classified and checked for supported struct/union constraints.
Function parameters preserve written array/function forms in `declared_type`
while exposing C-adjusted pointer forms in `type`. Raw conditional directives
and macro-shaped declarations are stored as metadata. Project parsing now
and macro-shaped declarations, including object-like declaration prefixes, are
stored as metadata. Project parsing now
records include graphs, system and unresolved includes, functions by file,
enum constants, header/source pairings, and basic cross-file typedef/tag
resolution with incomplete tag completion. Compatible top-level prototypes,
Expand All @@ -35,7 +36,7 @@ stable.
## Progress Snapshot

- Last updated: 2026-05-24
- Checklist progress: 590/856 checked (68.9%).
- Checklist progress: 623/873 checked (71.4%).
- Current parser status: partial C parser with raw directive metadata, top-level
source splitting, simple declarations/variables/typedefs, prototype-style
metadata, K&R diagnostics, simple function signatures, and start/end
Expand All @@ -53,8 +54,9 @@ stable.
diagnostics for invalid placement or union use. Array and function
parameter declarations preserve their source form in `declared_type` while
their effective `type` applies C parameter-to-pointer adjustment. Raw
conditional directives and macro-shaped declaration dependencies are recorded
as metadata. `parse_c_project` returns project include/index facts and
conditional directives and macro-shaped declaration dependencies, including
object-like declaration prefixes, are recorded as metadata. `parse_c_project`
returns project include/index facts and
resolves basic cross-file typedef and tag references while preserving
unresolved references for later diagnostics. Top-level compatible
redeclarations are merged, matching prototypes plus definitions prefer the
Expand Down Expand Up @@ -335,8 +337,20 @@ Scope:
- [x] Create `tests/data/c/errors/parser/`.
- [x] Create `tests/data/c/corpus/`.
- [x] Create `tests/data/c/scientific/`.
- [ ] Create `tests/parser/c/fixtures/general/`.
- [ ] Create `tests/parser/c/fixtures/errors/`.
- [x] Add `tests/data/c/json/` real-world partial-parser regression inputs.
- [x] Add `tests/data/c/tinyexpr/` real-world partial-parser regression inputs.
- [x] Add `tests/data/c/linmath/` header-only partial-parser regression input.
- [x] Add `tests/data/c/nanosvg/` dependent-header partial-parser regression
inputs.
- [x] Add top-level `tests/data/c/stb/` single-file library regression inputs
without vendoring nested repository metadata as fixtures.
- [x] Create `tests/parser/c/fixtures/general/`.
- [x] Create `tests/parser/c/fixtures/json/`.
- [x] Create `tests/parser/c/fixtures/tinyexpr/`.
- [x] Create `tests/parser/c/fixtures/linmath/`.
- [x] Create `tests/parser/c/fixtures/nanosvg/`.
- [x] Create `tests/parser/c/fixtures/stb/`.
- [x] Create `tests/parser/c/fixtures/errors/`.
- [x] Keep C fixture data separate from Fortran fixture data.
- [x] Add README files explaining each C fixture directory.
- [x] Add small `.h` and `.c` fixture files for C fixture coverage.
Expand Down Expand Up @@ -377,18 +391,26 @@ Scope:

### Golden Workflow Tasks

- [ ] Create `tests/parser/c/generate_c_parser_goldens.py`.
- [ ] Mirror the Fortran parser golden generator structure.
- [ ] Serialize only dataclass/JSON-stable C parse models.
- [ ] Strip parent/back-reference fields if future models need them.
- [ ] Support updating all fixtures.
- [ ] Support updating selected fixtures.
- [ ] Add an environment variable update flow, for example
- [x] Create `tests/parser/c/generate_c_parser_goldens.py`.
- [x] Mirror the Fortran parser golden generator structure.
- [x] Serialize only dataclass/JSON-stable C parse models.
- [x] Strip parent/back-reference fields if future models need them.
- [x] Generate one `CProject` golden per same-stem fixture group, pairing
`.c` and `.h` inputs when both exist.
- [x] Order `.c` before its matched `.h` project input to mirror compilation
while include-expanded parsing remains deferred.
- [x] Support explicit dependent-header project groups ordered from included
header to dependent header.
- [x] Generate separate one-file project goldens for STB single-file library
inputs.
- [x] Support updating all fixtures.
- [x] Support updating selected fixtures.
- [x] Add an environment variable update flow, for example
`C_PARSER_UPDATE_GOLDENS=1`.
- [ ] Document whether C uses `C_PARSER_UPDATE_GOLDENS` or a generic
- [x] Document whether C uses `C_PARSER_UPDATE_GOLDENS` or a generic
`X2PY_UPDATE_GOLDENS`.
- [ ] Create a C error golden generator.
- [ ] Store expected error type, message fragments, diagnostic fragments, and
- [x] Create a C error golden generator.
- [x] Store expected error type, message fragments, diagnostic fragments, and
parser entrypoint metadata.

### Focused Test Buckets
Expand All @@ -415,7 +437,7 @@ Scope:

- [x] C test directory structure is present.
- [x] C fixture directory structure is present.
- [ ] C golden update workflow is documented.
- [x] C golden update workflow is documented.
- [x] Partial parser and metadata tests pass against current behavior.
- [x] Fortran tests still pass.
- [x] No real parser claims are made without tests.
Expand Down Expand Up @@ -620,6 +642,8 @@ Scope:
and `1` evaluator for C API extraction unless a later design explicitly
justifies it.
- [x] Mark macro-shaped declarations as unsupported/deferred in raw mode.
- [x] Defer declarations prefixed by object-like macros in raw mode instead of
reporting invalid type specifier sequences.
- [x] Store macro-dependency metadata in C parser models.
- [x] Store preprocessing mode metadata in `CFile`.
- [x] Store raw directive metadata separately from compiler-preprocessor
Expand All @@ -631,6 +655,9 @@ Scope:
- [x] Add tests for include collection.
- [x] Add tests for object-like macro collection.
- [x] Add tests for function-like macro diagnostics.
- [x] Add tests for object-like macro declaration-prefix deferral.
- [x] Do not misclassify `#if MACRO(...)` or body `else if (...)` forms as
K&R-style definitions.
- [x] Add tests that raw conditional directives do not select active branches.
- [x] Add tests that macro-generated declarations are deferred in raw mode.

Expand Down Expand Up @@ -992,7 +1019,7 @@ Scope:
- [x] Shared declaration backend handles members and typedefs.
- [x] Validate legal and invalid flexible-array-member placement and retain
named, unnamed, and zero-width bit-field source facts with tests.
- [ ] JSON goldens cover composite type schema.
- [x] JSON goldens cover composite type schema.
- [x] Docs list supported and unsupported composite forms.

### Phase 7 Risks And Open Questions
Expand Down Expand Up @@ -1382,7 +1409,7 @@ Scope:
- [ ] Run `.pyi` tests.
- [ ] Run C corpus parse-only tests.
- [x] Run CLI tests.
- [ ] Run golden fixture tests.
- [x] Run golden fixture tests.
- [x] Confirm Fortran tests still pass.
- [ ] Audit JSON schema stability.
- [ ] Audit error diagnostic stability.
Expand All @@ -1396,7 +1423,7 @@ Scope:
- [ ] Decide criteria for merging `c-parser/main` into project `main`.
- [ ] Require green CI for Fortran and C suites.
- [ ] Require docs updated for implemented subset.
- [ ] Require fixture/golden workflow documented.
- [x] Require fixture/golden workflow documented.
- [ ] Require semantic and `.pyi` behavior documented.
- [ ] Require explicit non-goals still documented.
- [ ] Require migration notes for users.
Expand Down
Loading
Loading