diff --git a/c_parser/__init__.py b/c_parser/__init__.py index 76eb6db0f..36e5ac6d4 100644 --- a/c_parser/__init__.py +++ b/c_parser/__init__.py @@ -3,48 +3,102 @@ from .models import ( CArray, + CAtomic, + CBool, + CChar, + CComposedType, + CConst, + CDouble, + CDoubleComplex, CDiagnostic, CEnum, CEnumerator, - CField, CFile, + CFloat, + CFloatComplex, CFunction, - CGlobal, + CFunctionType, CInclude, + CInitializer, + CInt, + CLong, + CLongDouble, + CLongDoubleComplex, + CLongLong, CMacro, CParameter, CParseError, CPointer, CProject, + CQualifier, + CRestrict, + CShort, + CSignedChar, CSourceLocation, CStruct, - CTypeRef, + CType, CTypedef, + CUnknownType, CUnion, + CUnsignedChar, + CUnsignedInt, + CUnsignedLong, + CUnsignedLongLong, + CUnsignedShort, + CVariable, + CVoid, + CVolatile, ) from .parser import CParser, parse_c_file, parse_c_project __all__ = ( "CArray", + "CAtomic", + "CBool", + "CChar", + "CComposedType", + "CConst", + "CDouble", + "CDoubleComplex", "CDiagnostic", "CEnum", "CEnumerator", - "CField", "CFile", + "CFloat", + "CFloatComplex", "CFunction", - "CGlobal", + "CFunctionType", "CInclude", + "CInitializer", + "CInt", + "CLong", + "CLongDouble", + "CLongDoubleComplex", + "CLongLong", "CMacro", "CParameter", "CParseError", "CPointer", "CParser", "CProject", + "CQualifier", + "CRestrict", + "CShort", + "CSignedChar", "CSourceLocation", "CStruct", - "CTypeRef", + "CType", "CTypedef", "CUnion", + "CUnknownType", + "CUnsignedChar", + "CUnsignedInt", + "CUnsignedLong", + "CUnsignedLongLong", + "CUnsignedShort", + "CVariable", + "CVoid", + "CVolatile", "parse_c_file", "parse_c_project", ) diff --git a/c_parser/cli.py b/c_parser/cli.py index 83db76836..4797bbdf7 100644 --- a/c_parser/cli.py +++ b/c_parser/cli.py @@ -50,7 +50,7 @@ def format_c_report(report: dict[str, dict]) -> str: lines.append(f" Unions: {len(parsed.get('unions') or [])}") lines.append(f" Enums: {len(parsed.get('enums') or [])}") lines.append(f" Typedefs: {len(parsed.get('typedefs') or [])}") - lines.append(f" Globals: {len(parsed.get('globals') or [])}") + lines.append(f" Variables: {len(parsed.get('variables') or [])}") lines.append(f" Macros: {len(parsed.get('macros') or [])}") lines.append(f" Includes: {len(parsed.get('includes') or [])}") lines.append(f" Diagnostics: {len(parsed.get('diagnostics') or [])}") diff --git a/c_parser/lexer.py b/c_parser/lexer.py index 6a3ecca7e..097dc36d6 100644 --- a/c_parser/lexer.py +++ b/c_parser/lexer.py @@ -169,6 +169,15 @@ def top_level_partition(text: str, delimiter: str = "=") -> tuple[str, str | Non return text.strip(), None +def _is_aggregate_definition_header(header: str) -> bool: + """Identify a tag definition before deciding that a brace starts a body.""" + compact = " ".join(header.split()) + if "(" in compact or "=" in compact: + return False + words = compact.split() + return any(word in {"struct", "union", "enum"} for word in words) + + def split_top_level_c_source( source: str, filename: str | None = None, @@ -198,6 +207,7 @@ def split_top_level_c_source( block_start_line = 1 block_start_column = 1 block_source_line: str | None = None + aggregate_block = False while i < len(stripped): char = stripped[i] @@ -251,13 +261,15 @@ def split_top_level_c_source( block_start_line = start_line block_start_column = start_column block_source_line = _source_line(source_lines, start_line) + aggregate_block = _is_aggregate_definition_header(header) brace_depth = 1 - start_index = None + if not aggregate_block: + start_index = None elif char == "{" and brace_depth: brace_depth += 1 elif char == "}" and brace_depth: brace_depth -= 1 - if brace_depth == 0 and block_header: + if brace_depth == 0 and block_header and not aggregate_block: segments.append( CTopLevelSegment( text=block_header, @@ -296,6 +308,9 @@ def split_top_level_c_source( ) ) start_index = None + block_header = None + block_source_line = None + aggregate_block = False line, column = _advance_position(char, line, column) i += 1 diff --git a/c_parser/models.py b/c_parser/models.py index 61fd6250d..6698db245 100644 --- a/c_parser/models.py +++ b/c_parser/models.py @@ -46,16 +46,28 @@ def _enable_windows_ansi() -> None: # pragma: no cover - Windows-only console s colorama.just_fix_windows_console() -def c_model_to_dict(obj: Any) -> Any: +def c_model_to_dict(obj: Any, _seen: set[int] | None = None) -> Any: """Convert C parser dataclasses into stable JSON-compatible values.""" + if _seen is None: + _seen = set() + if isinstance(obj, CQualifier): + return obj.spelling + if isinstance(obj, CType): + if isinstance(obj, (CStruct, CUnion, CEnum, CTypedef)) and id(obj) in _seen: + return {"reference": obj.reference_name} + if isinstance(obj, (CStruct, CUnion, CEnum, CTypedef)): + _seen.add(id(obj)) + payload = {"model": type(obj).__name__} + payload.update({f.name: c_model_to_dict(getattr(obj, f.name), _seen) for f in fields(obj)}) + return payload if is_dataclass(obj): - return {f.name: c_model_to_dict(getattr(obj, f.name)) for f in fields(obj)} + return {f.name: c_model_to_dict(getattr(obj, f.name), _seen) for f in fields(obj)} if isinstance(obj, list): - return [c_model_to_dict(v) for v in obj] + return [c_model_to_dict(v, _seen) for v in obj] if isinstance(obj, dict): - return {k: c_model_to_dict(v) for k, v in obj.items()} + return {k: c_model_to_dict(v, _seen) for k, v in obj.items()} if isinstance(obj, set): - return sorted(c_model_to_dict(v) for v in obj) + return sorted(c_model_to_dict(v, _seen) for v in obj) return obj @@ -153,105 +165,244 @@ class CDiagnostic: unit_name: str | None = None +@dataclass(frozen=True) +class CQualifier: + spelling: str + + +@dataclass(frozen=True) +class CConst(CQualifier): + spelling: str = "const" + + +@dataclass(frozen=True) +class CVolatile(CQualifier): + spelling: str = "volatile" + + +@dataclass(frozen=True) +class CRestrict(CQualifier): + spelling: str = "restrict" + + +@dataclass(frozen=True) +class CAtomic(CQualifier): + spelling: str = "_Atomic" + + +@dataclass(kw_only=True) +class CType: + qualifiers: list[CQualifier] = field(default_factory=list) + source_text: str = "" + + @dataclass -class CPointer: - qualifiers: list[str] = field(default_factory=list) +class CUnknownType(CType): + spelling: str = "unknown" @dataclass -class CArray: - size: str | None = None - static: bool = False +class CVoid(CType): + pass @dataclass -class CTypeRef: - base: str | None = None - qualifiers: list[str] = field(default_factory=list) - storage_class: list[str] = field(default_factory=list) - sign: str | None = None - width: str | None = None - tag_kind: str | None = None - tag_name: str | None = None - typedef_name: str | None = None - pointers: list[CPointer] = field(default_factory=list) - arrays: list[CArray] = field(default_factory=list) - kind: str = "type" - source_text: str = "" - resolved: Any = None +class CBool(CType): + pass + + +@dataclass +class CChar(CType): + pass + + +@dataclass +class CSignedChar(CType): + pass + + +@dataclass +class CUnsignedChar(CType): + pass + + +@dataclass +class CShort(CType): + pass + + +@dataclass +class CUnsignedShort(CType): + pass + + +@dataclass +class CInt(CType): + pass + + +@dataclass +class CUnsignedInt(CType): + pass + + +@dataclass +class CLong(CType): + pass + + +@dataclass +class CUnsignedLong(CType): + pass + + +@dataclass +class CLongLong(CType): + pass + + +@dataclass +class CUnsignedLongLong(CType): + pass + + +@dataclass +class CFloat(CType): + pass + + +@dataclass +class CDouble(CType): + pass + + +@dataclass +class CLongDouble(CType): + pass + + +@dataclass +class CFloatComplex(CType): + pass + + +@dataclass +class CDoubleComplex(CType): + pass + + +@dataclass +class CLongDoubleComplex(CType): + pass + + +@dataclass +class CPointer(CType): + pass + + +@dataclass +class CArray(CType): + bound: str | None = None + is_static_minimum: bool = False + is_variable_length: bool = False + is_flexible: bool = False + + +@dataclass +class CFunctionType(CType): + result_type: CType = field(default_factory=CVoid) + parameter_types: list[CType] = field(default_factory=list) + is_variadic: bool = False + prototype_style: str | None = None + + +@dataclass +class CComposedType(CType): + components: list[CType] = field(default_factory=list) @property def pointer_depth(self) -> int: - return len(self.pointers) + return sum(isinstance(component, CPointer) for component in self.components) @property def array_rank(self) -> int: - return len(self.arrays) + return sum(isinstance(component, CArray) for component in self.components) - @property - def effective_type_text(self) -> str: - if self.source_text: - return self.source_text - if self.typedef_name: - return self.typedef_name - if self.tag_kind and self.tag_name: - return f"{self.tag_kind} {self.tag_name}" - return self.base or "unknown" - - @property - def is_const_pointer(self) -> bool: - return bool(self.pointers) and "const" in self.qualifiers - @property - def is_opaque_type(self) -> bool: - return bool(self.tag_kind and self.tag_name and self.resolved is None) +def _contains_function_pointer(type_: CType) -> bool: + if not isinstance(type_, CComposedType): + return False + for index, component in enumerate(type_.components): + if isinstance(component, CPointer) and any( + isinstance(later, CFunctionType) for later in type_.components[index + 1 :] + ): + return True + return False @dataclass class CParameter: name: str | None = None - type: CTypeRef = field(default_factory=CTypeRef) + type: CType = field(default_factory=CVoid) + declared_type: CType | None = None source_location: CSourceLocation | None = None + callback_policy: Any = None + + @property + def callback_candidate(self) -> bool: + return _contains_function_pointer(self.type) @dataclass class CFunction: name: str - return_type: CTypeRef = field(default_factory=CTypeRef) + result_type: CType = field(default_factory=CVoid) parameters: list[CParameter] = field(default_factory=list) storage: list[str] = field(default_factory=list) specifiers: list[str] = field(default_factory=list) - variadic: bool = False + is_variadic: bool = False is_definition: bool = False prototype_style: str | None = None source_location: CSourceLocation | None = None start: CSourceLocation | None = None end: CSourceLocation | None = None - -@dataclass -class CField: - name: str | None = None - type: CTypeRef = field(default_factory=CTypeRef) - source_location: CSourceLocation | None = None + @property + def type(self) -> CFunctionType: + return CFunctionType( + result_type=self.result_type, + parameter_types=[parameter.type for parameter in self.parameters], + is_variadic=self.is_variadic, + prototype_style=self.prototype_style, + ) @dataclass -class CStruct: +class CStruct(CType): name: str | None = None - fields: list[CField] = field(default_factory=list) + members: list["CVariable"] = field(default_factory=list) anonymous_id: str | None = None - opaque: bool = False + is_incomplete: bool = False source_location: CSourceLocation | None = None + @property + def reference_name(self) -> str: + return f"struct {self.name}" if self.name else self.anonymous_id or "anonymous struct" + @dataclass -class CUnion: +class CUnion(CType): name: str | None = None - fields: list[CField] = field(default_factory=list) + members: list["CVariable"] = field(default_factory=list) anonymous_id: str | None = None + is_incomplete: bool = False source_location: CSourceLocation | None = None + @property + def reference_name(self) -> str: + return f"union {self.name}" if self.name else self.anonymous_id or "anonymous union" + @dataclass class CEnumerator: @@ -261,25 +412,46 @@ class CEnumerator: @dataclass -class CEnum: +class CEnum(CType): name: str | None = None constants: list[CEnumerator] = field(default_factory=list) anonymous_id: str | None = None source_location: CSourceLocation | None = None + @property + def reference_name(self) -> str: + return f"enum {self.name}" if self.name else self.anonymous_id or "anonymous enum" + @dataclass -class CTypedef: +class CTypedef(CType): name: str - type: CTypeRef = field(default_factory=CTypeRef) + type: CType | None = None source_location: CSourceLocation | None = None + @property + def reference_name(self) -> str: + return self.name + @dataclass -class CGlobal: - name: str - type: CTypeRef = field(default_factory=CTypeRef) +class CInitializer: + source_text: str + + +@dataclass +class CVariable: + name: str | None + type: CType = field(default_factory=CVoid) + storage: list[str] = field(default_factory=list) + initializer: CInitializer | None = None + bit_width: str | None = None source_location: CSourceLocation | None = None + callback_policy: Any = None + + @property + def callback_candidate(self) -> bool: + return _contains_function_pointer(self.type) @dataclass @@ -310,7 +482,7 @@ class CFile: unions: list[CUnion] = field(default_factory=list) enums: list[CEnum] = field(default_factory=list) typedefs: list[CTypedef] = field(default_factory=list) - globals: list[CGlobal] = field(default_factory=list) + variables: list[CVariable] = field(default_factory=list) macros: list[CMacro] = field(default_factory=list) includes: list[CInclude] = field(default_factory=list) diagnostics: list[CDiagnostic] = field(default_factory=list) @@ -327,7 +499,7 @@ class CProject: unions: dict[str, CUnion] = field(default_factory=dict) enums: dict[str, CEnum] = field(default_factory=dict) typedefs: dict[str, CTypedef] = field(default_factory=dict) - globals: dict[str, CGlobal] = field(default_factory=dict) + variables: dict[str, CVariable] = field(default_factory=dict) macros: dict[str, CMacro] = field(default_factory=dict) includes: dict[str, CInclude] = field(default_factory=dict) diff --git a/c_parser/parser.py b/c_parser/parser.py index faa22ca09..299ac15e6 100644 --- a/c_parser/parser.py +++ b/c_parser/parser.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import annotations +from dataclasses import dataclass import re from collections.abc import Mapping, Sequence from pathlib import Path @@ -14,28 +15,66 @@ ) from .models import ( CArray, + CAtomic, + CBool, + CChar, + CComposedType, + CConst, + CDouble, + CDoubleComplex, + CDiagnostic, + CEnum, + CEnumerator, CFile, + CFloat, + CFloatComplex, CFunction, - CGlobal, + CFunctionType, CParameter, CParseError, CPointer, CProject, + CRestrict, CSourceLocation, - CTypeRef, + CShort, + CSignedChar, + CStruct, + CType, CTypedef, + CUnion, + CUnknownType, + CUnsignedChar, + CUnsignedInt, + CUnsignedLong, + CUnsignedLongLong, + CUnsignedShort, + CVariable, + CVoid, + CVolatile, + CInt, + CInitializer, + CLong, + CLongDouble, + CLongDoubleComplex, + CLongLong, ) from .preprocessor import collect_preprocessor_metadata _C_SOURCE_SUFFIXES = {".c", ".h"} _IDENTIFIER_RE = re.compile(r"[A-Za-z_]\w*") -_POINTER_TAIL_RE = re.compile(r"((?:\s*\*\s*(?:(?:const|restrict|volatile|_Atomic)\s*)*)+)$") -_ARRAY_RE = re.compile(r"\[([^\]]*)\]") _STORAGE_CLASSES = {"typedef", "extern", "static", "register", "_Thread_local"} _TYPE_QUALIFIERS = {"const", "restrict", "volatile", "_Atomic"} _FUNCTION_SPECIFIERS = {"inline", "_Noreturn"} _TAG_KINDS = {"struct", "union", "enum"} +_UNSUPPORTED_DECLARATION_MARKERS = ( + "__attribute__", + "__declspec", + "[[", + "_Alignas", + "alignas", + "_Atomic(", +) _PRIMITIVE_WORDS = { "void", "char", @@ -49,7 +88,80 @@ "_Bool", "_Complex", } -_TYPE_ONLY_WORDS = _PRIMITIVE_WORDS | _TYPE_QUALIFIERS | _TAG_KINDS +_QUALIFIER_CLASSES = { + "const": CConst, + "volatile": CVolatile, + "restrict": CRestrict, + "_Atomic": CAtomic, +} +_PRIMITIVE_TYPES = { + "void": CVoid, + "_Bool": CBool, + "char": CChar, + "signed char": CSignedChar, + "unsigned char": CUnsignedChar, + "short": CShort, + "short int": CShort, + "signed short": CShort, + "signed short int": CShort, + "unsigned short": CUnsignedShort, + "unsigned short int": CUnsignedShort, + "int": CInt, + "signed": CInt, + "signed int": CInt, + "unsigned": CUnsignedInt, + "unsigned int": CUnsignedInt, + "long": CLong, + "long int": CLong, + "signed long": CLong, + "signed long int": CLong, + "unsigned long": CUnsignedLong, + "unsigned long int": CUnsignedLong, + "long long": CLongLong, + "long long int": CLongLong, + "signed long long": CLongLong, + "signed long long int": CLongLong, + "unsigned long long": CUnsignedLongLong, + "unsigned long long int": CUnsignedLongLong, + "float": CFloat, + "double": CDouble, + "long double": CLongDouble, + "float _Complex": CFloatComplex, + "_Complex": CDoubleComplex, + "double _Complex": CDoubleComplex, + "long double _Complex": CLongDoubleComplex, +} + + +@dataclass +class _PointerOp: + qualifiers: list[str] + + +@dataclass +class _ArrayOp: + size: str | None = None + static: bool = False + qualifiers: list[str] | None = None + variable_length: bool = False + + +@dataclass +class _FunctionOp: + parameters: list[CParameter] + variadic: bool = False + prototype_style: str | None = None + + +@dataclass +class _ParsedDeclarator: + name: str | None + operations: list[_PointerOp | _ArrayOp | _FunctionOp] + source_text: str = "" + + +class _UnsupportedDeclaratorSyntax(ValueError): + pass def _looks_like_existing_source_path(value: object) -> bool: @@ -74,8 +186,8 @@ def _collect_c_paths(path: Path) -> list[Path]: class CParser: """C parser entrypoint for the currently implemented C subset. - The implemented subset covers raw preprocessing metadata plus simple - top-level declarations, typedefs, and function prototypes/definitions. + The implemented subset covers raw preprocessing metadata, recursive + declarators, aggregate declarations, typedefs, and function signatures. """ def _source_location(self, segment: CTopLevelSegment) -> CSourceLocation: @@ -86,6 +198,9 @@ def _source_location(self, segment: CTopLevelSegment) -> CSourceLocation: source_line=segment.original_source_line, ) + def _has_unsupported_declaration_marker(self, text: str) -> bool: + return any(marker in text for marker in _UNSUPPORTED_DECLARATION_MARKERS) + def _end_location(self, segment: CTopLevelSegment) -> CSourceLocation: return CSourceLocation( filename=segment.filename, @@ -115,16 +230,13 @@ def _last_identifier(self, text: str) -> re.Match[str] | None: matches.extend(_IDENTIFIER_RE.finditer(text, start, end)) return matches[-1] if matches else None - def _split_pointer_tail(self, prefix: str) -> tuple[str, str]: - match = _POINTER_TAIL_RE.search(prefix) - if not match: - return prefix.strip(), "" - return prefix[: match.start()].strip(), match.group(1).strip() - def _specifier_words(self, spec_text: str) -> list[str]: return _IDENTIFIER_RE.findall(spec_text) - def _parse_specifiers(self, spec_text: str) -> tuple[CTypeRef, list[str]]: + def _qualifiers(self, spellings: list[str]) -> list: + return [_QUALIFIER_CLASSES[spelling]() for spelling in spellings] + + def _parse_specifiers(self, spec_text: str) -> tuple[CType, list[str], list[str]]: words = self._specifier_words(spec_text) storage: list[str] = [] qualifiers: list[str] = [] @@ -141,115 +253,357 @@ def _parse_specifiers(self, spec_text: str) -> tuple[CTypeRef, list[str]]: else: type_words.append(word) - typeref = CTypeRef( - qualifiers=qualifiers, - storage_class=storage, - source_text=spec_text.strip(), - ) if type_words and type_words[0] in _TAG_KINDS: - typeref.tag_kind = type_words[0] - if len(type_words) > 1: - typeref.tag_name = type_words[1] - typeref.base = " ".join(type_words[:2]) + tag_name = type_words[1] if len(type_words) > 1 else None + tag_type = {"struct": CStruct, "union": CUnion, "enum": CEnum}[type_words[0]] + tag_kwargs = { + "name": tag_name, + "qualifiers": self._qualifiers(qualifiers), + "source_text": " ".join([*qualifiers, *type_words]), + } + if tag_type in {CStruct, CUnion}: + tag_kwargs["is_incomplete"] = True + type_: CType = tag_type(**tag_kwargs) elif type_words: - base = " ".join(type_words) - typeref.base = base - if len(type_words) == 1 and type_words[0] not in _PRIMITIVE_WORDS: - typeref.typedef_name = type_words[0] - if "signed" in type_words: - typeref.sign = "signed" - elif "unsigned" in type_words: - typeref.sign = "unsigned" - widths = [word for word in type_words if word in {"char", "short", "int", "long"}] - if widths: - typeref.width = " ".join(widths) + spelling = " ".join(type_words) + primitive = _PRIMITIVE_TYPES.get(spelling) + if primitive is not None: + type_ = primitive( + qualifiers=self._qualifiers(qualifiers), + source_text=" ".join([*qualifiers, *type_words]), + ) + elif len(type_words) == 1: + type_ = CTypedef( + name=type_words[0], + qualifiers=self._qualifiers(qualifiers), + source_text=" ".join([*qualifiers, *type_words]), + ) + else: + type_ = CUnknownType( + spelling=spelling, + qualifiers=self._qualifiers(qualifiers), + source_text=" ".join([*qualifiers, *type_words]), + ) else: - typeref.base = "unknown" - - return typeref, function_specifiers - - def _parse_pointers(self, pointer_text: str) -> list[CPointer]: - pointers: list[CPointer] = [] - current: CPointer | None = None - for token in re.findall(r"\*|const|restrict|volatile|_Atomic", pointer_text): - if token == "*": - current = CPointer() - pointers.append(current) - elif current is not None: - current.qualifiers.append(token) - return pointers - - def _parse_arrays(self, declarator_tail: str) -> list[CArray]: - arrays: list[CArray] = [] - for match in _ARRAY_RE.finditer(declarator_tail): - content = " ".join(match.group(1).strip().split()) - is_static = False - size = content or None - if content.startswith("static "): + type_ = CUnknownType( + qualifiers=self._qualifiers(qualifiers), + source_text=spec_text.strip(), + ) + + return type_, storage, function_specifiers + + def _skip_whitespace(self, text: str, index: int) -> int: + while index < len(text) and text[index].isspace(): + index += 1 + return index + + def _read_identifier(self, text: str, index: int) -> tuple[str, int] | None: + if index >= len(text): + return None + first = text[index] + if not (first == "_" or first.isalpha()): + return None + end = index + 1 + while end < len(text) and (text[end] == "_" or text[end].isalnum()): + end += 1 + return text[index:end], end + + def _find_matching_delimiter( + self, + text: str, + open_index: int, + open_char: str, + close_char: str, + ) -> int | None: + depth = 0 + state = "normal" + quote = "" + escaped = False + for index in range(open_index, len(text)): + char = text[index] + if state in {"string", "char"}: + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif char == quote: + state = "normal" + quote = "" + continue + if char in {'"', "'"}: + state = "string" if char == '"' else "char" + quote = char + escaped = False + continue + if char == open_char: + depth += 1 + elif char == close_char: + depth -= 1 + if depth == 0: + return index + return None + + def _split_declaration_specifiers(self, text: str) -> tuple[str, str]: + index = 0 + spec_end = 0 + consumed_type = False + consumed_typedef_name = False + + while True: + index = self._skip_whitespace(text, index) + identifier = self._read_identifier(text, index) + if identifier is None: + break + word, end = identifier + + if word in _STORAGE_CLASSES or word in _TYPE_QUALIFIERS or word in _FUNCTION_SPECIFIERS: + index = end + spec_end = end + continue + + if word in _PRIMITIVE_WORDS: + consumed_type = True + index = end + spec_end = end + continue + + if word in _TAG_KINDS: + consumed_type = True + index = self._skip_whitespace(text, end) + tag_identifier = self._read_identifier(text, index) + if tag_identifier is not None: + _tag_name, index = tag_identifier + spec_end = index + continue + + if not consumed_type and not consumed_typedef_name: + consumed_type = True + consumed_typedef_name = True + index = end + spec_end = end + continue + + break + + return text[:spec_end].strip(), text[spec_end:].strip() + + def _parse_pointer_ops( + self, + text: str, + index: int, + ) -> tuple[list[_PointerOp], int]: + pointers: list[_PointerOp] = [] + index = self._skip_whitespace(text, index) + while index < len(text) and text[index] == "*": + index += 1 + qualifiers: list[str] = [] + while True: + index = self._skip_whitespace(text, index) + identifier = self._read_identifier(text, index) + if identifier is None: + break + word, end = identifier + if word not in _TYPE_QUALIFIERS: + break + qualifiers.append(word) + index = end + pointers.append(_PointerOp(qualifiers=qualifiers)) + index = self._skip_whitespace(text, index) + return pointers, index + + def _parse_array_op(self, content: str) -> _ArrayOp: + words = content.strip().split() + qualifiers: list[str] = [] + is_static = False + remaining: list[str] = [] + for word in words: + if word == "static": is_static = True - size = content[len("static ") :].strip() or None - arrays.append(CArray(size=size, static=is_static)) - return arrays + elif word in _TYPE_QUALIFIERS: + qualifiers.append(word) + else: + remaining.append(word) + normalized = " ".join(remaining) + variable_length = normalized == "*" + return _ArrayOp( + size=None if variable_length else normalized or None, + static=is_static, + qualifiers=qualifiers, + variable_length=variable_length, + ) - def _build_type( + def _parse_declarator_suffixes( + self, + text: str, + index: int, + ) -> tuple[list[_ArrayOp | _FunctionOp], int]: + operations: list[_ArrayOp | _FunctionOp] = [] + while True: + index = self._skip_whitespace(text, index) + if index >= len(text): + return operations, index + + if text[index] == "[": + close_index = self._find_matching_delimiter(text, index, "[", "]") + if close_index is None: + return operations, index + operations.append(self._parse_array_op(text[index + 1 : close_index])) + index = close_index + 1 + continue + + if text[index] == "(": + close_index = self._find_matching_delimiter(text, index, "(", ")") + if close_index is None: + return operations, index + parameters_text = text[index + 1 : close_index] + parameters, variadic = self._parse_parameters(parameters_text) + operations.append( + _FunctionOp( + parameters=parameters, + variadic=variadic, + prototype_style=self._prototype_style(parameters_text), + ) + ) + index = close_index + 1 + continue + + return operations, index + + def _parse_direct_declarator_at( + self, + text: str, + index: int, + ) -> tuple[str | None, list[_PointerOp | _ArrayOp | _FunctionOp], int]: + index = self._skip_whitespace(text, index) + if index < len(text) and text[index] == "(": + close_index = self._find_matching_delimiter(text, index, "(", ")") + if close_index is None: + return None, [], index + inner = self._parse_declarator(text[index + 1 : close_index]) + suffixes, index = self._parse_declarator_suffixes(text, close_index + 1) + return inner.name, [*reversed(suffixes), *inner.operations], index + + identifier = self._read_identifier(text, index) + if identifier is None: + suffixes, index = self._parse_declarator_suffixes(text, index) + return None, list(reversed(suffixes)), index + + name, index = identifier + suffixes, index = self._parse_declarator_suffixes(text, index) + return name, list(reversed(suffixes)), index + + def _parse_declarator_at( + self, + text: str, + index: int, + ) -> tuple[str | None, list[_PointerOp | _ArrayOp | _FunctionOp], int]: + pointers, index = self._parse_pointer_ops(text, index) + name, direct_operations, index = self._parse_direct_declarator_at(text, index) + return name, [*pointers, *direct_operations], index + + def _parse_declarator(self, text: str) -> _ParsedDeclarator: + stripped = text.strip() + if not stripped: + return _ParsedDeclarator(name=None, operations=[], source_text="") + name, operations, index = self._parse_declarator_at(stripped, 0) + remainder = stripped[self._skip_whitespace(stripped, index) :].strip() + if remainder: + raise _UnsupportedDeclaratorSyntax( + f"Unsupported declarator syntax after parsed type layers: {remainder!r}." + ) + return _ParsedDeclarator(name=name, operations=operations, source_text=stripped) + + def _prepend_component(self, component: CType, current: CType) -> CComposedType: + if isinstance(current, CComposedType): + return CComposedType( + components=[component, *current.components], + source_text=current.source_text, + ) + return CComposedType(components=[component, current], source_text=current.source_text) + + def _apply_pointer_operation(self, current: CType, operation: _PointerOp) -> CType: + return self._prepend_component( + CPointer(qualifiers=self._qualifiers(operation.qualifiers)), + current, + ) + + def _apply_array_operation(self, current: CType, operation: _ArrayOp) -> CType: + array = CArray( + bound=operation.size, + is_static_minimum=operation.static, + qualifiers=self._qualifiers(operation.qualifiers or []), + is_variable_length=operation.variable_length, + ) + return self._prepend_component(array, current) + + def _apply_function_operation(self, current: CType, operation: _FunctionOp) -> CType: + return CFunctionType( + result_type=current, + parameter_types=[parameter.type for parameter in operation.parameters], + is_variadic=operation.variadic, + prototype_style=operation.prototype_style, + source_text=current.source_text, + ) + + def _apply_declarator_operations( + self, + base_type: CType, + operations: list[_PointerOp | _ArrayOp | _FunctionOp], + ) -> CType: + current = base_type + for operation in operations: + if isinstance(operation, _PointerOp): + current = self._apply_pointer_operation(current, operation) + elif isinstance(operation, _ArrayOp): + current = self._apply_array_operation(current, operation) + else: + current = self._apply_function_operation(current, operation) + return current + + def _build_declared_type( self, spec_text: str, declarator_fragment: str = "", - ) -> tuple[CTypeRef, list[str]]: - spec_without_pointer, spec_pointer = self._split_pointer_tail(spec_text) - typeref, function_specifiers = self._parse_specifiers(spec_without_pointer) - typeref.pointers = self._parse_pointers(f"{spec_pointer} {declarator_fragment}") - typeref.arrays = self._parse_arrays(declarator_fragment) + ) -> tuple[str | None, CType, list[str], list[str], _FunctionOp | None]: + base_type, storage, function_specifiers = self._parse_specifiers(spec_text) + parsed = self._parse_declarator(declarator_fragment) + type_ = self._apply_declarator_operations(base_type, parsed.operations) source_parts = [spec_text.strip(), declarator_fragment.strip()] - typeref.source_text = " ".join(part for part in source_parts if part).strip() - return typeref, function_specifiers - - def _split_first_declarator(self, first_declarator: str) -> tuple[str, str]: - declaration, _initializer = top_level_partition(first_declarator, "=") - name_match = self._last_identifier(declaration) - if name_match is None: - return declaration.strip(), "" - - prefix = declaration[: name_match.start()] - suffix = declaration[name_match.end() :] - spec_text, pointer_tail = self._split_pointer_tail(prefix) - declarator = f"{pointer_tail} {name_match.group(0)}{suffix}".strip() - return spec_text, declarator - - def _declarator_name(self, declarator: str) -> tuple[str | None, str]: - declaration, _initializer = top_level_partition(declarator, "=") - name_match = self._last_identifier(declaration) - if name_match is None: - return None, declaration.strip() - return name_match.group(0), declaration.strip() - - def _looks_like_type_only_parameter(self, text: str) -> bool: - stripped = text.strip() - name_match = self._last_identifier(stripped) - if name_match is None: - return True - words = self._specifier_words(stripped) - last = name_match.group(0) - if last in _TYPE_ONLY_WORDS: - return True - if len(words) >= 2 and words[-2] in _TAG_KINDS: - return True - return False + type_.source_text = " ".join(part for part in source_parts if part).strip() + direct_function = ( + parsed.operations[-1] + if parsed.operations and isinstance(parsed.operations[-1], _FunctionOp) + else None + ) + return parsed.name, type_, storage, function_specifiers, direct_function + + def _build_type( + self, + spec_text: str, + declarator_fragment: str = "", + ) -> tuple[CType, list[str]]: + _name, type_, _storage, function_specifiers, _direct_function = self._build_declared_type( + spec_text, + declarator_fragment, + ) + return type_, function_specifiers def _parse_parameter(self, text: str) -> CParameter | None: stripped = text.strip() if not stripped or stripped == "void": return None - if self._looks_like_type_only_parameter(stripped): - typeref, _function_specifiers = self._build_type(stripped) - return CParameter(name=None, type=typeref) - - spec_text, declarator = self._split_first_declarator(stripped) + spec_text, declarator = self._split_declaration_specifiers(stripped) if not spec_text: return None - name, declaration = self._declarator_name(declarator) - typeref, _function_specifiers = self._build_type(spec_text, declaration) - return CParameter(name=name, type=typeref) + name, type_, _storage, _function_specifiers, _direct_function = self._build_declared_type( + spec_text, + declarator, + ) + return CParameter( + name=name, + type=type_, + declared_type=type_, + ) def _find_parameter_list(self, text: str) -> tuple[int, int] | None: close_index = len(text) - 1 @@ -379,118 +733,502 @@ def _prototype_style(self, parameters_text: str) -> str: def _parse_function(self, segment: CTopLevelSegment) -> CFunction | None: text = segment.text.strip() - if text.startswith(("typedef ", "struct ", "union ", "enum ")): - return None - - parameter_bounds = self._find_parameter_list(text) - if parameter_bounds is None: + if text.startswith(("typedef ", "_Static_assert")) or self._has_unsupported_declaration_marker(text): return None - open_index, close_index = parameter_bounds - before_parameters = text[:open_index].strip() - parameters_text = text[open_index + 1 : close_index] - name_match = self._last_identifier(before_parameters) - if name_match is None: + spec_text, declarator = self._split_declaration_specifiers(text) + if not spec_text or not declarator: return None - - name = name_match.group(0) - return_spec = before_parameters[: name_match.start()].strip() - return_declarator = before_parameters[name_match.end() :].strip() - if not return_spec: - return None - if "(" in return_spec or ")" in return_spec: + name, function_type, storage, function_specifiers, direct_function = self._build_declared_type( + spec_text, + declarator, + ) + if name is None or not isinstance(function_type, CFunctionType) or direct_function is None: return None - if self._is_knr_definition(segment, parameters_text): - raise CParseError( - "K&R style function definitions are not supported", - filename=segment.filename, - line_number=segment.original_start_line, - column=segment.original_start_column, - source_line=segment.original_source_line, - code="CPARSE002", - ) + parameter_bounds = self._find_parameter_list(text) + if parameter_bounds is not None: + open_index, close_index = parameter_bounds + parameters_text = text[open_index + 1 : close_index] + if "(" not in text[:open_index] and self._is_knr_definition(segment, parameters_text): + raise CParseError( + "K&R style function definitions are not supported", + filename=segment.filename, + line_number=segment.original_start_line, + column=segment.original_start_column, + source_line=segment.original_source_line, + code="CPARSE002", + ) + return self._function_from_type( + name, + function_type, + direct_function.parameters, + storage, + function_specifiers, + segment, + ) - return_type, function_specifiers = self._build_type(return_spec, return_declarator) - parameters, variadic = self._parse_parameters(parameters_text) + def _function_from_type( + self, + name: str, + function_type: CFunctionType, + parameters: list[CParameter], + storage: list[str], + function_specifiers: list[str], + segment: CTopLevelSegment, + ) -> CFunction: return CFunction( name=name, - return_type=return_type, - parameters=parameters, - storage=list(return_type.storage_class), + result_type=function_type.result_type, + parameters=list(parameters), + storage=list(storage), specifiers=function_specifiers, - variadic=variadic, + is_variadic=function_type.is_variadic, is_definition=segment.terminator == "block", - prototype_style=self._prototype_style(parameters_text), + prototype_style=function_type.prototype_style, source_location=self._source_location(segment), start=self._source_location(segment), end=self._end_location(segment) if segment.terminator == "block" else None, ) - def _parse_declaration(self, segment: CTopLevelSegment) -> tuple[list[CTypedef], list[CGlobal]]: - text = segment.text.strip() - if not text or text.startswith(("struct ", "union ", "enum ")): - return [], [] - if "(" in text or ")" in text: - return [], [] + def _anonymous_tag_id(self, kind: str, segment: CTopLevelSegment) -> str: + filename = segment.filename or "" + return f"{kind}@{filename}:{segment.original_start_line}:{segment.original_start_column}" - declarator_texts = top_level_split(text, ",") - if not declarator_texts: - return [], [] + def _tag_definition_header(self, text: str) -> tuple[str, list[str], str | None] | None: + words = self._specifier_words(text) + for index, word in enumerate(words): + if word not in _TAG_KINDS: + continue + prefix = words[:index] + suffix = words[index + 1 :] + if any(item not in _STORAGE_CLASSES | _TYPE_QUALIFIERS for item in prefix): + return None + if len(suffix) > 1: + return None + return word, prefix, suffix[0] if suffix else None + return None - spec_text, first_declarator = self._split_first_declarator(declarator_texts[0]) - if not spec_text or not first_declarator: - return [], [] + def _forward_tag(self, segment: CTopLevelSegment) -> CStruct | CUnion | None: + words = self._specifier_words(segment.text.strip()) + if len(words) != 2 or words[0] not in {"struct", "union"}: + return None + if words[0] == "struct": + return CStruct( + name=words[1], + is_incomplete=True, + source_location=self._source_location(segment), + ) + return CUnion( + name=words[1], + is_incomplete=True, + source_location=self._source_location(segment), + ) + def _use_aggregate_definition( + self, + type_: CType, + aggregate: CStruct | CUnion | CEnum, + ) -> CType: + if isinstance(type_, CComposedType): + terminal = type_.components[-1] + if isinstance(terminal, (CStruct, CUnion, CEnum)) and not terminal.qualifiers: + type_.components[-1] = aggregate + return type_ + if isinstance(type_, (CStruct, CUnion, CEnum)) and not type_.qualifiers: + return aggregate + return type_ + + def _declarations_from_declarators( + self, + spec_text: str, + declarator_list: str, + segment: CTopLevelSegment, + *, + resolved: CStruct | CUnion | CEnum | None = None, + ) -> tuple[list[CFunction], list[CTypedef], list[CVariable], list[CDiagnostic]]: + functions: list[CFunction] = [] typedefs: list[CTypedef] = [] - globals_: list[CGlobal] = [] - all_declarators = [first_declarator, *declarator_texts[1:]] - - for declarator in all_declarators: - name, declaration = self._declarator_name(declarator) + variables: list[CVariable] = [] + diagnostics: list[CDiagnostic] = [] + + for declarator in top_level_split(declarator_list, ","): + declaration, initializer = top_level_partition(declarator, "=") + try: + name, type_, storage, function_specifiers, direct_function = self._build_declared_type( + spec_text, + declaration, + ) + except _UnsupportedDeclaratorSyntax as error: + diagnostics.append(self._declarator_diagnostic(segment, str(error))) + continue if not name: continue - typeref, _function_specifiers = self._build_type(spec_text, declaration) + if resolved is not None: + type_ = self._use_aggregate_definition(type_, resolved) location = self._source_location(segment) - if "typedef" in typeref.storage_class: - typedefs.append(CTypedef(name=name, type=typeref, source_location=location)) + if "typedef" in storage: + typedefs.append(CTypedef(name=name, type=type_, source_location=location)) + elif isinstance(type_, CFunctionType) and direct_function is not None: + functions.append( + self._function_from_type( + name, + type_, + direct_function.parameters, + storage, + function_specifiers, + segment, + ) + ) + else: + variables.append( + CVariable( + name=name, + type=type_, + storage=storage, + initializer=CInitializer(initializer) if initializer is not None else None, + source_location=location, + ) + ) + + return functions, typedefs, variables, diagnostics + + def _declarator_diagnostic(self, segment: CTopLevelSegment, message: str) -> CDiagnostic: + return CDiagnostic( + code="C_UNSUPPORTED_DECLARATOR", + message=message, + severity="warning", + location=self._source_location(segment), + unit_kind="declarator", + unit_name=None, + ) + + def _field_diagnostic( + self, + segment: CTopLevelSegment, + owner_kind: str, + message: str, + ) -> CDiagnostic: + return CDiagnostic( + code="C_UNSUPPORTED_FIELD_DECLARATION", + message=message, + severity="warning", + location=self._source_location(segment), + unit_kind=f"{owner_kind}_field", + unit_name=None, + ) + + def _parse_fields( + self, + body: str, + segment: CTopLevelSegment, + owner_kind: str, + ) -> tuple[list[CVariable], list[CDiagnostic]]: + members: list[CVariable] = [] + diagnostics: list[CDiagnostic] = [] + for text in top_level_split(body, ";"): + if self._has_unsupported_declaration_marker(text): + diagnostics.append( + self._field_diagnostic( + segment, + owner_kind, + "Declaration attributes and alignment specifiers are not supported in fields yet.", + ) + ) + continue + if "{" in text or "}" in text: + diagnostics.append( + self._field_diagnostic( + segment, + owner_kind, + "Nested aggregate field definitions are not supported yet.", + ) + ) + continue + spec_text, declarator_list = self._split_declaration_specifiers(text) + if not spec_text or not declarator_list: + diagnostics.append( + self._field_diagnostic(segment, owner_kind, "Unsupported field declaration.") + ) + continue + for declarator in top_level_split(declarator_list, ","): + declaration, _initializer = top_level_partition(declarator, "=") + declaration, bit_width = top_level_partition(declaration, ":") + try: + name, type_, _storage, _function_specifiers, _direct_function = self._build_declared_type( + spec_text, + declaration, + ) + except _UnsupportedDeclaratorSyntax as error: + diagnostics.append(self._field_diagnostic(segment, owner_kind, str(error))) + continue + if name is None and bit_width is None: + diagnostics.append( + self._field_diagnostic(segment, owner_kind, "Unnamed field type is not supported.") + ) + continue + members.append( + CVariable( + name=name, + type=type_, + source_location=self._source_location(segment), + bit_width=bit_width, + ) + ) + return members, diagnostics + + def _parse_enumerators(self, body: str, segment: CTopLevelSegment) -> list[CEnumerator]: + constants: list[CEnumerator] = [] + for item in top_level_split(body, ","): + name_text, value = top_level_partition(item, "=") + identifier = self._read_identifier(name_text.strip(), 0) + if identifier is None: + continue + name, end = identifier + if name_text[end:].strip(): + continue + constants.append( + CEnumerator( + name=name, + value=value, + source_location=self._source_location(segment), + ) + ) + return constants + + def _parse_tag_definition( + self, + segment: CTopLevelSegment, + ) -> tuple[ + CStruct | CUnion | CEnum, + list[CFunction], + list[CTypedef], + list[CVariable], + list[CDiagnostic], + ] | None: + text = segment.text.strip() + if self._has_unsupported_declaration_marker(text): + return None + open_index = text.find("{") + if open_index < 0: + return None + close_index = self._find_matching_delimiter(text, open_index, "{", "}") + if close_index is None: + return None + + header = self._tag_definition_header(text[:open_index].strip()) + if header is None: + return None + kind, prefix, tag_name = header + body = text[open_index + 1 : close_index] + declarators = text[close_index + 1 :].strip() + location = self._source_location(segment) + anonymous_id = None if tag_name else self._anonymous_tag_id(kind, segment) + diagnostics: list[CDiagnostic] = [] + + if kind == "enum": + aggregate: CStruct | CUnion | CEnum = CEnum( + name=tag_name, + constants=self._parse_enumerators(body, segment), + anonymous_id=anonymous_id, + source_location=location, + ) + else: + members, diagnostics = self._parse_fields(body, segment, kind) + if kind == "struct": + aggregate = CStruct( + name=tag_name, + members=members, + anonymous_id=anonymous_id, + source_location=location, + ) else: - globals_.append(CGlobal(name=name, type=typeref, source_location=location)) + aggregate = CUnion( + name=tag_name, + members=members, + anonymous_id=anonymous_id, + source_location=location, + ) + + functions: list[CFunction] = [] + typedefs: list[CTypedef] = [] + variables: list[CVariable] = [] + if declarators: + spec_text = " ".join([*prefix, kind, *([tag_name] if tag_name else [])]) + functions, typedefs, variables, declarator_diagnostics = self._declarations_from_declarators( + spec_text, + declarators, + segment, + resolved=aggregate, + ) + diagnostics.extend(declarator_diagnostics) + elif "typedef" in prefix: + diagnostics.append( + CDiagnostic( + code="C_UNSUPPORTED_DECLARATION", + message="Typedef aggregate definition has no alias declarator.", + severity="warning", + location=location, + unit_kind=f"{kind}_typedef", + unit_name=tag_name, + ) + ) + + return aggregate, functions, typedefs, variables, diagnostics + + def _parse_declaration( + self, + segment: CTopLevelSegment, + ) -> tuple[list[CFunction], list[CTypedef], list[CVariable], list[CDiagnostic]]: + text = segment.text.strip() + if ( + not text + or "{" in text + or "}" in text + or text.startswith("_Static_assert") + or self._has_unsupported_declaration_marker(text) + ): + return [], [], [], [] + + spec_text, declarator_list = self._split_declaration_specifiers(text) + if not spec_text or not declarator_list: + return [], [], [], [] + + return self._declarations_from_declarators(spec_text, declarator_list, segment) + + def _unsupported_declaration_diagnostic(self, segment: CTopLevelSegment) -> CDiagnostic | None: + text = segment.text.strip() + if not text: + return None - return typedefs, globals_ + kind = "unsupported_declaration" + message = "Unsupported C declaration form." + + if text.startswith("struct "): + kind = "struct_definition" + message = "Struct definitions are not supported yet." + elif text.startswith("union "): + kind = "union_definition" + message = "Union definitions are not supported yet." + elif text.startswith("enum "): + kind = "enum_definition" + message = "Enum definitions are not supported yet." + elif text.startswith("_Static_assert"): + kind = "static_assert" + message = "Static assertions are recorded but not evaluated." + elif "__attribute__" in text or "__declspec" in text or "[[" in text: + kind = "attribute_declaration" + message = "Compiler-specific declaration attributes are not supported yet." + elif "_Alignas" in text or "alignas" in text: + kind = "alignment_declaration" + message = "Declaration alignment specifiers are not supported yet." + elif "_Atomic(" in text: + kind = "atomic_type_declaration" + message = "_Atomic(type) declarations are not supported yet." + + return CDiagnostic( + code="C_UNSUPPORTED_DECLARATION", + message=message, + severity="warning", + location=self._source_location(segment), + unit_kind=kind, + unit_name=None, + ) def _parse_translation_unit( self, source: str, filename: str | None, - ) -> tuple[list[CFunction], list[CTypedef], list[CGlobal]]: + ) -> tuple[ + list[CFunction], + list[CStruct], + list[CUnion], + list[CEnum], + list[CTypedef], + list[CVariable], + list[CDiagnostic], + ]: self._raise_for_unsupported_old_style_definitions(source, filename) functions: list[CFunction] = [] + structs: list[CStruct] = [] + unions: list[CUnion] = [] + enums: list[CEnum] = [] typedefs: list[CTypedef] = [] - globals_: list[CGlobal] = [] + variables: list[CVariable] = [] + diagnostics: list[CDiagnostic] = [] for segment in split_top_level_c_source(source, filename=filename): - function = self._parse_function(segment) - if function is not None: - functions.append(function) + tag_definition = self._parse_tag_definition(segment) + if tag_definition is not None: + aggregate, parsed_functions, parsed_typedefs, parsed_variables, parsed_diagnostics = tag_definition + if isinstance(aggregate, CStruct): + structs.append(aggregate) + elif isinstance(aggregate, CUnion): + unions.append(aggregate) + else: + enums.append(aggregate) + functions.extend(parsed_functions) + typedefs.extend(parsed_typedefs) + variables.extend(parsed_variables) + diagnostics.extend(parsed_diagnostics) continue if segment.terminator != ";": + try: + function = self._parse_function(segment) + except _UnsupportedDeclaratorSyntax as error: + diagnostics.append(self._declarator_diagnostic(segment, str(error))) + continue + if function is not None: + functions.append(function) + continue + unsupported = self._unsupported_declaration_diagnostic(segment) + if unsupported is not None: + diagnostics.append(unsupported) continue - parsed_typedefs, parsed_globals = self._parse_declaration(segment) + forward_tag = self._forward_tag(segment) + if isinstance(forward_tag, CStruct): + structs.append(forward_tag) + continue + if isinstance(forward_tag, CUnion): + unions.append(forward_tag) + continue + parsed_functions, parsed_typedefs, parsed_variables, declarator_diagnostics = self._parse_declaration( + segment + ) + functions.extend(parsed_functions) typedefs.extend(parsed_typedefs) - globals_.extend(parsed_globals) - - return functions, typedefs, globals_ + variables.extend(parsed_variables) + diagnostics.extend(declarator_diagnostics) + if ( + not parsed_functions + and not parsed_typedefs + and not parsed_variables + and not declarator_diagnostics + ): + unsupported = self._unsupported_declaration_diagnostic(segment) + if unsupported is not None: + diagnostics.append(unsupported) + + return functions, structs, unions, enums, typedefs, variables, diagnostics def _build_project(self, parsed_files: dict[str, CFile]) -> CProject: project = CProject(files=parsed_files) for file in parsed_files.values(): for function in file.functions: project.functions[function.name] = function + for struct in file.structs: + if struct.name is not None: + project.structs[struct.name] = struct + for union in file.unions: + if union.name is not None: + project.unions[union.name] = union + for enum in file.enums: + if enum.name is not None: + project.enums[enum.name] = enum for typedef in file.typedefs: project.typedefs[typedef.name] = typedef - for global_ in file.globals: - project.globals[global_.name] = global_ + for variable in file.variables: + project.variables[variable.name] = variable for macro in file.macros: project.macros[macro.name] = macro for include in file.includes: @@ -526,10 +1264,17 @@ def visit_file( parsed.includes = metadata.includes parsed.macros = metadata.macros parsed.diagnostics = metadata.diagnostics - functions, typedefs, globals_ = self._parse_translation_unit(source, filename) + functions, structs, unions, enums, typedefs, variables, parser_diagnostics = self._parse_translation_unit( + source, + filename, + ) parsed.functions = functions + parsed.structs = structs + parsed.unions = unions + parsed.enums = enums parsed.typedefs = typedefs - parsed.globals = globals_ + parsed.variables = variables + parsed.diagnostics.extend(parser_diagnostics) return parsed def visit_project( diff --git a/docs/c_parser/c_parser_architecture.md b/docs/c_parser/c_parser_architecture.md index 04b842d1f..41ee78267 100644 --- a/docs/c_parser/c_parser_architecture.md +++ b/docs/c_parser/c_parser_architecture.md @@ -4,7 +4,9 @@ Status: partial parser plus raw directive metadata implemented. The `c_parser` package, typed parser models, public entrypoints, explicit `x2py --language c --parse` CLI path, raw include/macro/undef metadata collection, top-level source splitting, and a first simple -declaration/function subset with function-definition start/end locations exist. +declaration/function subset with function-definition start/end locations and +aggregate declarations exist. Declarators are parsed through a recursive +grammar-style path for pointer, array, function, and parenthesized combinations. This document records the target architecture for the C parser frontend in x2py. The initial skeleton has grown into a partial parser, and the remaining @@ -30,11 +32,28 @@ Implemented now: - `c_parser.preprocessor` records raw `#include` directives, simple object-like macros, `#undef` directives, and unsupported function-like macro diagnostics without expanding macros. -- `c_parser.parser` parses simple globals, typedefs, function prototypes, and - function-definition signatures while skipping bodies. Function models include - `prototype_style`; definitions preserve direct `start` and `end` locations - from the signature start through the closing brace; and K&R-style function - definitions raise focused diagnostics. +- `c_parser.parser` parses variables, typedefs, incomplete `struct`/`union` + tags, basic struct/union/enum definitions, function prototypes, and + function-definition signatures while skipping bodies. Declarator handling + follows the C declarator grammar for pointer prefixes, parenthesized direct + declarators, and array/function suffixes, including functions returning + function pointers and arrays of callback pointers. Incomplete tags are + recorded as `CStruct` or `CUnion` objects with `is_incomplete=True`. + Declaration types are represented by concrete `CType` subclasses: + primitives, `CPointer`, `CArray`, `CFunctionType`, and + `CComposedType`. Aggregate members are `CVariable` objects using that same + type path and preserve arrays, callback candidates, and bit-width text. + Inline tag definitions followed by aliases or objects produce concrete + `CTypedef` or `CVariable` records linked to the aggregate object. Function + models expose `result_type` and named `parameters`; their derived + `CFunctionType` is the nameless callable signature. Selected unsupported + declaration forms, including attributes, alignment specifiers, + `_Atomic(type)`, nested aggregate member definitions, and static assertions, + are reported as diagnostics with + explicit `unit_kind` values. A declarator must be fully consumed before a + concrete object is returned; unknown suffixes become diagnostics. Definitions preserve direct + `start` and `end` locations from the signature start through the closing + brace; and K&R-style function definitions raise focused diagnostics. - `c_parser.cli` provides C-specific partial report formatting. - `x2py.cli` dispatches `--language c --parse` to the C parser path. - `--language c --semantics`, `--language c --pyi`, and C wrap-readiness are @@ -48,11 +67,18 @@ Implemented now: Deferred: -- recursive/parenthesized declarator parsing -- function pointer and callback metadata -- struct, union, and enum extraction +- typedef/tag resolution beyond an inline aggregate declaration and callback + policy metadata, for example resolving `size_t count(void);` to a prior + `typedef unsigned long size_t;` +- parameter adjustment and flexible array member classification, for example + `void process(int values[4]);` and + `struct packet { unsigned size; unsigned char data[]; };` +- nested aggregate member definitions, braced initializers, compiler + attributes, alignment specifiers, and `_Atomic(type)` declarations, for + example `struct outer { struct { int x; } inner; };` and + `int values[3] = {1, 2, 3};` - preprocessed-input support, line mapping, and macro-expanded declaration - parsing + parsing, for example `API(int) run(void);` after macro expansion - include graph and project type resolution - C semantic readiness, semantic IR conversion, and `.pyi` output @@ -181,16 +207,17 @@ Current and planned responsibilities: - `c_parser/models.py` - Implemented: typed parser models, `CParseError`, compiler-style - diagnostic rendering, and JSON-stable dataclass serialization. - - Planned: richer source facts for recursive declarators, composite types, - macros/constants, and project indexes. + diagnostic rendering, concrete `CType` composition, and JSON-stable + dataclass serialization. + - Planned: resolved symbol links, richer macro/preprocessing provenance, + parameter-adjustment facts, and project indexes. - `c_parser/lexer.py` - Implemented: safe comment removal that preserves line mapping, logical record folding for backslash-newline, string/character literal awareness, lightweight tokens with source locations, top-level splitting with block end locations, and delimiter splitting aware of nesting and literals. - - Planned: richer token helpers as recursive declarator parsing requires - them. + - Planned: richer token helpers as extension and initializer-expression + parsing require them. - `c_parser/preprocessor.py` - Implemented: lightweight raw directive metadata for includes, object-like macros, `#undef` directives, function-like macro diagnostics, @@ -199,15 +226,14 @@ Current and planned responsibilities: source mapping for preprocessed input. - `c_parser/parser.py` - Implemented: `CParser`, `parse_c_file`, `parse_c_project`, - translation-unit visiting, simple declaration/function visitors, simple - declaration-specifier handling, and simple pointer/array declarator - extraction. Helper methods live on `CParser` rather than as broad - module-level functions. Current function models record prototype-style - versus unspecified empty parameter lists, function definitions preserve - start/end locations, and K&R-style definitions are rejected with - `CParseError`. - - Planned: recursive declarator/function/composite-type visitors and a - richer shared declaration/declarator backend. + translation-unit visiting, declaration/function visitors, grammar-shaped + recursive declarator parsing, concrete `CType` construction, and aggregate + member extraction. Helper methods live on `CParser` rather than as broad + module-level functions. Function models record prototype-style versus + unspecified empty parameter lists, function definitions preserve start/end + locations, and K&R-style definitions are rejected with `CParseError`. + - Planned: symbol resolution, parameter array/function adjustment, and + additional declaration-specifier and extension coverage. - `c_parser/project.py` - Placeholder now. - Planned: file discovery for `.c`, `.h`, and possibly `.i`. @@ -216,11 +242,11 @@ Current and planned responsibilities: - Cross-file type and typedef resolution. - `c_parser/type_resolver.py` - Placeholder now. - - Planned: C primitive type normalization. - - Qualifier/storage-class handling. - - Typedef chain resolution. - - Pointer/array/function-pointer type helpers. - - Safe constant expression folding for simple compile-time values. + - Planned: resolve the concrete primitive/tag/typedef types constructed by + the parser across declarations and files. + - Resolve typedef chains and aggregate references. + - Validate or adjust parameter array/function forms. + - Safely fold simple compile-time constant expressions. - `c_parser/cli.py` - Implemented: report formatting and serialization helpers called by `x2py.cli` behind explicit C flags. @@ -256,87 +282,56 @@ x2py API promise stable C behavior before the frontend matures. ## Core Model Families -Implemented parser models: +All declared types inherit from `CType`, which stores `qualifiers` and +`source_text`. Type qualifiers are concrete values: `CConst`, `CVolatile`, +`CRestrict`, and `CAtomic`. -- `CSourceLocation` - - `filename` - - `line` - - `column` - - `source_line` -- `CDiagnostic` - - `code` - - `message` - - `severity` - - `location` - - `unit_kind` - - `unit_name` -- `CTypeRef` - - `base` - - `qualifiers` - - `storage_class` - - `sign` - - `width` - - `tag_kind` - - `tag_name` - - `typedef_name` - - `pointers` - - `arrays` - - `kind` - - `source_text` - - `resolved` -- `CPointer` - - `qualifiers` -- `CArray` - - `size` - - `static` -- `CParameter` - - `name` - - `type` - - `source_location` -- `CFunction` - - `name` - - `return_type` - - `parameters` - - `storage` - - `specifiers` - - `variadic` - - `is_definition` - - `prototype_style` - - `source_location` - - `start` - - `end` -- `CField` - - `name` - - `type` - - `source_location` -- `CStruct` - - `name` - - `fields` - - `anonymous_id` - - `opaque` - - `source_location` -- `CUnion` - - `name` - - `fields` - - `anonymous_id` - - `source_location` -- `CEnum` - - `name` - - `constants` - - `anonymous_id` - - `source_location` -- `CEnumerator` - - `name` - - `value` - - `source_location` -- `CTypedef` - - `name` - - `type` - - `source_location` -- `CGlobal` - - `name` - - `type` - - `source_location` +The implemented primitive `CType` subclasses are: + +- `CVoid`, `CBool`, `CChar`, `CSignedChar`, and `CUnsignedChar` +- `CShort`, `CUnsignedShort`, `CInt`, and `CUnsignedInt` +- `CLong`, `CUnsignedLong`, `CLongLong`, and `CUnsignedLongLong` +- `CFloat`, `CDouble`, and `CLongDouble` +- `CFloatComplex`, `CDoubleComplex`, and `CLongDoubleComplex` + +Derived and named `CType` subclasses are: + +- `CPointer`, whose qualifiers apply to that pointer component +- `CArray`, with `bound`, `is_static_minimum`, `is_variable_length`, and + `is_flexible`; bound/static/VLA metadata is populated now, while + flexible-array-member parsing and validation are deferred +- `CFunctionType`, the nameless callable signature with `result_type`, + `parameter_types`, `is_variadic`, and `prototype_style` +- `CComposedType`, whose `components` are read from the declared name outward +- `CStruct`, `CUnion`, and `CEnum`, which are tag types as well as aggregate + declaration objects +- `CTypedef`, which represents either a declared alias with its underlying + `type`, or an unresolved typedef-name use until symbol resolution is added +- `CUnknownType`, which preserves an unrecognized type spelling + +For example, composition order distinguishes the following declarations: + +```python +int *values[4]; # CComposedType([CArray(bound="4"), CPointer(), CInt()]) +int (*matrix)[4]; # CComposedType([CPointer(), CArray(bound="4"), CInt()]) +int *(*table)[4]; # CComposedType([CPointer(), CArray(bound="4"), CPointer(), CInt()]) +``` + +Declaration objects are separate from the type components: + +- `CVariable` has `name`, `type`, `storage`, optional `initializer`, optional + `bit_width`, and source/callback metadata. Struct and union `members` are + also `CVariable` objects; there is no separate field class. +- `CFunction` has `name`, `result_type`, named `parameters`, storage and + function specifiers, `is_variadic`, prototype style, and source/definition + locations. Its `type` property builds the corresponding nameless + `CFunctionType`. +- `CParameter` has a source name, a `type`, and a reserved `declared_type` for + later C parameter adjustment handling. +- `CInitializer` preserves initializer source text without claiming evaluation. +- `CStruct` and `CUnion` expose `members` and `is_incomplete`; `CEnum` exposes + `constants`; `CEnumerator` preserves enumerator name and value text. +- `CTypedef` has its alias name and declared `type`. - `CMacro` - `name` - `value` @@ -357,7 +352,7 @@ Implemented parser models: - `unions` - `enums` - `typedefs` - - `globals` + - `variables` - `macros` - `includes` - `diagnostics` @@ -368,14 +363,20 @@ Implemented parser models: - `unions` - `enums` - `typedefs` - - `globals` + - `variables` - `macros` - `includes` -Future parser phases can add fields such as bit widths, function pointer -metadata, conditional-region metadata, include graphs, and project diagnostics -when the corresponding behavior lands. Additions should be documented and -tested with stable serialization expectations. +Serialization uses `"model"` to identify concrete `CType` nodes; `"type"` is +reserved for semantic type relationships such as `CVariable.type` and +`CTypedef.type`. Concrete qualifier objects serialize using their canonical +spellings, such as `"const"`. Reused aggregate/typedef objects serialize as +references to avoid cycles. + +Future parser phases can add symbol links, parameter adjustment, conditional +region metadata, include graphs, and project diagnostics when the corresponding +behavior lands. Additions should be documented and tested with stable +serialization expectations. ## Grammar-Style Parsing Strategy @@ -407,21 +408,24 @@ be: - typedef - function prototype - function definition - - struct/union/enum definition - - global variable/static const + - forward struct declaration or struct/union/enum definition + - file-scope variable/static const - unsupported/macro-dependent declaration 6. Dispatch to a small visitor for that declaration kind. 7. Use a shared declaration-specifier and declarator parser to build type - references for functions, parameters, fields, globals, and typedefs. + references for functions, parameters, members, variables, and typedefs. 8. Ignore executable function bodies except where needed to find the matching brace and preserve function start/end locations. ## Declarator-Centered Design -C type syntax is declarator-centered. The future parser should make declarator -parsing a first-class subsystem, not a pile of ad hoc string splitting. +C type syntax is declarator-centered. Declarator parsing stays a first-class +subsystem, not a pile of ad hoc string splitting. The current parser applies +one recursive declarator path to variables, typedefs, function signatures, +parameters, and aggregate members; future work should extend that grammar-shaped +path rather than adding parallel splitting rules. -Required layered pieces: +Layered pieces: - declaration specifier parser - storage classes: `extern`, `static`, `typedef`, `register`, `_Thread_local` @@ -441,7 +445,7 @@ Required layered pieces: - anonymous abstract declarators where needed - entity applier - convert one declaration specifier plus one declarator into a typed model - - reuse this for function returns, parameters, fields, globals, and typedefs + - reuse this for function returns, parameters, members, variables, and typedefs This is the C equivalent of the Fortran parser's shared declaration backend. @@ -516,7 +520,7 @@ Current behavior: - Returned `CProject` objects contain `CFile` parser models with raw include, macro, metadata diagnostics, and supported declarations populated per file. - Basic project-level indexes are populated for parsed functions, typedefs, - globals, macros, and includes. + variables, macros, and includes. - Include graphs, duplicate analysis, and type resolution are not populated yet. Planned behavior after project-resolution phases: @@ -561,9 +565,10 @@ These should become parser diagnostics or parser metadata, not a parser-side ## Parser-First Model Policy For the current planning horizon, the C parser should store C-specific facts in -`c_parser/models.py`. This includes preprocessing origin, macro dependencies, -function pointer signatures, callback-like parameters, pointer qualifiers, -ownership ambiguity, include dependencies, and typedef resolution state. +`c_parser/models.py`. It currently stores function-pointer signatures, +callback-candidate markers, type qualifiers, raw include dependencies, and +unresolved typedef/tag uses. Later phases should add preprocessing origin, +macro dependencies, ownership policy, and resolved symbol links. Semantic IR conversion is deliberately later work. When that phase starts, the IR model may need extensions for C pointer ownership, unsigned integer types, diff --git a/docs/c_parser/c_parser_cli_workflow.md b/docs/c_parser/c_parser_cli_workflow.md index 6b8866531..7d30a3449 100644 --- a/docs/c_parser/c_parser_cli_workflow.md +++ b/docs/c_parser/c_parser_cli_workflow.md @@ -2,13 +2,14 @@ Status: C parser partial subset plus raw directive metadata implemented. The CLI command shape exists and parse reports can include raw includes, simple -macros, `#undef` provenance, metadata diagnostics, simple globals, typedefs, -function prototypes, prototype-style metadata, and function-definition -signatures with start/end locations. +macros, `#undef` provenance, metadata diagnostics, variables, typedefs, +aggregate declarations, function prototypes, prototype-style metadata, and +function-definition signatures with start/end locations. Declarator output can +represent parenthesized pointer/array precedence through concrete +`CComposedType` components and nameless `CFunctionType` signatures. -The C parser CLI workflow should be designed before parser implementation so -future parser work lands behind a stable command shape, output schema, and -diagnostic contract. +This document records the implemented C parse command shape, output schema, +and diagnostic contract, plus deferred CLI behavior. ## Current Status @@ -27,10 +28,19 @@ behavior. The C parser output differs from Fortran parser output by using C-specific top-level sections: `functions`, `structs`, `unions`, `enums`, `typedefs`, -`globals`, `macros`, `includes`, and `diagnostics`. The current partial parser -can populate `functions`, `typedefs`, and `globals` for the supported subset, -while composite type sections remain empty. Raw `includes`, `macros`, and -metadata `diagnostics` can also be populated. The parser reports +`variables`, `macros`, `includes`, and `diagnostics`. The current partial parser +can populate `functions`, `typedefs`, `variables`, `structs`, `unions`, and +`enums` in the supported subset. Typedefs, variables, parameters, and aggregate +members can include concrete composed types for pointer/array/function forms, +including function pointers and functions returning function pointers. Raw +`includes`, `macros`, and metadata `diagnostics` can also be +populated. The object class distinguishes declarations (`CFunction`, +`CVariable`, `CTypedef`, `CStruct`, `CUnion`, or `CEnum`), and incomplete tag +declarations set `is_incomplete=True`. +Known unsupported declaration forms such as declaration attributes, alignment +specifiers, `_Atomic(type)`, nested aggregate member definitions, and static assertions are +reported in diagnostics with explicit `unit_kind` values; unconsumed declarator +suffixes are diagnosed instead of silently omitted. The parser reports `parser_status: "partial"`. C parse diagnostics, currently including unsupported K&R-style function definitions, honor `--no-color` and `NO_COLOR=1`. Function definitions do not store executable body text; they preserve a @@ -117,7 +127,7 @@ Auto-detection should wait until C parser behavior is mature enough to handle mixed source trees predictably. Until then, `--parse` without `--language` should keep existing Fortran behavior. -## Planned Flags +## Flags And Deferred Options Initial flags: @@ -181,7 +191,7 @@ File: include/example.h Unions: 0 Enums: 0 Typedefs: 0 - Globals: 0 + Variables: 0 Macros: 0 Includes: 0 Diagnostics: 0 @@ -200,15 +210,18 @@ JSON output for a file without raw directives: "functions": [ { "name": "run", - "return_type": { - "base": "int" + "result_type": { + "model": "CInt", + "qualifiers": [], + "source_text": "int" }, "parameters": [], "storage": [], "specifiers": [], - "variadic": false, + "is_variadic": false, "is_definition": false, "prototype_style": "prototype", + "source_location": {"filename": "include/example.h", "line": 1, "...": "..."}, "start": {"filename": "include/example.h", "line": 1, "...": "..."}, "end": null } @@ -217,7 +230,7 @@ JSON output for a file without raw directives: "unions": [], "enums": [], "typedefs": [], - "globals": [], + "variables": [], "macros": [], "includes": [], "diagnostics": [] @@ -246,8 +259,8 @@ fields still point back to the original `.h` or `.c` file. ## JSON Parse Schema -The C parse JSON should be per-file and should not reuse Fortran key names when -the concepts differ. Proposed top-level per-file keys: +The C parse JSON is per-file and does not reuse Fortran key names when the +concepts differ. Current top-level per-file keys: ```text language @@ -259,13 +272,15 @@ structs unions enums typedefs -globals +variables macros includes diagnostics ``` -Every model should include source-location metadata once implementation begins: +Declaration/directive records include `source_location`; diagnostics use +`location`. Concrete type components preserve `source_text` rather than their +own source-location object: ```text source_location: { @@ -274,12 +289,26 @@ source_location: { column: int | null, source_line: str | null } + +location: { + filename: str | null, + line: int | null, + column: int | null, + source_line: str | null +} ``` +Concrete `CType` values serialize with a `"model"` discriminator, for example +`{"model": "CInt", "qualifiers": [], "source_text": "int"}`. The `"type"` +key is reserved for a semantic relationship such as a `CVariable` or +`CTypedef` pointing to its declared type. Qualifier objects serialize as +canonical spelling strings such as `"const"`. Aggregate/typedef object reuse +emits a reference rather than a recursive JSON cycle. + ## Human Tree Output -The tree should mirror the Fortran parser style: compact by default, expanded -with explicit flags. +The current human tree is the count-only report shown above. A later expanded +tree could mirror the Fortran parser style: Initial mature output shape: @@ -292,7 +321,7 @@ File: src/api.c - int add(int a, int b) - void scale(double *x, size_t n) Structs: 1 - - struct vector (fields=2) + - struct vector (members=2) Typedefs: 1 - vector_t -> struct vector Macros: 1 @@ -304,8 +333,8 @@ documentation, not in the parser CLI workflow. ## Diagnostics Behavior -C parser errors should use a `CParseError` model with the same user experience -as `FortranParseError`: +Fatal C syntax errors use `CParseError` with the same user experience as +`FortranParseError`: ```text src/api.h:12:5: error[CPARSE001]: Unsupported declaration. @@ -321,6 +350,9 @@ Default CLI behavior: - do not show Python traceback - colorize when color is enabled +Unsupported but recoverable declarations and raw preprocessor limitations are +stored as non-fatal `CDiagnostic` entries in the parse report instead. + Debug behavior: - `--debug-traceback` re-raises the error @@ -349,9 +381,11 @@ The active CLI/parser tests cover the current partial subset: - `--language c --parse --debug-traceback` is accepted. - raw comment stripping, line-continuation folding, top-level splitting, include collection, simple macro collection, function-like macro diagnostics, - conditional non-selection, simple declarations, globals, typedefs, and - function signatures with definition start/end locations are covered by focused C - tests. + conditional non-selection, simple declarations, variables, typedefs, + parenthesized declarators, function pointer typedefs/parameters, recursive + declarator combinations, concrete declaration objects, aggregate + definitions/members/enumerators, incomplete struct/union tags, and function + signatures with definition start/end locations are covered by focused C tests. - `--show-vars` and `--print-limit` are rejected in C mode until C-specific display controls exist. - `--semantics` with `--language c` is rejected until C semantic conversion is @@ -373,10 +407,20 @@ Completed order: continuations, includes, simple macros, and unsupported function-like macros. 7. Added top-level splitting and a first partial grammar subset for simple - globals, typedefs, function prototypes, and function-definition headers. + variables, typedefs, function prototypes, and function-definition headers. 8. Added function-definition start/end locations while continuing to skip executable bodies. - -Next implementation work should continue with richer declarator support, -preprocessed-input line mapping, composite types, and project resolution while -keeping the explicit `--language c` gate in place. +9. Added forward `struct name;` extraction as incomplete `CStruct` source facts. +10. Replaced ad hoc declarator splitting with a recursive grammar-style + declarator parser for pointer prefixes, parenthesized direct declarators, + and array/function suffixes. +11. Classified each declarator from its recursive type and added basic + struct/union/enum declarations, members, tag typedefs, and trailing tag + variables as concrete parser models. +12. Replaced generic type references and declaration-kind tags with concrete + `CType` subclasses, `CComposedType` components, and concrete declaration + objects. + +Next implementation work should continue with tag/typedef resolution, +preprocessed-input line mapping, compiler extension policy, and project +resolution while keeping the explicit `--language c` gate in place. diff --git a/docs/c_parser/c_parser_implementation_checklist.md b/docs/c_parser/c_parser_implementation_checklist.md index 671ac7f0d..c7d0be2d0 100644 --- a/docs/c_parser/c_parser_implementation_checklist.md +++ b/docs/c_parser/c_parser_implementation_checklist.md @@ -4,9 +4,14 @@ Status: implementation checklist with Phase 1 skeleton, selected Phase 2 fixture scaffolding, selected Phase 3 model/error work, Phase 4 raw lexer/directive metadata, and a first Phase 5/6 partial declaration/function subset complete. The `c_parser` package and explicit C -parse path exist, and simple globals, typedefs, function prototypes, -function-definition signatures, and function-definition start/end locations are now -parsed. +parse path exist, and simple variables, typedefs, function prototypes, +function-definition signatures, function-definition start/end locations, and +incomplete `struct`/`union` declarations and basic aggregate definitions are +now parsed. Declarators use a recursive grammar-style parser for pointer, +array, function, and parenthesized combinations. Declaration types are concrete +`CType` subclasses combined by `CComposedType`; aggregate members are +`CVariable` objects using the same declared-type path. Selected unsupported +extensions are diagnosed. This checklist is intentionally detailed so future work can proceed one branch, one checklist item, and one tested capability at a time. The C parser initiative @@ -15,12 +20,19 @@ stable. ## Progress Snapshot -- Last updated: 2026-05-22 -- Checklist progress: 445/848 checked (52.5%). +- Last updated: 2026-05-23 +- Checklist progress: 512/848 checked (60.4%). - Current parser status: partial C parser with raw directive metadata, top-level - source splitting, simple declarations/globals/typedefs, prototype-style + source splitting, simple declarations/variables/typedefs, prototype-style metadata, K&R diagnostics, simple function signatures, and start/end - locations for function definitions. + locations for function definitions. Incomplete struct/union declarations are + recorded as `CStruct`/`CUnion` values with `is_incomplete=True`; named tags + are indexed at project level. Parenthesized declarators, function pointer + typedefs/parameters, callback members, and functions returning function + pointers are represented with concrete `CType` objects and + `CComposedType.components`. Primitive specifiers have concrete type classes, + and functions, variables, typedefs, and aggregates are distinguished by + their concrete declaration objects rather than a kind field. ## Global Rules @@ -34,7 +46,7 @@ stable. - [x] Gate all integration through explicit C flags or C-specific APIs. - [ ] Keep any C wrappability assessment in the semantic layer, not inside the parser package. -- [ ] Implement C parsing as a grammar-style recursive parser with scoped +- [x] Implement C parsing as a grammar-style recursive parser with scoped visitors, source slicing, shared declaration/declarator parsing helpers, and typed model objects. - [x] Store initial C-specific facts in `c_parser/models.py`; defer semantic IR @@ -227,7 +239,7 @@ Scope: - [x] Return empty `unions` list. - [x] Return empty `enums` list. - [x] Return empty `typedefs` list. -- [x] Return empty `globals` list. +- [x] Return empty `variables` list. - [x] Return `macros` list, empty until raw macro directives are found. - [x] Return `includes` list, empty until raw include directives are found. - [x] Return `diagnostics` list, empty unless skeleton or raw metadata @@ -454,27 +466,30 @@ Scope: - [x] Define `CSourceLocation`. - [x] Define `CDiagnostic`. -- [x] Define `CTypeRef`. -- [x] Define `CPointer`. -- [x] Define `CArray`. +- [x] Define concrete `CType` subclasses, including all supported primitive + scalar and complex types. +- [x] Define `CQualifier` objects (`CConst`, `CVolatile`, `CRestrict`, and + `CAtomic`). +- [x] Define `CPointer`, `CArray`, `CFunctionType`, and `CComposedType`. - [x] Define `CParameter`. - [x] Define `CFunction`. -- [x] Define `CField`. +- [x] Use `CVariable` for struct and union members rather than a separate + field object. - [x] Define `CStruct`. - [x] Define `CUnion` or use `CStruct(is_union=True)`. - [x] Define `CEnum`. - [x] Define `CEnumerator`. - [x] Define `CTypedef`. -- [x] Define `CGlobal`. +- [x] Define `CVariable`. - [x] Define `CMacro`. - [x] Define `CInclude`. - [x] Define `CFile`. - [x] Define `CProject`. - [x] Add helper properties for pointer depth. - [x] Add helper properties for array rank. -- [x] Add helper properties for effective type text. -- [x] Add helper properties for `is_const_pointer`. -- [x] Add helper properties for `is_opaque_type`. +- [x] Preserve effective declaration type source text. +- [x] Store qualifiers on the exact `CType` component they qualify. +- [x] Record incomplete aggregate tags with `is_incomplete=True`. - [x] Add helper properties for source-location display. ### Serialization Tasks @@ -486,7 +501,7 @@ Scope: - [x] Ensure dataclass defaults produce stable JSON. - [x] Add tests for empty `CFile` serialization. - [ ] Add tests for each model's minimal JSON shape. -- [ ] Add tests for source-location serialization. +- [x] Add tests for source-location serialization. - [ ] Add tests that unknown/unresolved metadata is preserved. ### Public API Skeleton And Partial Parser Tasks @@ -519,10 +534,10 @@ Scope: ### Phase 3 Risks And Open Questions - [x] Decide whether `CUnion` should subclass/share `CStruct`. -- [ ] Decide how to represent anonymous structs/unions/enums. +- [x] Decide how to represent anonymous structs/unions/enums. - [ ] Decide whether macros belong in `CFile.macros` only or also symbols. -- [ ] Decide whether `CTypeRef` should be a single recursive model or contain - normalized pointer/array/function layers. +- [x] Decide on concrete `CType` objects and name-outward + `CComposedType.components` instead of a kind-driven type reference. ## Phase 4: Lexer And Lightweight Preprocessor @@ -692,41 +707,63 @@ Scope: - [x] Parse array declarators. - [x] Parse multidimensional array declarators. - [x] Parse static array parameter qualifiers, for example `int a[static 4]`. -- [ ] Parse parenthesized declarators. -- [ ] Parse function declarators. -- [ ] Parse function pointer declarators. +- [x] Parse parenthesized declarators. +- [x] Parse function declarators. +- [x] Parse function pointer declarators. - [x] Parse abstract declarators where needed for unnamed parameters. - [x] Parse multiple declarators in one declaration. - [x] Keep declarator entity order stable. - [x] Preserve original declarator source text. - [x] Add source locations for each declared entity. -- [ ] Reject or diagnose unsupported declarator forms explicitly. +- [x] Reject or diagnose unsupported declarator forms explicitly. ### Shared Declaration Backend Tasks - [x] Implement a helper analogous to `_helper_parse_declaration_line`. - [x] Feed procedure parameters through the same declaration backend. - [x] Feed function return types through the same declaration backend. -- [ ] Feed struct/union fields through the same declaration backend. +- [x] Feed struct/union members through the same declaration backend. - [x] Feed typedefs through the same declaration backend. -- [x] Feed global variables/constants through the same declaration backend. +- [x] Feed file-scope variables/constants through the same declaration backend. - [x] Apply declaration specifiers to declarator-derived type layers. -- [x] Normalize C type spelling into `CTypeRef`. +- [x] Normalize supported C type spelling into concrete `CType` subclasses. - [x] Preserve typedef references before project resolution. -- [ ] Add tests for each declaration role. +- [x] Add tests for each declaration role. - [x] Add tests for declarations with multiple variables. - [x] Add tests for declarations with initializers. - [x] Add tests that local executable statements are not parsed as declarations. - [x] Add exhaustive tests for every supported storage class. -- [ ] Add exhaustive tests for every supported type qualifier. -- [ ] Add exhaustive tests for every supported primitive spelling. +- [x] Add exhaustive tests for every supported type qualifier. +- [x] Add exhaustive tests for every supported primitive spelling. - [x] Add tests for typedef-name references outside `size_t`-style examples. - [x] Add tests for `struct name`, `union name`, and `enum name` references in - globals and parameters. + variables and parameters. - [x] Add tests for multidimensional arrays. - [ ] Add diagnostics for declarations ignored by the current partial parser. - [ ] Add structured source facts for declarations that depend on macros. +Known declaration implementation gaps, with representative syntax: + +- parameter array/function adjustment: + `void process(int values[4], int callback(int));` +- flexible array member classification and validation: + `struct packet { unsigned size; unsigned char data[]; };` +- braced/designated initializer preservation: + `int values[3] = {1, 2, 3};` +- nested anonymous aggregate members: + `struct outer { struct { int x; } inner; };` +- cross-declaration resolution/conflict behavior: + `typedef unsigned long size_t; size_t count(void);` +- preprocessed declarations with line mapping: + `#define API(ret) ret` followed by `API(int) run(void);` + +Represented shapes still needing dedicated active regression tests: + +- multi-level qualifier placement: + `const int * const * volatile chain;` +- unnamed and zero-width bit-fields: + `struct flags { unsigned : 0; unsigned mode : 3; };` + ### Phase 5 Definition Of Done - [x] Shared declaration/declarator parser exists. @@ -737,10 +774,10 @@ Scope: ### Phase 5 Risks And Open Questions -- [ ] Function pointer parsing is complex; decide which forms are supported in - extraction and which only produce diagnostics. -- [ ] Typedef-name recognition may require project/type context; decide how to - represent unresolved names before Phase 8. +- [x] Represent the currently supported pointer/array/function declarator + combinations as concrete type components and diagnose unconsumed forms. +- [x] Represent unresolved typedef-name uses as `CTypedef` type objects until + project/type resolution is implemented. ## Phase 6: Function Parsing @@ -767,7 +804,7 @@ Scope: - [x] Mark `is_variadic`. - [x] Parse pointer parameters. - [x] Parse array parameters. -- [ ] Parse function pointer parameters. +- [x] Parse function pointer parameters. - [x] Parse `const` parameters. - [x] Parse `restrict` parameters. - [x] Parse `volatile` parameters. @@ -780,13 +817,13 @@ Scope: - [x] Add tests for pointer and array parameters. - [x] Add tests for const pointer variants. - [x] Add tests for variadic prototypes. -- [ ] Add tests for function pointer parameters. +- [x] Add tests for function pointer parameters. - [x] Add model field for prototype style so `int f(void)` and `int f()` can be distinguished. - [x] Add tests that distinguish explicit `void` parameter lists from unspecified empty parameter lists. -- [ ] Add parser source facts or diagnostics for variadic functions. -- [ ] Add parser source facts for callback candidates once function pointer +- [x] Add parser source facts for variadic functions through `is_variadic`. +- [x] Add parser source facts for callback candidates once function pointer parameters are supported. ### Function Definition Tasks @@ -826,7 +863,8 @@ Scope: - [x] Basic C function signatures parse from `.h` and `.c`. - [x] Function bodies are skipped safely. -- [ ] Variadic and function pointer cases are represented and diagnosed. +- [x] Variadic and supported function pointer cases are represented as typed + source facts. - [x] CLI human and JSON output show functions. - [ ] Parser diagnostics report no-functions only when appropriate. @@ -849,79 +887,81 @@ Scope: ### Struct Tasks -- [ ] Parse named `struct name { ... };`. -- [ ] Parse forward declaration `struct name;`. -- [ ] Parse anonymous `struct { ... }`. -- [ ] Parse typedef anonymous struct `typedef struct { ... } name;`. -- [ ] Parse typedef named struct `typedef struct tag name;`. -- [ ] Parse fields with shared declaration backend. -- [ ] Parse pointer fields. -- [ ] Parse array fields. -- [ ] Parse nested anonymous structs as unsupported or metadata. -- [ ] Parse bitfields as metadata with semantic limitations. -- [ ] Preserve field order. -- [ ] Preserve source locations. -- [ ] Mark incomplete structs. -- [ ] Add tests for named structs. -- [ ] Add tests for forward declarations. -- [ ] Add tests for typedef structs. -- [ ] Add tests for pointer fields. -- [ ] Add tests for array fields. +- [x] Parse named `struct name { ... };`. +- [x] Parse forward declaration `struct name;`. +- [x] Parse anonymous `struct { ... }`. +- [x] Parse typedef anonymous struct `typedef struct { ... } name;`. +- [x] Parse typedef named struct `typedef struct tag name;`. +- [x] Parse members with shared declaration backend. +- [x] Parse pointer members. +- [x] Parse array members. +- [x] Parse nested anonymous structs as unsupported or metadata. +- [x] Parse bit-fields as member metadata with semantic limitations. +- [x] Preserve member order. +- [ ] Preserve precise per-member source locations; members currently carry + the enclosing aggregate declaration location. +- [x] Mark incomplete structs. +- [x] Add tests for named structs. +- [x] Add tests for forward declarations. +- [x] Add tests for typedef structs. +- [x] Add tests for pointer members. +- [x] Add tests for array members. - [ ] Add tests for bitfield diagnostics. ### Union Tasks -- [ ] Parse named unions. -- [ ] Parse forward union declarations. -- [ ] Parse anonymous unions. -- [ ] Parse typedef unions. -- [ ] Parse union fields with shared declaration backend. -- [ ] Mark union fields distinctly from struct fields. +- [x] Parse named unions. +- [x] Parse forward union declarations. +- [x] Parse anonymous unions. +- [x] Parse typedef unions. +- [x] Parse union members with shared declaration backend. +- [x] Retain union member ownership through the containing `CUnion`. - [ ] Add diagnostics for by-value unions if unsafe. -- [ ] Add tests for named unions. -- [ ] Add tests for typedef unions. +- [x] Add tests for named unions. +- [x] Add tests for typedef unions. - [ ] Add tests for union diagnostics. ### Enum Tasks -- [ ] Parse named enums. -- [ ] Parse anonymous enums. -- [ ] Parse typedef enums. -- [ ] Parse enumerator names. -- [ ] Parse explicit enumerator values. -- [ ] Preserve symbolic enumerator values. +- [x] Parse named enums. +- [x] Parse anonymous enums. +- [x] Parse typedef enums. +- [x] Parse enumerator names. +- [x] Parse explicit enumerator values. +- [x] Preserve symbolic enumerator values. - [ ] Safely fold simple integer expressions. -- [ ] Preserve expression text when folding is unsafe. -- [ ] Add tests for plain enums. -- [ ] Add tests for explicit values. -- [ ] Add tests for expression values. -- [ ] Add tests for typedef enums. +- [x] Preserve expression text when folding is unsafe. +- [x] Add tests for plain enums. +- [x] Add tests for explicit values. +- [x] Add tests for expression values. +- [x] Add tests for typedef enums. ### Typedef Tasks - [x] Parse primitive typedefs. - [x] Parse pointer typedefs. - [x] Parse array typedefs. -- [ ] Parse function pointer typedefs. -- [ ] Parse struct/union/enum typedefs. -- [ ] Preserve alias chains before resolution. +- [x] Parse function pointer typedefs. +- [x] Parse struct/union/enum typedefs. +- [x] Preserve alias chains before resolution. - [ ] Detect duplicate typedefs in same scope. -- [ ] Add tests for typedef chains. +- [x] Add tests for typedef chains. - [x] Add tests for primitive typedefs. -- [ ] Add tests for opaque handle typedefs. +- [x] Add tests for opaque handle typedefs. - [ ] Add tests for function pointer typedef diagnostics. ### Phase 7 Definition Of Done -- [ ] C composite and typedef models are populated from basic fixtures. -- [ ] Shared declaration backend handles fields and typedefs. -- [ ] Incomplete/anonymous/bitfield cases are represented or diagnosed. +- [x] C composite and typedef models are populated from basic fixtures. +- [x] Shared declaration backend handles members and typedefs. +- [ ] Validate remaining bit-field/flexible-member cases beyond the basic + incomplete, anonymous, and named bit-field forms already represented. - [ ] JSON goldens cover composite type schema. -- [ ] Docs list supported and unsupported composite forms. +- [x] Docs list supported and unsupported composite forms. ### Phase 7 Risks And Open Questions -- [ ] Decide when anonymous structs should become generated internal names. +- [x] Decide when anonymous structs should become generated internal names. - [ ] Decide how much enum expression folding is safe without compiler semantics. - [ ] Decide how unions map to semantic IR, if at all in v1. @@ -968,18 +1008,18 @@ Scope: - [x] Index functions by name. - [ ] Index functions by file. - [x] Index typedefs by name. -- [ ] Index struct tags by tag namespace. -- [ ] Index union tags by tag namespace. -- [ ] Index enum tags by tag namespace. +- [x] Index struct tags by tag namespace. +- [x] Index union tags by tag namespace. +- [x] Index enum tags by tag namespace. - [ ] Index enum constants in ordinary identifier namespace. - [x] Index macros/constants separately. -- [x] Index globals by name. +- [x] Index variables by name. - [ ] Detect duplicate definitions. - [ ] Distinguish compatible redeclarations from conflicts. - [ ] Add tests for duplicate handling. - [ ] Add tests for project-level function indexes. - [ ] Add tests for project-level typedef indexes. -- [ ] Add tests for project-level global indexes. +- [ ] Add tests for project-level file-scope variable indexes. - [ ] Add tests for project-level macro indexes. ### Type Resolution Tasks @@ -1087,7 +1127,7 @@ Scope: - [ ] Assign union field blockers to union units. - [ ] Assign enum value blockers to enum units. - [ ] Assign typedef blockers to typedef units. -- [ ] Assign include/macro/global blockers to file units when no narrower owner +- [ ] Assign include/macro/file-scope-variable blockers to file units when no narrower owner exists. - [ ] Use qualified names where available. - [ ] Avoid per-unit ready flags; keep readiness file-level like Fortran. @@ -1280,8 +1320,8 @@ Scope: - [ ] Treat cJSON corpus tests as parse-only at first. - [ ] Use cJSON to exercise typedef structs, recursive pointers, `const char *` APIs, `size_t`, public declaration macros, numeric/string constants, - and callback hook fields. -- [ ] Keep callback hook fields modeled in `c_parser/models.py`, with + and callback hook members. +- [ ] Keep callback hook members modeled in `c_parser/models.py`, with semantic readiness requiring explicit user `.pyi` callback policy metadata before claiming they are wrappable. - [ ] Add zlib or another macro-heavier C library only after diff --git a/docs/c_parser/c_parser_reference.md b/docs/c_parser/c_parser_reference.md index 290a130e6..83eedf961 100644 --- a/docs/c_parser/c_parser_reference.md +++ b/docs/c_parser/c_parser_reference.md @@ -3,8 +3,11 @@ Status: partial parser reference with raw directive metadata. The `c_parser` package and explicit C CLI parse path exist, raw includes/simple macros are recorded, and a first grammar-shaped subset parses simple declarations, -typedefs, globals, function prototypes, and function-definition headers with -start/end locations. +typedefs, variables, function prototypes, and function-definition headers with +start/end locations. Incomplete struct/union declarations and basic +struct/union/enum definitions are represented as concrete objects, and +declarators are parsed through a recursive grammar-style path for pointer, +array, function, and parenthesized combinations. This document is the future home for the C parser user and developer reference. It should evolve into the C equivalent of `fortran_parser.md` as implementation @@ -63,8 +66,19 @@ Implemented: - simple object-like `#define` macro collection - function-like macro metadata with unsupported diagnostics - raw `#undef` directive provenance in macro metadata -- simple primitive, pointer, array, and qualifier type extraction -- simple global variable and `typedef` extraction +- concrete primitive `CType` objects, pointer/array composition, and concrete + qualifier objects +- recursive declarator extraction for parenthesized pointer/array precedence +- nameless `CFunctionType` signatures for function pointer typedefs and + parameter source facts +- simple file-scope variable and `typedef` extraction +- incomplete `struct name;` and `union name;` extraction as concrete tag types + with `is_incomplete=True` +- named and anonymous struct/union/enum definitions +- aggregate member extraction as `CVariable` objects through the declarator backend, including pointer, + array, callback-pointer, and bitfield source facts +- inline tag typedef aliases and trailing tag object declarators as separate + concrete models - simple function prototype extraction - prototype-style metadata distinguishing `int f(void)` from `int f()` - simple function-definition signature extraction with body skipping @@ -74,11 +88,13 @@ Implemented: - C fixture inputs under `tests/data/c/general/` plus C fixture directory scaffolding for errors, corpus, and scientific APIs -Placeholder only: +Still deferred: -- recursive declarator models for parenthesized pointer/array distinctions -- function pointer declarators and callback metadata -- structs, unions, enums, and complex typedef parsing +- callback policy metadata beyond parser-side callback candidates +- nested aggregate member definitions and broad compiler-extension declarators +- parameter array/function adjustment, flexible-array-member validation, and + braced/designated initializer preservation +- cross-declaration and cross-file typedef/tag resolution - project include graph and cross-file type resolution - preprocessed-input parsing with `#line`/linemarker source mapping - macro-expanded declaration parsing from preprocessed input @@ -92,7 +108,7 @@ The initial supported subset should focus on stable wrapper-relevant APIs: - function definitions with extractable signatures - primitive C scalar types - pointers -- arrays in parameters and fields +- arrays in parameters and aggregate members - `const`, `restrict`, and `volatile` qualifiers - `static` and `extern` storage classes where wrapper-relevant - `struct` definitions @@ -168,9 +184,9 @@ should not implement recursive, compiler-compatible macro expansion internally; it should consume compiler-preprocessed output with preserved origin metadata. -## Planned Public API +## Public API -Target module-level entrypoints: +Implemented module-level entrypoints: ```python from c_parser import parse_c_file, parse_c_project @@ -199,10 +215,55 @@ parse_c_project( ``` These return typed parser models analogous to the Fortran parser API. The -current partial phase can populate `functions`, `typedefs`, `globals`, -`includes`, `macros`, and metadata `diagnostics`. Composite-type lists such as -`structs`, `unions`, and `enums` remain empty until their dedicated parser -phase lands. Functions include `prototype_style`, currently `"prototype"` for +current partial phase can populate `functions`, `structs`, `unions`, `enums`, +`typedefs`, `variables`, `includes`, `macros`, and metadata `diagnostics`. +Incomplete `struct name;` and `union name;` declarations are concrete +`CStruct`/`CUnion` types with `is_incomplete=True` and source locations. The +parser returns concrete objects instead of a declaration-kind tag: +`CFunction`, `CVariable`, `CTypedef`, `CStruct`, `CUnion`, and `CEnum`. +A declaration such as +`typedef struct node { int value; } node_t;` produces a `CStruct` plus a +`CTypedef`, while `struct point { int x; } origin;` produces a `CStruct` plus +a `CVariable`. + +All types inherit from `CType`. Implemented primitive type classes are +`CVoid`, `CBool`, `CChar`, `CSignedChar`, `CUnsignedChar`, `CShort`, +`CUnsignedShort`, `CInt`, `CUnsignedInt`, `CLong`, `CUnsignedLong`, +`CLongLong`, `CUnsignedLongLong`, `CFloat`, `CDouble`, `CLongDouble`, +`CFloatComplex`, `CDoubleComplex`, and `CLongDoubleComplex`. Qualifiers are +`CConst`, `CVolatile`, `CRestrict`, and `CAtomic`, attached to the precise +type component they qualify. `_Atomic int value;` is stored with a `CAtomic` +qualifier; the distinct `_Atomic(int) value;` type-specifier form remains +diagnosed as unsupported. + +Nested declarators are `CComposedType` objects whose `components` are read +from the declared name outward: + +```python +int *values[4]; # CComposedType([CArray(bound="4"), CPointer(), CInt()]) +int (*matrix)[4]; # CComposedType([CPointer(), CArray(bound="4"), CInt()]) +int *(*table)[4]; # CComposedType([CPointer(), CArray(bound="4"), CPointer(), CInt()]) +``` + +`CFunction` has `result_type` and named `CParameter` objects. Its `.type` +property provides the corresponding nameless `CFunctionType`, which is also +used inside pointer typedefs and variables: + +```python +int add(int a, int b); # CFunction(name="add", result_type=CInt(), parameters=[...]) +int (*compare)(int, int); # CVariable(type=CComposedType([CPointer(), CFunctionType(...)])) +``` + +Callback-bearing parameters are marked as parser-side callback candidates, +without claiming semantic wrappability. Struct and union `members` are +`CVariable` objects; optional `bit_width` and `initializer` fields preserve +source facts without inventing separate field or valued-variable classes. +Selected unsupported forms, such as static assertions, +attributes, alignment specifiers, `_Atomic(type)`, and nested aggregate member +definitions, are reported in `diagnostics` with explicit `unit_kind` values. +Unconsumed declarator suffixes are also diagnosed instead of producing partial +objects. Functions +include `prototype_style`, currently `"prototype"` for typed or explicit `void` parameter lists and `"unspecified"` for empty parameter lists such as `int f()`. Function definitions do not store executable body text; they include direct `start` and `end` locations. @@ -217,7 +278,7 @@ wrappability assessment, that should live in the semantic layer after C parser models are converted to semantic IR or edited `.pyi` policy is loaded, matching the current Fortran and `.pyi` readiness boundary. -## Planned CLI Usage +## CLI Usage Initial explicit mode: @@ -235,7 +296,7 @@ x2py path/to/api.h --parse-c Auto-detection should come later, after the frontend is stable. -## Planned JSON Output +## Current JSON Output Per-file shape: @@ -249,11 +310,11 @@ Per-file shape: "functions": [ { "name": "run", - "return_type": {"base": "int", "...": "..."}, + "result_type": {"model": "CInt", "qualifiers": [], "source_text": "int"}, "parameters": [], "storage": [], "specifiers": [], - "variadic": false, + "is_variadic": false, "is_definition": false, "prototype_style": "prototype", "source_location": {"filename": "", "line": 1, "...": "..."}, @@ -265,7 +326,7 @@ Per-file shape: "unions": [], "enums": [], "typedefs": [], - "globals": [], + "variables": [], "macros": [], "includes": [], "diagnostics": [] @@ -276,7 +337,13 @@ Per-file shape: JSON compatibility rules: - prefer additive schema changes -- include source locations for populated include, macro, and diagnostic models +- serialize concrete `CType` identity using `"model"`; reserve `"type"` for + actual type relationships such as `CTypedef.type` +- serialize qualifier objects as canonical spellings such as `"const"` +- include `source_location` for declaration/directive records and `location` + for diagnostics +- emit references for reused aggregate or typedef objects rather than + recursive JSON cycles - preserve unknown or unresolved information rather than dropping it silently - keep model fields stable enough for golden fixture testing - document every intentional schema break @@ -325,9 +392,9 @@ The parser has the error type and formatter. Raw directive collection can emit non-fatal metadata diagnostics, such as unresolved local includes or function-like macros that were recorded but not expanded. K&R-style function definitions now raise `CParseError` because the current function parser only -models prototype-style declarations and definitions. The current grammar subset -is otherwise intentionally tolerant for unsupported declaration forms; more hard -syntax errors should be added only with focused tests. +models prototype-style declarations and definitions. Known unsupported +declaration extensions are diagnosed rather than partially modeled; additional +syntax diagnostics should be added only with focused tests. ## Planned Testing Workflow @@ -356,12 +423,62 @@ public entrypoints, empty model serialization, CLI discovery, JSON/output-file behavior, unsupported C stages, comment stripping, line-continuation folding, top-level splitting, include collection, simple macro collection, macro-shaped declaration deferral, raw conditional branch non-selection, simple declarations, -globals, typedefs, and simple function prototypes/definitions, including +variables, typedefs, recursive declarator composition, aggregate definitions, +members, enums, and simple function prototypes/definitions, including function-definition start/end locations. The broader roadmap tests remain skipped until their matching implementation branches land. Future implementation branches should unskip only the tests for the capability they implement, then merge those branches back into `c-parser/main`. +### Declaration Coverage Boundary + +Active declaration tests currently cover: + +- every implemented primitive spelling mapped to its concrete `CType` +- all qualifier objects, storage metadata, simple expression initializers, and + multiple declarators +- pointer/array precedence, multidimensional arrays, parameter VLA/static + metadata, function pointers, callback arrays, and functions returning + function pointers +- functions, variables, typedefs, struct/union members, enums, incomplete + tags, inline aggregate aliases, anonymous aggregate typedefs, and recursive + struct pointers +- concrete-type JSON serialization, source locations, and cycle-safe aggregate + references +- diagnostics for selected unsupported attributes, alignment, `_Atomic(type)`, + nested aggregate definitions, K&R definitions, and trailing declarator + extensions + +This is enough coverage for the currently implemented subset, not for all C +declarations. + +### Missing Implementation With Examples + +| Capability | C example | Current parser boundary | Needed behavior | +| --- | --- | --- | --- | +| Parameter adjustment | `void process(int values[4], int callback(int));` | Preserves the declared array and function parameter types; it does not expose C's adjusted pointer parameter type. | Keep `declared_type`, and expose the adjusted effective type (`int *` and pointer-to-function). | +| Flexible array members | `struct packet { unsigned size; unsigned char data[]; };` | Represents `data` as `CArray(bound=None)` but does not set `is_flexible` or validate that it is a legal final struct member. | Mark the flexible member and diagnose illegal placement or union usage. | +| Braced/designated initializers | `int values[3] = {1, 2, 3};` and `struct point origin = {.x = 1, .y = 2};` | Simple initializer text such as `int answer = 42;` is preserved; braced forms are not reliably emitted as `CVariable` initializer facts. | Parse or preserve balanced initializer source without treating its braces as an aggregate declaration. | +| Nested aggregate members | `struct outer { struct { int x; } inner; };` | Produces an unsupported-member diagnostic and does not model `inner`. | Build an anonymous `CStruct`/`CUnion` type used by the member variable. | +| Typedef/tag resolution | `typedef unsigned long size_t; size_t count(void);` and `struct state { int id; }; void step(struct state *s);` | Preserves uses as unresolved `CTypedef` or incomplete tag-type objects unless attached inline. | Link uses to declarations across a file/project and diagnose conflicts. | +| Preprocessed declarations | `#define API(ret) ret` followed by `API(int) run(void);` | Raw mode records macro metadata and does not claim the expanded declaration; preprocessed input with line mapping is not implemented. | Accept compiler-expanded input and map each declaration back through `#line` markers. | +| Additional extension families | `int run(void) __attribute__((visibility("default")));` | Known attribute/alignment/`_Atomic(type)` forms are diagnosed; broader compiler extensions are not modeled. | Add fixture-driven support or a focused diagnostic for each required extension family. | + +### Represented But Requiring Stronger Tests + +These forms are not absent from the model, but need explicit active regression +tests before they can be treated as stable: + +```c +const int * const * volatile chain; +struct flags { unsigned : 0; unsigned mode : 3; }; +``` + +The current parser creates distinct qualified `CPointer` components for +`chain`, and creates a `CVariable(name=None, bit_width="0")` for the unnamed +zero-width bit-field. Tests should lock down those shapes and any later +semantic validation rules. + Fixture layout should be separate from Fortran: ```text @@ -381,7 +498,7 @@ The first real-world corpus target should be cJSON, pinned to an exact release or commit with license and source provenance. cJSON is small enough for early stabilization while still covering typedef structs, recursive pointers, public macro declaration wrappers, constants, `const char *` APIs, `size_t`, and -callback hook fields. +callback hook members. ## Planned Documentation Set diff --git a/tests/parser/c/test_c_cli_skeleton.py b/tests/parser/c/test_c_cli_skeleton.py index 0637685ce..ed0349512 100644 --- a/tests/parser/c/test_c_cli_skeleton.py +++ b/tests/parser/c/test_c_cli_skeleton.py @@ -46,7 +46,7 @@ def test_cli_c_parse_json_stdout_for_header(tmp_path: Path): assert file_payload["unions"] == [] assert file_payload["enums"] == [] assert file_payload["typedefs"] == [] - assert file_payload["globals"] == [] + assert file_payload["variables"] == [] assert file_payload["macros"] == [] assert file_payload["includes"] == [] assert file_payload["diagnostics"] == [] diff --git a/tests/parser/c/test_c_corpus.py b/tests/parser/c/test_c_corpus.py index b5cf78fa3..dc8205d0e 100644 --- a/tests/parser/c/test_c_corpus.py +++ b/tests/parser/c/test_c_corpus.py @@ -61,9 +61,9 @@ def test_cjson_callback_hook_fields_are_modeled_with_policy_placeholders(): assert "cJSON_Hooks" in {struct.name for struct in parsed.structs} hooks = next(struct for struct in parsed.structs if struct.name == "cJSON_Hooks") - callback_fields = [field for field in hooks.fields if field.type.kind == "function_pointer"] - assert callback_fields - assert all(field.callback_policy is None for field in callback_fields) + callback_members = [member for member in hooks.members if member.callback_candidate] + assert callback_members + assert all(member.callback_policy is None for member in callback_members) def test_cjson_source_file_parse_skips_function_bodies_safely(): diff --git a/tests/parser/c/test_c_declarations_and_declarators.py b/tests/parser/c/test_c_declarations_and_declarators.py index 0467c57be..239549059 100644 --- a/tests/parser/c/test_c_declarations_and_declarators.py +++ b/tests/parser/c/test_c_declarations_and_declarators.py @@ -1,11 +1,11 @@ # -*- coding: utf-8 -*- -"""C declaration-specifier and declarator parser tests.""" +"""C declaration-specifier and composed-type parser tests.""" import pytest -def test_declaration_specifiers_parse_primitive_signedness_and_widths(): - from c_parser import parse_c_file +def test_primitive_specifiers_create_concrete_primitive_types(): + from c_parser import CBool, CShort, CUnsignedLongLong, parse_c_file parsed = parse_c_file( """ @@ -16,71 +16,125 @@ def test_declaration_specifiers_parse_primitive_signedness_and_widths(): filename="primitives.h", ) - functions = {fn.name: fn for fn in parsed.functions} - assert functions["next_id"].return_type.base == "unsigned long long" - assert functions["clamp_short"].parameters[0].type.base == "signed short" - assert functions["enabled"].return_type.base == "_Bool" - - -def test_pointer_qualifiers_are_preserved_in_order(): - from c_parser import parse_c_file + functions = {function.name: function for function in parsed.functions} + assert isinstance(functions["next_id"].result_type, CUnsignedLongLong) + assert isinstance(functions["clamp_short"].parameters[0].type, CShort) + assert isinstance(functions["enabled"].result_type, CBool) + + +@pytest.mark.parametrize( + ("spelling", "expected_name"), + [ + ("void", "CVoid"), + ("_Bool", "CBool"), + ("char", "CChar"), + ("signed char", "CSignedChar"), + ("unsigned char", "CUnsignedChar"), + ("short", "CShort"), + ("short int", "CShort"), + ("signed short", "CShort"), + ("signed short int", "CShort"), + ("unsigned short", "CUnsignedShort"), + ("unsigned short int", "CUnsignedShort"), + ("int", "CInt"), + ("signed", "CInt"), + ("signed int", "CInt"), + ("unsigned", "CUnsignedInt"), + ("unsigned int", "CUnsignedInt"), + ("long", "CLong"), + ("long int", "CLong"), + ("signed long", "CLong"), + ("signed long int", "CLong"), + ("unsigned long", "CUnsignedLong"), + ("unsigned long int", "CUnsignedLong"), + ("long long", "CLongLong"), + ("long long int", "CLongLong"), + ("signed long long", "CLongLong"), + ("signed long long int", "CLongLong"), + ("unsigned long long", "CUnsignedLongLong"), + ("unsigned long long int", "CUnsignedLongLong"), + ("float", "CFloat"), + ("double", "CDouble"), + ("long double", "CLongDouble"), + ("float _Complex", "CFloatComplex"), + ("_Complex", "CDoubleComplex"), + ("double _Complex", "CDoubleComplex"), + ("long double _Complex", "CLongDoubleComplex"), + ], +) +def test_every_supported_primitive_spelling_creates_a_concrete_ctype(spelling, expected_name): + import c_parser + from c_parser import CType, parse_c_file + + function = parse_c_file(f"{spelling} primitive(void);\n", filename="primitive_table.h").functions[0] + expected = getattr(c_parser, expected_name) + + assert isinstance(function.result_type, expected) + assert isinstance(function.result_type, CType) + + +def test_pointer_qualifiers_belong_to_the_component_they_qualify(): + from c_parser import CComposedType, CConst, CDouble, CPointer, CRestrict, parse_c_file parsed = parse_c_file( "void copy(const double * restrict src, double * restrict dst);\n", filename="qualifiers.h", ) - params = {param.name: param for param in parsed.functions[0].parameters} - assert params["src"].type.qualifiers == ["const"] - assert params["src"].type.pointers[0].qualifiers == ["restrict"] - assert params["dst"].type.pointers[0].qualifiers == ["restrict"] + params = {parameter.name: parameter for parameter in parsed.functions[0].parameters} + src = params["src"].type + dst = params["dst"].type + assert isinstance(src, CComposedType) + assert isinstance(src.components[0], CPointer) + assert src.components[0].qualifiers == [CRestrict()] + assert isinstance(src.components[1], CDouble) + assert src.components[1].qualifiers == [CConst()] + assert dst.components[0].qualifiers == [CRestrict()] -def test_array_declarators_preserve_dimensions_and_static_bounds(): - from c_parser import parse_c_file +def test_array_components_preserve_bounds_static_minimum_and_qualifiers(): + from c_parser import CArray, CComposedType, CConst, parse_c_file parsed = parse_c_file( - "void solve(size_t n, double a[static 4], const int shape[2]);\n", + "void solve(size_t n, double a[static 4], const int shape[2], int work[const *]);\n", filename="arrays.h", ) - params = {param.name: param for param in parsed.functions[0].parameters} - assert params["a"].type.arrays[0].size == "4" - assert params["a"].type.arrays[0].static is True - assert params["shape"].type.arrays[0].size == "2" - - -def test_multiple_declarators_share_specifiers_but_keep_distinct_types(): - from c_parser import parse_c_file - - parsed = parse_c_file( - "extern const int *left, right[4];\n", - filename="globals.h", - ) - - globals_by_name = {glob.name: glob for glob in parsed.globals} - assert globals_by_name["left"].type.pointers - assert globals_by_name["right"].type.arrays[0].size == "4" - assert globals_by_name["right"].type.qualifiers == ["const"] + params = {parameter.name: parameter for parameter in parsed.functions[0].parameters} + a = params["a"].type + shape = params["shape"].type + work = params["work"].type + assert isinstance(a, CComposedType) + assert isinstance(a.components[0], CArray) + assert a.components[0].bound == "4" + assert a.components[0].is_static_minimum is True + assert shape.components[0].bound == "2" + assert shape.components[-1].qualifiers == [CConst()] + assert work.components[0].qualifiers == [CConst()] + assert work.components[0].is_variable_length is True -def test_typedef_declaration_preserves_alias_and_underlying_type_text(): - from c_parser import parse_c_file +def test_multiple_declarators_share_specifiers_but_have_distinct_compositions(): + from c_parser import CArray, CComposedType, CConst, CInt, CPointer, parse_c_file - parsed = parse_c_file("typedef unsigned long api_size;\n", filename="typedefs.h") + parsed = parse_c_file("extern const int *left, right[4];\n", filename="variables.h") - typedef = parsed.typedefs[0] - assert typedef.name == "api_size" - assert typedef.type.base == "unsigned long" - assert typedef.type.storage_class == ["typedef"] + variables = {variable.name: variable for variable in parsed.variables} + assert variables["left"].storage == ["extern"] + assert isinstance(variables["left"].type, CComposedType) + assert [type(component) for component in variables["left"].type.components] == [CPointer, CInt] + assert [type(component) for component in variables["right"].type.components] == [CArray, CInt] + assert variables["right"].type.components[0].bound == "4" + assert variables["right"].type.components[-1].qualifiers == [CConst()] -def test_pointer_array_typedefs_and_typedef_name_references_are_preserved(): - from c_parser import parse_c_file +def test_typedefs_and_typedef_references_are_concrete_types(): + from c_parser import CArray, CComposedType, CDouble, CPointer, CStruct, CTypedef, CUnsignedLong, parse_c_file parsed = parse_c_file( """ struct state; +typedef unsigned long api_size; typedef const struct state *state_ref; typedef double vector3[3]; typedef vector3 basis3[3]; @@ -91,17 +145,20 @@ def test_pointer_array_typedefs_and_typedef_name_references_are_preserved(): ) typedefs = {typedef.name: typedef for typedef in parsed.typedefs} - assert typedefs["state_ref"].type.tag_kind == "struct" - assert typedefs["state_ref"].type.tag_name == "state" - assert typedefs["state_ref"].type.pointers - assert typedefs["vector3"].type.arrays[0].size == "3" - assert typedefs["basis3"].type.typedef_name == "vector3" - assert typedefs["basis3"].type.arrays[0].size == "3" - assert parsed.functions[1].parameters[0].type.typedef_name == "basis3" + assert isinstance(typedefs["api_size"].type, CUnsignedLong) + state_ref = typedefs["state_ref"].type + assert isinstance(state_ref, CComposedType) + assert [type(component) for component in state_ref.components] == [CPointer, CStruct] + assert state_ref.components[-1].name == "state" + assert isinstance(typedefs["vector3"].type.components[0], CArray) + assert isinstance(typedefs["vector3"].type.components[-1], CDouble) + assert isinstance(typedefs["basis3"].type.components[-1], CTypedef) + assert typedefs["basis3"].type.components[-1].name == "vector3" + assert isinstance(parsed.functions[1].parameters[0].type, CTypedef) -def test_globals_with_initializers_multidimensional_arrays_and_tag_refs_parse(): - from c_parser import parse_c_file +def test_variables_preserve_initializer_text_arrays_and_concrete_tag_types(): + from c_parser import CArray, CComposedType, CEnum, CInt, CStruct, CUnion, parse_c_file parsed = parse_c_file( """ @@ -111,38 +168,65 @@ def test_globals_with_initializers_multidimensional_arrays_and_tag_refs_parse(): double matrix[3][4]; int answer = 42; """, - filename="globals_richer.h", + filename="variables_richer.h", ) - globals_by_name = {glob.name: glob for glob in parsed.globals} - assert globals_by_name["global_state"].type.tag_kind == "struct" - assert globals_by_name["global_state"].type.tag_name == "state" - assert globals_by_name["global_state"].type.pointers - assert globals_by_name["global_scalar"].type.tag_kind == "union" - assert globals_by_name["last_status"].type.tag_kind == "enum" - assert [array.size for array in globals_by_name["matrix"].type.arrays] == ["3", "4"] - assert globals_by_name["answer"].type.base == "int" + variables = {variable.name: variable for variable in parsed.variables} + assert isinstance(variables["global_state"].type.components[-1], CStruct) + assert variables["global_state"].type.components[-1].name == "state" + assert variables["global_state"].initializer.source_text == "0" + assert isinstance(variables["global_scalar"].type.components[-1], CUnion) + assert isinstance(variables["last_status"].type, CEnum) + assert variables["last_status"].initializer.source_text == "STATUS_OK" + assert [component.bound for component in variables["matrix"].type.components[:2]] == ["3", "4"] + assert all(isinstance(component, CArray) for component in variables["matrix"].type.components[:2]) + assert isinstance(variables["answer"].type, CInt) + assert variables["answer"].initializer.source_text == "42" -def test_parameters_preserve_struct_union_and_enum_references(): - from c_parser import parse_c_file +def test_parameters_preserve_concrete_struct_union_and_enum_uses(): + from c_parser import CEnum, CStruct, CUnion, parse_c_file parsed = parse_c_file( "void consume(const struct state *s, union scalar *u, enum status status);\n", filename="tag_params.h", ) - params = {param.name: param for param in parsed.functions[0].parameters} - assert params["s"].type.tag_kind == "struct" - assert params["s"].type.tag_name == "state" - assert params["u"].type.tag_kind == "union" - assert params["u"].type.tag_name == "scalar" - assert params["status"].type.tag_kind == "enum" - assert params["status"].type.tag_name == "status" + params = {parameter.name: parameter for parameter in parsed.functions[0].parameters} + assert isinstance(params["s"].type.components[-1], CStruct) + assert params["s"].type.components[-1].name == "state" + assert isinstance(params["u"].type.components[-1], CUnion) + assert isinstance(params["status"].type, CEnum) -def test_storage_classes_and_qualifiers_are_recorded_for_globals(): - from c_parser import parse_c_file +def test_incomplete_structs_and_pointer_uses_are_concrete_objects(): + from c_parser import CComposedType, CPointer, CStruct, parse_c_file + + parsed = parse_c_file( + """ +struct handle; +struct handle *open_handle(void); +void close_handle(struct handle *handle); +""", + filename="opaque.h", + ) + + handle = parsed.structs[0] + assert isinstance(handle, CStruct) + assert handle.name == "handle" + assert handle.is_incomplete is True + assert handle.members == [] + + functions = {function.name: function for function in parsed.functions} + result = functions["open_handle"].result_type + assert isinstance(result, CComposedType) + assert isinstance(result.components[0], CPointer) + assert isinstance(result.components[1], CStruct) + assert result.components[1].name == "handle" + + +def test_storage_is_declaration_metadata_and_qualifiers_are_type_metadata(): + from c_parser import CAtomic, CConst, CUnsignedLong, CVolatile, parse_c_file parsed = parse_c_file( """ @@ -150,83 +234,189 @@ def test_storage_classes_and_qualifiers_are_recorded_for_globals(): static const double scale_factor = 1.0; _Thread_local unsigned long tls_counter; register volatile int scratch; +_Atomic int atomic_counter; """, - filename="storage_globals.h", + filename="storage_variables.h", ) - globals_by_name = {glob.name: glob for glob in parsed.globals} - assert globals_by_name["api_errno"].type.storage_class == ["extern"] - assert globals_by_name["scale_factor"].type.storage_class == ["static"] - assert globals_by_name["scale_factor"].type.qualifiers == ["const"] - assert globals_by_name["tls_counter"].type.storage_class == ["_Thread_local"] - assert globals_by_name["scratch"].type.storage_class == ["register"] - assert globals_by_name["scratch"].type.qualifiers == ["volatile"] + variables = {variable.name: variable for variable in parsed.variables} + assert variables["api_errno"].storage == ["extern"] + assert variables["scale_factor"].storage == ["static"] + assert variables["scale_factor"].type.qualifiers == [CConst()] + assert variables["tls_counter"].storage == ["_Thread_local"] + assert isinstance(variables["tls_counter"].type, CUnsignedLong) + assert variables["scratch"].storage == ["register"] + assert variables["scratch"].type.qualifiers == [CVolatile()] + assert variables["atomic_counter"].type.qualifiers == [CAtomic()] -def test_function_bodies_do_not_contribute_local_declarations_to_globals(): +def test_function_bodies_do_not_contribute_local_variables(): from c_parser import parse_c_file parsed = parse_c_file( """ -int compute(int x) -{ - int local_value = x + 1; - return local_value; -} +int compute(int x) { int local_value = x + 1; return local_value; } extern int exported_value; """, filename="locals.c", ) - assert [global_.name for global_ in parsed.globals] == ["exported_value"] + assert [variable.name for variable in parsed.variables] == ["exported_value"] -@pytest.mark.skip(reason="recursive pointer/array type layers are not implemented yet.") -def test_parenthesized_declarators_distinguish_pointer_arrays_from_array_pointers(): - from c_parser import parse_c_file +def test_declarations_return_concrete_objects_instead_of_kind_fields(): + from c_parser import CArray, CFunction, CFunctionType, CInt, CPointer, CStruct, CTypedef, CVariable, parse_c_file parsed = parse_c_file( """ +struct handle; +typedef int (*compare_fn)(const void *, const void *); extern int *values[4]; extern int (*matrix)[4]; +int add(int a, int b); +void sort_items(int (*fallback)(const void *, const void *)); """, - filename="paren_decl.h", + filename="declaration_matrix.h", ) - values = {glob.name: glob for glob in parsed.globals} - assert values["values"].type.arrays[0].size == "4" - assert values["values"].type.element_type.pointers - assert values["matrix"].type.pointers - assert values["matrix"].type.pointee.arrays[0].size == "4" + assert isinstance(parsed.structs[0], CStruct) + assert all(isinstance(typedef, CTypedef) for typedef in parsed.typedefs) + assert all(isinstance(variable, CVariable) for variable in parsed.variables) + assert all(isinstance(function, CFunction) for function in parsed.functions) + compare = parsed.typedefs[0].type + assert [type(component) for component in compare.components] == [CPointer, CFunctionType] + values, matrix = parsed.variables + assert [type(component) for component in values.type.components] == [CArray, CPointer, CInt] + assert [type(component) for component in matrix.type.components] == [CPointer, CArray, CInt] + assert parsed.functions[1].parameters[0].callback_candidate is True -@pytest.mark.skip(reason="function pointer declarators are not implemented yet.") -def test_function_pointer_declarator_is_modeled_not_flattened(): - from c_parser import parse_c_file + +def test_composite_definitions_are_concrete_objects_and_static_assert_is_diagnostic(): + from c_parser import CEnum, CStruct, CUnion, CVariable, parse_c_file parsed = parse_c_file( - "typedef int (*compare_fn)(const void *a, const void *b);\n", + """ +struct point { double x; double y; }; +union value { int i; double d; }; +enum status { STATUS_OK = 0 }; +_Static_assert(sizeof(int) == 4, "expected int width"); +""", + filename="composites.h", + ) + + assert isinstance(parsed.structs[0], CStruct) + assert all(isinstance(member, CVariable) for member in parsed.structs[0].members) + assert [member.name for member in parsed.structs[0].members] == ["x", "y"] + assert isinstance(parsed.unions[0], CUnion) + assert isinstance(parsed.enums[0], CEnum) + assert [diagnostic.unit_kind for diagnostic in parsed.diagnostics] == ["static_assert"] + + +def test_parenthesized_declarators_preserve_pointer_array_order(): + from c_parser import CArray, CInt, CPointer, parse_c_file + + parsed = parse_c_file("extern int *values[4];\nextern int (*matrix)[4];\n", filename="paren_decl.h") + variables = {variable.name: variable for variable in parsed.variables} + + assert [type(component) for component in variables["values"].type.components] == [CArray, CPointer, CInt] + assert [type(component) for component in variables["matrix"].type.components] == [CPointer, CArray, CInt] + + +def test_function_type_discards_placeholder_parameter_names(): + from c_parser import CFunctionType, CPointer, parse_c_file + + parsed = parse_c_file( + "typedef int (*compare_fn)(const void *left, const void *right);\n", filename="callback_typedef.h", ) - typedef = parsed.typedefs[0] - assert typedef.name == "compare_fn" - assert typedef.type.kind == "function_pointer" - assert [param.name for param in typedef.type.parameters] == ["a", "b"] + type_ = parsed.typedefs[0].type + assert isinstance(type_.components[0], CPointer) + signature = type_.components[1] + assert isinstance(signature, CFunctionType) + assert len(signature.parameter_types) == 2 + assert not hasattr(signature, "parameters") + + +def test_recursive_compositions_cover_tables_callback_arrays_and_function_results(): + from c_parser import CArray, CFunctionType, CInt, CPointer, parse_c_file + parsed = parse_c_file( + """ +extern int *(*table)[4]; +typedef int (*callback_table[8])(int); +int (*factory(void))(int); +int direct(void), *value; +""", + filename="recursive_declarators.h", + ) -def test_storage_class_and_inline_attributes_are_recorded(): + variables = {variable.name: variable for variable in parsed.variables} + assert [type(component) for component in variables["table"].type.components] == [ + CPointer, + CArray, + CPointer, + CInt, + ] + assert [type(component) for component in variables["value"].type.components] == [CPointer, CInt] + callbacks = parsed.typedefs[0].type + assert [type(component) for component in callbacks.components] == [CArray, CPointer, CFunctionType] + functions = {function.name: function for function in parsed.functions} + assert set(functions) == {"factory", "direct"} + assert [type(component) for component in functions["factory"].result_type.components] == [ + CPointer, + CFunctionType, + ] + + +def test_unimplemented_declaration_extensions_are_diagnosed_not_partially_modeled(): from c_parser import parse_c_file parsed = parse_c_file( """ -static inline int local_add(int a, int b) { return a + b; } -extern int exported_add(int a, int b); +int visible __attribute__((visibility("default"))); +int outdated [[deprecated]]; +_Alignas(16) int aligned_value; +_Atomic(int) atomic_value; """, + filename="extensions.h", + ) + + assert parsed.variables == [] + assert [diagnostic.unit_kind for diagnostic in parsed.diagnostics] == [ + "attribute_declaration", + "attribute_declaration", + "alignment_declaration", + "atomic_type_declaration", + ] + + +def test_unconsumed_declarator_suffixes_are_diagnosed_not_silently_discarded(): + from c_parser import parse_c_file + + parsed = parse_c_file( + 'extern int retained, pinned asm("r0");\nint run(int value asm("r0"));\n', + filename="declarator_extensions.h", + ) + + assert [variable.name for variable in parsed.variables] == ["retained"] + assert parsed.functions == [] + assert [diagnostic.code for diagnostic in parsed.diagnostics] == [ + "C_UNSUPPORTED_DECLARATOR", + "C_UNSUPPORTED_DECLARATOR", + ] + + +def test_storage_class_and_inline_specifiers_are_recorded_on_functions(): + from c_parser import parse_c_file + + parsed = parse_c_file( + "static inline int local_add(int a, int b) { return a + b; }\nextern int exported_add(int a, int b);\n", filename="storage.c", ) - functions = {fn.name: fn for fn in parsed.functions} + functions = {function.name: function for function in parsed.functions} assert functions["local_add"].storage == ["static"] assert "inline" in functions["local_add"].specifiers assert functions["exported_add"].storage == ["extern"] diff --git a/tests/parser/c/test_c_functions.py b/tests/parser/c/test_c_functions.py index ff38b11b9..4a7c43979 100644 --- a/tests/parser/c/test_c_functions.py +++ b/tests/parser/c/test_c_functions.py @@ -4,18 +4,21 @@ import pytest -def test_function_prototypes_preserve_return_type_parameter_order_and_names(): - from c_parser import parse_c_file +def test_named_function_exposes_result_type_named_parameters_and_derived_type(): + from c_parser import CDouble, CFunctionType, CTypedef, parse_c_file parsed = parse_c_file( "double dot(size_t n, const double *x, const double *y);\n", filename="functions.h", ) - fn = parsed.functions[0] - assert fn.name == "dot" - assert fn.return_type.base == "double" - assert [param.name for param in fn.parameters] == ["n", "x", "y"] + function = parsed.functions[0] + assert function.name == "dot" + assert isinstance(function.result_type, CDouble) + assert [parameter.name for parameter in function.parameters] == ["n", "x", "y"] + assert isinstance(function.parameters[0].type, CTypedef) + assert isinstance(function.type, CFunctionType) + assert function.type.parameter_types == [parameter.type for parameter in function.parameters] def test_function_definitions_skip_bodies_but_preserve_start_and_end_locations(): @@ -31,26 +34,20 @@ def test_function_definitions_skip_bodies_but_preserve_start_and_end_locations() filename="definitions.c", ) - fn = parsed.functions[0] - assert fn.name == "add" - assert fn.start is not None - assert fn.end is not None - assert fn.start.line == 2 - assert fn.end.line == 5 + function = parsed.functions[0] + assert function.name == "add" + assert function.start is not None + assert function.end is not None + assert function.start.line == 2 + assert function.end.line == 5 def test_void_parameter_list_and_empty_parameter_list_are_distinguished(): from c_parser import parse_c_file - parsed = parse_c_file( - """ -int explicit_void(void); -int unspecified(); -""", - filename="void_params.h", - ) + parsed = parse_c_file("int explicit_void(void);\nint unspecified();\n", filename="void_params.h") - functions = {fn.name: fn for fn in parsed.functions} + functions = {function.name: function for function in parsed.functions} assert functions["explicit_void"].parameters == [] assert functions["explicit_void"].prototype_style == "prototype" assert functions["unspecified"].prototype_style == "unspecified" @@ -61,10 +58,11 @@ def test_variadic_functions_are_parsed_as_source_facts(): parsed = parse_c_file("int log_msg(const char *fmt, ...);\n", filename="variadic.h") - assert parsed.functions[0].variadic is True + assert parsed.functions[0].is_variadic is True + assert parsed.functions[0].type.is_variadic is True -def test_old_style_knr_function_definition_raises_or_records_unsupported_diagnostic(): +def test_old_style_knr_function_definition_raises_unsupported_diagnostic(): from c_parser import CParseError, parse_c_file source = """ @@ -80,9 +78,8 @@ def test_old_style_knr_function_definition_raises_or_records_unsupported_diagnos parse_c_file(source, filename="knr.c") -@pytest.mark.skip(reason="function pointer parameters are not implemented yet.") -def test_function_pointer_parameter_is_modeled_as_callback_candidate(): - from c_parser import parse_c_file +def test_function_pointer_parameter_is_a_callback_candidate_with_nameless_signature(): + from c_parser import CFunctionType, CInt, CPointer, parse_c_file parsed = parse_c_file( "void sort_items(void *items, int (*compare)(const void *, const void *));\n", @@ -90,14 +87,17 @@ def test_function_pointer_parameter_is_modeled_as_callback_candidate(): ) compare = parsed.functions[0].parameters[1] - assert compare.type.kind == "function_pointer" assert compare.callback_candidate is True assert compare.callback_policy is None + assert [type(component) for component in compare.type.components] == [CPointer, CFunctionType] + signature = compare.type.components[1] + assert isinstance(signature.result_type, CInt) + assert len(signature.parameter_types) == 2 @pytest.mark.skip(reason="function pointer typedef resolution is not implemented yet.") def test_callback_typedef_parameter_links_to_typedef_signature(): - from c_parser import parse_c_file + from c_parser import CFunctionType, CTypedef, parse_c_file parsed = parse_c_file( """ @@ -107,23 +107,21 @@ def test_callback_typedef_parameter_links_to_typedef_signature(): filename="callback_typedef.h", ) - fn = parsed.functions[0] - assert fn.parameters[1].type.typedef_name == "compare_fn" - assert fn.parameters[1].type.resolved.kind == "function_pointer" + referenced = parsed.functions[0].parameters[1].type + assert isinstance(referenced, CTypedef) + assert isinstance(referenced.type.components[1], CFunctionType) def test_function_returning_pointer_to_const_struct_is_preserved(): - from c_parser import parse_c_file + from c_parser import CComposedType, CConst, CPointer, CStruct, parse_c_file parsed = parse_c_file( - """ -struct state; -const struct state *current_state(void); -""", + "struct state;\nconst struct state *current_state(void);\n", filename="return_pointer.h", ) - fn = parsed.functions[0] - assert fn.return_type.qualifiers == ["const"] - assert fn.return_type.tag_name == "state" - assert fn.return_type.pointers + result = parsed.functions[0].result_type + assert isinstance(result, CComposedType) + assert isinstance(result.components[0], CPointer) + assert isinstance(result.components[1], CStruct) + assert result.components[1].qualifiers == [CConst()] diff --git a/tests/parser/c/test_c_json_sanity.py b/tests/parser/c/test_c_json_sanity.py index 30057202b..2561316f8 100644 --- a/tests/parser/c/test_c_json_sanity.py +++ b/tests/parser/c/test_c_json_sanity.py @@ -35,7 +35,7 @@ def test_c_json_fixtures_have_stable_top_level_shape(): "unions", "enums", "typedefs", - "globals", + "variables", "macros", "includes", "diagnostics", @@ -50,7 +50,7 @@ def test_c_json_functions_have_names_types_and_source_locations(): for path, payload in _iter_json_payloads(): for fn in payload.get("functions", []): assert fn["name"], f"function without name in {path}" - assert fn["return_type"], f"function without return type in {path}" + assert fn["result_type"], f"function without result type in {path}" assert isinstance(fn["parameters"], list) assert fn["source_location"]["line"] >= 1 assert fn["source_location"]["column"] >= 1 @@ -73,4 +73,3 @@ def test_c_json_diagnostics_have_codes_locations_and_severities(): assert diagnostic.get("message") if diagnostic.get("source_location"): assert diagnostic["source_location"]["line"] >= 1 - diff --git a/tests/parser/c/test_c_project_includes.py b/tests/parser/c/test_c_project_includes.py index 614bbced0..c592b3554 100644 --- a/tests/parser/c/test_c_project_includes.py +++ b/tests/parser/c/test_c_project_includes.py @@ -42,7 +42,7 @@ def test_project_resolves_typedefs_across_headers_and_sources(tmp_path: Path): project = parse_c_project(tmp_path) - assert project.functions["count"].return_type.resolved.base == "unsigned long" + assert project.functions["count"].result_type.type is project.typedefs["api_size"].type def test_project_resolves_struct_tags_across_includes(tmp_path: Path): @@ -54,8 +54,8 @@ def test_project_resolves_struct_tags_across_includes(tmp_path: Path): project = parse_c_project(tmp_path) param_type = project.functions["step"].parameters[0].type - assert param_type.tag == "state" - assert param_type.resolved.fields[0].name == "id" + assert param_type.components[-1] is project.structs["state"] + assert param_type.components[-1].members[0].name == "id" def test_include_guards_do_not_duplicate_symbols(tmp_path: Path): @@ -97,4 +97,3 @@ def test_header_source_pairing_links_matching_stems(tmp_path: Path): project = parse_c_project(tmp_path) assert project.header_source_pairs["solver.h"] == {"solver.c"} - diff --git a/tests/parser/c/test_c_public_api_skeleton.py b/tests/parser/c/test_c_public_api_skeleton.py index 3df083cd2..429e89b81 100644 --- a/tests/parser/c/test_c_public_api_skeleton.py +++ b/tests/parser/c/test_c_public_api_skeleton.py @@ -54,6 +54,34 @@ def test_parse_c_project_accepts_mapping_sources(): assert set(project.functions) == {"answer"} +def test_parse_c_project_indexes_forward_structs_by_tag_name(): + from c_parser import parse_c_project + + project = parse_c_project( + { + "types.h": "struct handle;\n", + "api.h": "struct handle *open_handle(void);\n", + } + ) + + assert set(project.structs) == {"handle"} + assert project.structs["handle"].is_incomplete is True + assert project.files["types.h"].structs[0].name == "handle" + + +def test_parse_c_project_indexes_named_union_and_enum_tags(): + from c_parser import parse_c_project + + project = parse_c_project( + { + "types.h": "union value { int i; }; enum status { STATUS_OK = 0 };", + } + ) + + assert set(project.unions) == {"value"} + assert set(project.enums) == {"status"} + + def test_parse_c_project_accepts_directory_input_with_c_and_h_files(tmp_path: Path): from c_parser import parse_c_project @@ -81,13 +109,54 @@ def test_c_file_serialization_is_json_stable(): "unions": [], "enums": [], "typedefs": [], - "globals": [], + "variables": [], "macros": [], "includes": [], "diagnostics": [], } +def test_concrete_type_serialization_preserves_semantic_type_fields_and_locations(): + from c_parser import parse_c_file + + parsed = parse_c_file( + "typedef int (*compare_fn)(const void *, const void *);\n" + "compare_fn select_compare(void);\n", + filename="types.h", + ) + payload = parsed.to_dict() + + typedef = payload["typedefs"][0] + assert typedef["model"] == "CTypedef" + assert typedef["type"]["model"] == "CComposedType" + assert typedef["type"]["components"][0]["model"] == "CPointer" + assert typedef["type"]["components"][1]["model"] == "CFunctionType" + assert typedef["type"]["components"][1]["result_type"]["model"] == "CInt" + first_parameter = typedef["type"]["components"][1]["parameter_types"][0] + assert first_parameter["components"][-1]["qualifiers"] == ["const"] + assert typedef["source_location"]["filename"] == "types.h" + assert typedef["source_location"]["line"] == 1 + + function = payload["functions"][0] + assert function["result_type"]["model"] == "CTypedef" + assert function["result_type"]["name"] == "compare_fn" + assert function["source_location"]["line"] == 2 + + +def test_inline_aggregate_typedef_serialization_uses_references_without_cycles(): + from c_parser import parse_c_file + + payload = parse_c_file( + "typedef struct node { struct node *next; } node_t;\n", + filename="node.h", + ).to_dict() + + assert payload["structs"][0]["model"] == "CStruct" + assert payload["structs"][0]["members"][0]["type"]["components"][-1]["model"] == "CStruct" + assert payload["typedefs"][0]["model"] == "CTypedef" + assert payload["typedefs"][0]["type"] == {"reference": "struct node"} + + def test_public_c_parser_entrypoints_do_not_include_parser_side_readiness(): import c_parser diff --git a/tests/parser/c/test_c_structs_unions_enums_typedefs.py b/tests/parser/c/test_c_structs_unions_enums_typedefs.py index b3ce69ee0..81b0d76d7 100644 --- a/tests/parser/c/test_c_structs_unions_enums_typedefs.py +++ b/tests/parser/c/test_c_structs_unions_enums_typedefs.py @@ -1,85 +1,85 @@ # -*- coding: utf-8 -*- -"""Planned C aggregate type, enum, and typedef parser tests.""" +"""C aggregate type, enum, and typedef parser tests.""" -import pytest -pytestmark = pytest.mark.skip( - reason="C parser type roadmap tests; unskip with struct/union/enum/typedef implementation." -) - - -def test_named_struct_fields_are_parsed_with_source_order(): - from c_parser import parse_c_file +def test_named_struct_members_are_variables_in_source_order(): + from c_parser import CArray, CComposedType, CVariable, parse_c_file parsed = parse_c_file( - """ -struct point { - double x; - double y; -}; -""", + "struct point { double x; double y; double coordinates[2]; };\n", filename="structs.h", ) point = parsed.structs[0] - assert point.name == "point" - assert [field.name for field in point.fields] == ["x", "y"] + assert [member.name for member in point.members] == ["x", "y", "coordinates"] + assert all(isinstance(member, CVariable) for member in point.members) + assert isinstance(point.members[2].type, CComposedType) + assert isinstance(point.members[2].type.components[0], CArray) + assert point.members[2].type.components[0].bound == "2" -def test_typedef_struct_alias_links_alias_to_struct_definition(): +def test_typedef_struct_alias_refers_to_the_concrete_struct_object(): from c_parser import parse_c_file parsed = parse_c_file( - """ -typedef struct point { - double x; - double y; -} point_t; -""", + "typedef struct point { double x; double y; } point_t;\n", filename="typedef_struct.h", ) assert parsed.structs[0].name == "point" assert parsed.typedefs[0].name == "point_t" - assert parsed.typedefs[0].type.resolved is parsed.structs[0] + assert parsed.typedefs[0].type is parsed.structs[0] def test_anonymous_struct_typedef_gets_stable_anonymous_id(): from c_parser import parse_c_file - parsed = parse_c_file( - """ -typedef struct { - int code; -} result_t; -""", - filename="anon_struct.h", - ) + parsed = parse_c_file("typedef struct { int code; } result_t;\n", filename="anon_struct.h") assert parsed.structs[0].name is None assert parsed.structs[0].anonymous_id - assert parsed.typedefs[0].name == "result_t" + assert parsed.typedefs[0].type is parsed.structs[0] -def test_union_fields_are_parsed_without_confusing_struct_tags(): - from c_parser import parse_c_file +def test_union_members_are_variables_without_struct_field_class(): + from c_parser import CUnion, CVariable, parse_c_file + + parsed = parse_c_file("union value { int i; double d; };\n", filename="union.h") + + value = parsed.unions[0] + assert isinstance(value, CUnion) + assert [member.name for member in value.members] == ["i", "d"] + assert all(isinstance(member, CVariable) for member in value.members) + + +def test_anonymous_union_typedef_refers_to_the_concrete_union_object(): + from c_parser import CUnion, parse_c_file + + parsed = parse_c_file("typedef union { int i; double d; } value_t;\n", filename="anon_union.h") + + assert isinstance(parsed.unions[0], CUnion) + assert parsed.unions[0].anonymous_id + assert parsed.typedefs[0].type is parsed.unions[0] + + +def test_incomplete_union_and_tag_typedef_aliases_use_concrete_tag_classes(): + from c_parser import CStruct, CUnion, parse_c_file parsed = parse_c_file( - """ -union value { - int i; - double d; -}; -""", - filename="union.h", + "struct handle;\nunion payload;\ntypedef struct handle handle_t;\ntypedef union payload payload_t;\n", + filename="tag_aliases.h", ) - value = parsed.unions[0] - assert value.name == "value" - assert [field.name for field in value.fields] == ["i", "d"] + assert parsed.structs[0].is_incomplete is True + assert parsed.unions[0].is_incomplete is True + typedefs = {typedef.name: typedef for typedef in parsed.typedefs} + assert isinstance(typedefs["handle_t"].type, CStruct) + assert typedefs["handle_t"].type.name == "handle" + assert isinstance(typedefs["payload_t"].type, CUnion) + assert typedefs["payload_t"].type.name == "payload" -def test_enum_constants_preserve_explicit_and_implicit_values(): +def test_enum_constants_preserve_explicit_implicit_and_symbolic_values(): from c_parser import parse_c_file parsed = parse_c_file( @@ -87,68 +87,93 @@ def test_enum_constants_preserve_explicit_and_implicit_values(): enum status { STATUS_OK = 0, STATUS_WARN, - STATUS_ERROR = 10 + STATUS_ERROR = 10, + STATUS_NEXT = STATUS_ERROR + 1 }; """, filename="enum.h", ) - enum = parsed.enums[0] - assert [(item.name, item.value) for item in enum.constants] == [ + assert [(item.name, item.value) for item in parsed.enums[0].constants] == [ ("STATUS_OK", "0"), ("STATUS_WARN", None), ("STATUS_ERROR", "10"), + ("STATUS_NEXT", "STATUS_ERROR + 1"), ] -def test_forward_declared_struct_pointer_is_modeled_as_opaque_type(): - from c_parser import parse_c_file +def test_typedef_enum_and_trailing_tag_variable_are_separate_objects(): + from c_parser import CEnum, CStruct, parse_c_file parsed = parse_c_file( - """ -struct handle; -struct handle *open_handle(void); -void close_handle(struct handle *handle); -""", - filename="opaque.h", + "typedef enum { FLAG_NONE = 0, FLAG_READ = 1 } flag_t;\nstruct point { int x; } origin;\n", + filename="tag_declarators.h", ) - assert parsed.structs[0].name == "handle" - assert parsed.structs[0].opaque is True - assert parsed.structs[0].fields == [] - assert parsed.structs[0].requires_user_policy is True + assert parsed.enums[0].anonymous_id + assert isinstance(parsed.typedefs[0].type, CEnum) + assert parsed.typedefs[0].type is parsed.enums[0] + assert parsed.variables[0].name == "origin" + assert isinstance(parsed.variables[0].type, CStruct) + assert parsed.variables[0].type is parsed.structs[0] -def test_recursive_struct_pointer_does_not_recurse_infinitely(): - from c_parser import parse_c_file +def test_recursive_struct_pointer_uses_an_incomplete_struct_component_without_cycles(): + from c_parser import CComposedType, CPointer, CStruct, parse_c_file parsed = parse_c_file( - """ -typedef struct node { - int value; - struct node *next; -} node_t; -""", + "typedef struct node { int value; struct node *next; } node_t;\n", filename="recursive_struct.h", ) node = parsed.structs[0] - assert node.fields[1].type.pointers - assert node.fields[1].type.tag == "node" + next_type = node.members[1].type + assert isinstance(next_type, CComposedType) + assert isinstance(next_type.components[0], CPointer) + assert isinstance(next_type.components[1], CStruct) + assert next_type.components[1].name == "node" + assert next_type.components[1].is_incomplete is True -def test_typedef_chains_resolve_to_final_underlying_type(): - from c_parser import parse_c_file +def test_typedef_chains_preserve_typedef_objects_before_resolution(): + from c_parser import CTypedef, CUnsignedLong, parse_c_file parsed = parse_c_file( - """ -typedef unsigned long size_type; -typedef size_type api_size; -api_size count(void); -""", + "typedef unsigned long size_type;\ntypedef size_type api_size;\napi_size count(void);\n", filename="typedef_chain.h", ) - fn = parsed.functions[0] - assert fn.return_type.typedef_name == "api_size" - assert fn.return_type.resolved.base == "unsigned long" + typedefs = {typedef.name: typedef for typedef in parsed.typedefs} + assert isinstance(typedefs["size_type"].type, CUnsignedLong) + assert isinstance(typedefs["api_size"].type, CTypedef) + assert typedefs["api_size"].type.name == "size_type" + assert isinstance(parsed.functions[0].result_type, CTypedef) + + +def test_struct_members_use_same_components_for_callbacks_arrays_and_bitfields(): + from c_parser import CArray, CFunctionType, CPointer, parse_c_file + + parsed = parse_c_file( + "struct hooks { int (*compare)(const void *, const void *); unsigned enabled : 1; int values[4]; };\n", + filename="members.h", + ) + + compare, enabled, values = parsed.structs[0].members + assert [type(component) for component in compare.type.components] == [CPointer, CFunctionType] + assert compare.callback_candidate is True + assert enabled.bit_width == "1" + assert isinstance(values.type.components[0], CArray) + assert values.type.components[0].bound == "4" + + +def test_nested_aggregate_member_definition_is_diagnosed_explicitly(): + from c_parser import parse_c_file + + parsed = parse_c_file( + "struct outer { struct { int nested; } inner; int kept; };\n", + filename="nested_member.h", + ) + + assert [member.name for member in parsed.structs[0].members] == ["kept"] + assert parsed.diagnostics[0].code == "C_UNSUPPORTED_FIELD_DECLARATION" + assert parsed.diagnostics[0].unit_kind == "struct_field"