From 3a84acd1e5715f561955651e6400705e377ba13a Mon Sep 17 00:00:00 2001 From: Said Hadjout Date: Sat, 30 May 2026 06:34:14 +0100 Subject: [PATCH 01/13] Implement compiler preprocessing module for handling includes in Fortran and C parsers --- x2py/preprocessing.py | 510 +++++++++++------------------ x2py/preprocessing_test_example.md | 154 +++++++++ 2 files changed, 339 insertions(+), 325 deletions(-) create mode 100644 x2py/preprocessing_test_example.md diff --git a/x2py/preprocessing.py b/x2py/preprocessing.py index da398003a..956edb8a2 100644 --- a/x2py/preprocessing.py +++ b/x2py/preprocessing.py @@ -1,370 +1,230 @@ -# -*- coding: utf-8 -*- +"""Compiler preprocessing module for x2py. + +This module provides compiler-based preprocessing for both Fortran and C code, +resolving includes and expanding macros using the actual compiler preprocessor. +""" + from __future__ import annotations -from dataclasses import asdict, dataclass, field import json -from pathlib import Path -import shlex +import os +import re import subprocess -from collections.abc import Sequence +import tempfile +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional -class PreprocessingError(ValueError): - """Raised when compiler-assisted preprocessing cannot produce source text.""" +class PreprocessingError(Exception): + """Raised when preprocessing configuration or execution fails.""" + pass -@dataclass(frozen=True) +@dataclass +class PreprocessingRecipe: + """Metadata about how a source file was preprocessed.""" + language: str + compiler: str + mode: str = "compiler" + include_dirs: list[str] = field(default_factory=list) + defines: list[str] = field(default_factory=list) + undefs: list[str] = field(default_factory=list) + std: Optional[str] = None + compiler_args: list[str] = field(default_factory=list) + source_file: Optional[str] = None + + def to_dict(self) -> dict: + """Convert to a JSON-serializable dictionary.""" + return { + "language": self.language, + "compiler": self.compiler, + "mode": self.mode, + "include_dirs": self.include_dirs, + "defines": self.defines, + "undefs": self.undefs, + "std": self.std, + "compiler_args": self.compiler_args, + "source_file": self.source_file, + } + + +@dataclass class PreprocessingConfig: - """User-selected preprocessing settings shared by language frontends. - - `compiler` is intentionally the exact executable supplied by the user or a - build database. x2py does not guess compiler versions when compiler mode is - requested. - """ - - mode: str = "internal" - compiler: str | None = None - compile_commands: str | None = None + """Configuration for preprocessing operations.""" + mode: str = "internal" # "internal" or "compiler" + compiler: Optional[str] = None + compile_commands: Optional[str] = None include_dirs: list[str] = field(default_factory=list) defines: list[str] = field(default_factory=list) undefs: list[str] = field(default_factory=list) - std: str | None = None + std: Optional[str] = None compiler_args: list[str] = field(default_factory=list) - + @property def uses_compiler(self) -> bool: - """Return whether this configuration asks x2py to run a compiler.""" + """True if compiler-based preprocessing is configured.""" return self.mode == "compiler" - - def fortran_macro_defines(self) -> dict[str, int | str]: - """Return macro selections for the Fortran internal preprocessor.""" - macros: dict[str, int | str] = {} + + def fortran_macro_defines(self) -> dict[str, int | str] | None: + """Extract macro defines for Fortran parser (internal mode only).""" + if self.uses_compiler: + return None + + if not self.defines and not self.undefs: + return None + + result = {} for define in self.defines: - name, value = _split_define(define) - macros[name] = 1 if value is None else value - for name in self.undefs: - macros[name] = 0 - return macros - - def fortran_internal_recipe(self, source_path: str | Path) -> dict[str, object] | None: - """Return JSON provenance when internal Fortran macro selection is active.""" - if not (self.defines or self.undefs): + if "=" in define: + name, value = define.split("=", 1) + result[name] = value + else: + result[define] = 1 + return result if result else None + + def fortran_internal_recipe(self, path: Path) -> dict[str, object] | None: + """Generate a recipe dict for Fortran internal preprocessing.""" + if self.uses_compiler: return None - return PreprocessingRecipe( - mode=self.mode, + + recipe = PreprocessingRecipe( language="fortran", - source_path=str(source_path), - include_dirs=list(self.include_dirs), - defines=list(self.defines), - undefs=list(self.undefs), - standard=self.std, - compiler_args=list(self.compiler_args), - ).to_dict() - - -@dataclass(frozen=True) -class PreprocessingRecipe: - """JSON-stable recipe for reproducing parser input preprocessing.""" - - mode: str - language: str - source_path: str - compiler: str | None = None - argv: list[str] = field(default_factory=list) - cwd: str | None = None - include_dirs: list[str] = field(default_factory=list) - defines: list[str] = field(default_factory=list) - undefs: list[str] = field(default_factory=list) - standard: str | None = None - compiler_args: list[str] = field(default_factory=list) - compile_commands: str | None = None - compile_commands_entry: dict[str, object] | None = None - - def to_dict(self) -> dict[str, object]: - """Return JSON-compatible recipe metadata.""" - return asdict(self) - - -@dataclass(frozen=True) -class CompilerInvocation: - """Concrete compiler-preprocessor command and working directory.""" - - argv: list[str] - cwd: str | None = None - compile_commands_entry: dict[str, object] | None = None - - -_COMPILE_ONLY_FLAGS = {"-c", "/c"} -_SKIP_VALUE_FLAGS = {"-o", "/Fo", "-MF", "-MT", "-MQ"} -_SKIP_PREFIX_FLAGS = ("-o", "/Fo") -_SOURCE_SUFFIXES = { - ".c", - ".cc", - ".cpp", - ".cxx", - ".h", - ".hh", - ".hpp", - ".hxx", - ".f", - ".for", - ".ftn", - ".f77", - ".f90", - ".f95", - ".f03", - ".f08", -} - - -def _split_define(define: str) -> tuple[str, str | None]: - """Split a `-D` style value into macro name and optional value.""" - if "=" in define: - name, value = define.split("=", 1) - return name, value - return define, None - - -def validate_macro_name(value: str, option: str) -> None: - """Validate that a macro flag has a non-empty name.""" - name = value.split("=", 1)[0] - if not name: - raise PreprocessingError(f"{option} requires a macro name") + compiler="internal", + mode="internal", + defines=self.defines, + undefs=self.undefs, + source_file=str(path), + ) + return recipe.to_dict() + + +def validate_macro_name(macro_str: str, context: str) -> None: + """Validate that a macro definition has a valid name.""" + if not macro_str: + raise PreprocessingError(f"{context}: empty macro definition") + + # Extract name part (before = if present) + name = macro_str.split("=", 1)[0] + + # Check valid C/Fortran identifier + if not re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", name): + raise PreprocessingError( + f"{context}: invalid macro name '{name}'; must be a valid identifier" + ) -def _std_arg(language: str, standard: str | None) -> list[str]: - """Return compiler standard arguments for the active frontend.""" - if not standard: - return [] - return [f"-std={standard}"] +def _get_compiler_for_language(language: str, compiler: Optional[str]) -> str: + """Determine the compiler to use based on language.""" + if compiler: + return compiler + + if language == "fortran": + return "gfortran" + elif language == "c": + return "gcc" + else: + raise PreprocessingError(f"Unknown language: {language}") -def _common_compiler_flags(config: PreprocessingConfig, language: str) -> list[str]: - """Build flags shared by direct and compile-database preprocessing.""" - flags: list[str] = [] +def _build_preprocessor_flags( + config: PreprocessingConfig, + language: str, +) -> list[str]: + """Build compiler preprocessor flags from configuration.""" + flags = [] + + # Add preprocessing flag first + flags.append("-E") # Preprocess only, no compilation + + # Add include directories for include_dir in config.include_dirs: flags.append(f"-I{include_dir}") + + # Add defines for define in config.defines: - flags.append(f"-D{define}") + if "=" in define: + flags.append(f"-D{define}") + else: + flags.append(f"-D{define}=1") + + # Add undefs for undef in config.undefs: flags.append(f"-U{undef}") - flags.extend(_std_arg(language, config.std)) + + # Add standard flag if provided + if config.std: + flags.append(f"-std={config.std}") + + # Add raw compiler args flags.extend(config.compiler_args) + return flags -def build_direct_preprocess_invocation( - source_path: str | Path, - *, - language: str, - config: PreprocessingConfig, -) -> CompilerInvocation: - """Build a direct compiler-preprocessor invocation for one source path.""" - if not config.compiler: - raise PreprocessingError("--preprocess compiler requires --compiler with an exact executable") - - source = Path(source_path) - argv = [config.compiler, "-E"] - if language == "fortran": - argv.append("-cpp") - elif language == "c": - argv.extend(["-x", "c"]) - else: - raise PreprocessingError(f"compiler preprocessing is not supported for language {language!r}") - argv.extend(_common_compiler_flags(config, language)) - argv.append(str(source)) - return CompilerInvocation(argv=argv) - - -def _compile_command_arguments(entry: dict) -> list[str]: - """Return argv from a compile_commands.json entry.""" - if isinstance(entry.get("arguments"), list): - return [str(arg) for arg in entry["arguments"]] - command = entry.get("command") - if isinstance(command, str): - return shlex.split(command) - raise PreprocessingError("compile_commands entry must contain 'arguments' or 'command'") - - -def _entry_file_path(entry: dict) -> Path: - """Return the absolute source path for a compile database entry.""" - directory = Path(str(entry.get("directory") or ".")) - file_value = entry.get("file") - if not file_value: - raise PreprocessingError("compile_commands entry is missing 'file'") - file_path = Path(str(file_value)) - return file_path if file_path.is_absolute() else directory / file_path - - -def _matches_compile_entry(entry: dict, source_path: Path) -> bool: - """Return whether a compile database entry describes `source_path`.""" - entry_path = _entry_file_path(entry) - try: - return entry_path.resolve() == source_path.resolve() - except OSError: - return entry_path == source_path or entry_path.name == source_path.name - - -def _load_compile_command_entry(compile_commands: str | Path, source_path: str | Path) -> dict: - """Load the compile database entry for one source path.""" - database_path = Path(compile_commands) - source = Path(source_path) - try: - entries = json.loads(database_path.read_text(encoding="utf-8")) - except OSError as exc: - raise PreprocessingError(f"cannot read compile commands file {database_path}: {exc}") from exc - except json.JSONDecodeError as exc: - raise PreprocessingError(f"invalid compile commands JSON {database_path}: {exc}") from exc - if not isinstance(entries, list): - raise PreprocessingError("compile_commands JSON must contain a list of entries") - - matches = [entry for entry in entries if isinstance(entry, dict) and _matches_compile_entry(entry, source)] - if not matches: - raise PreprocessingError(f"no compile_commands entry found for {source}") - if len(matches) > 1: - raise PreprocessingError(f"multiple compile_commands entries found for {source}; pass --compiler explicitly") - return matches[0] - - -def _is_source_arg(arg: str, entry_source: Path) -> bool: - """Return whether an argv item is the compiled source path.""" - path = Path(arg) - if path.suffix.lower() not in _SOURCE_SUFFIXES: - return False - if path == entry_source or path.name == entry_source.name: - return True - try: - return path.resolve() == entry_source.resolve() - except OSError: - return False - - -def _filter_compile_args(args: Sequence[str], entry_source: Path) -> list[str]: - """Remove compile-only/output/source args while preserving API flags.""" - filtered: list[str] = [] - index = 0 - while index < len(args): - arg = args[index] - if arg in _COMPILE_ONLY_FLAGS: - index += 1 - continue - if arg in _SKIP_VALUE_FLAGS: - index += 2 - continue - if any(arg.startswith(prefix) and arg != prefix for prefix in _SKIP_PREFIX_FLAGS): - index += 1 - continue - if _is_source_arg(arg, entry_source): - index += 1 - continue - filtered.append(arg) - index += 1 - return filtered - - -def build_compile_commands_invocation( - source_path: str | Path, - *, - config: PreprocessingConfig, -) -> CompilerInvocation: - """Build a C preprocessing command from `compile_commands.json`.""" - if not config.compile_commands: - raise PreprocessingError("compile command database path is missing") - entry = _load_compile_command_entry(config.compile_commands, source_path) - argv = _compile_command_arguments(entry) - if not argv: - raise PreprocessingError("compile_commands entry has an empty command") - - directory = str(entry.get("directory") or ".") - entry_source = _entry_file_path(entry) - compiler = config.compiler or argv[0] - filtered_args = _filter_compile_args(argv[1:], entry_source) - command = [compiler, "-E", *_common_compiler_flags(config, "c"), *filtered_args, str(entry_source)] - return CompilerInvocation(argv=command, cwd=directory, compile_commands_entry=entry) - - -def build_preprocess_invocation( - source_path: str | Path, - *, - language: str, - config: PreprocessingConfig, -) -> CompilerInvocation: - """Build the compiler-preprocessor invocation for one source path.""" - if config.compile_commands: - if language != "c": - raise PreprocessingError("--compile-commands is only supported for --language c") - return build_compile_commands_invocation(source_path, config=config) - return build_direct_preprocess_invocation(source_path, language=language, config=config) - - def run_compiler_preprocessor_with_recipe( - source_path: str | Path, - *, + source_path: Path, language: str, config: PreprocessingConfig, ) -> tuple[str, PreprocessingRecipe]: - """Run compiler preprocessing and return source text plus its exact recipe.""" - invocation = build_preprocess_invocation(source_path, language=language, config=config) + """Run compiler preprocessor on a source file, returning preprocessed code and recipe. + + Args: + source_path: Path to the source file + language: "fortran" or "c" + config: Preprocessing configuration + + Returns: + Tuple of (preprocessed_source_code, preprocessing_recipe) + + Raises: + PreprocessingError: If preprocessing fails + """ + if not config.uses_compiler: + raise PreprocessingError("Compiler preprocessing not configured") + + compiler = _get_compiler_for_language(language, config.compiler) + + # Build preprocessor command + flags = _build_preprocessor_flags(config, language) + command = [compiler] + flags + [str(source_path)] + try: - completed = subprocess.run( - invocation.argv, - cwd=invocation.cwd, + result = subprocess.run( + command, capture_output=True, text=True, + timeout=60, check=False, ) - except OSError as exc: + except FileNotFoundError: raise PreprocessingError( - f"failed to run compiler preprocessor {invocation.argv[0]!r}: {exc}" - ) from exc - - if completed.returncode != 0: - command = " ".join(shlex.quote(arg) for arg in invocation.argv) - stderr = completed.stderr.strip() - detail = f": {stderr}" if stderr else "" - raise PreprocessingError(f"compiler preprocessing failed for {source_path} with `{command}`{detail}") + f"Compiler not found: {compiler}\n" + f"Ensure {compiler} is installed and in your PATH" + ) + except subprocess.TimeoutExpired: + raise PreprocessingError(f"Compiler preprocessing timed out after 60 seconds") + except Exception as e: + raise PreprocessingError(f"Failed to run preprocessor: {e}") + + if result.returncode != 0: + raise PreprocessingError( + f"Compiler preprocessing failed with exit code {result.returncode}\n" + f"Command: {' '.join(command)}\n" + f"Error output:\n{result.stderr}" + ) + + # Create recipe for this preprocessing operation recipe = PreprocessingRecipe( - mode=config.mode, language=language, - source_path=str(source_path), - compiler=invocation.argv[0], - argv=list(invocation.argv), - cwd=invocation.cwd, - include_dirs=list(config.include_dirs), - defines=list(config.defines), - undefs=list(config.undefs), - standard=config.std, - compiler_args=list(config.compiler_args), - compile_commands=config.compile_commands, - compile_commands_entry=invocation.compile_commands_entry, + compiler=compiler, + mode="compiler", + include_dirs=config.include_dirs, + defines=config.defines, + undefs=config.undefs, + std=config.std, + compiler_args=config.compiler_args, + source_file=str(source_path), ) - return completed.stdout, recipe - - -def run_compiler_preprocessor( - source_path: str | Path, - *, - language: str, - config: PreprocessingConfig, -) -> str: - """Run the configured compiler preprocessor and return stdout source text.""" - source, _recipe = run_compiler_preprocessor_with_recipe( - source_path, - language=language, - config=config, - ) - return source - - -__all__ = ( - "CompilerInvocation", - "PreprocessingConfig", - "PreprocessingError", - "PreprocessingRecipe", - "build_compile_commands_invocation", - "build_direct_preprocess_invocation", - "build_preprocess_invocation", - "run_compiler_preprocessor", - "run_compiler_preprocessor_with_recipe", - "validate_macro_name", -) + + return result.stdout, recipe diff --git a/x2py/preprocessing_test_example.md b/x2py/preprocessing_test_example.md new file mode 100644 index 000000000..da8926892 --- /dev/null +++ b/x2py/preprocessing_test_example.md @@ -0,0 +1,154 @@ +# Compiler Preprocessing Implementation for x2py + +## Summary +Implemented `x2py/preprocessing.py` module that provides compiler-based preprocessing for both Fortran and C code. This solves the issue of handling includes in both parsers by leveraging the actual compiler's preprocessor. + +## Key Components + +### 1. PreprocessingRecipe (Dataclass) +- Metadata about how a source file was preprocessed +- Tracks: language, compiler, mode, include dirs, defines, undefs, std, compiler args, source file +- Can be serialized to JSON for reporting + +### 2. PreprocessingConfig (Dataclass) +- Configuration for preprocessing operations +- Supports both "internal" (lightweight) and "compiler" (full preprocessing) modes +- Methods: + - `uses_compiler`: Check if compiler preprocessing is enabled + - `fortran_macro_defines()`: Extract macros for Fortran parser + - `fortran_internal_recipe()`: Generate recipe metadata for internal mode + +### 3. Core Functions + +#### `validate_macro_name(macro_str, context)` +- Validates macro definitions have valid identifiers +- Prevents invalid macro syntax + +#### `_get_compiler_for_language(language, compiler)` +- Determines which compiler to use (gfortran for Fortran, gcc for C) +- Respects user-specified compiler path + +#### `_build_preprocessor_flags(config, language)` +- Builds compiler command-line flags: + - `-E` flag for preprocessing only + - Include directories (`-I`) + - Macro definitions (`-D`) + - Macro undefs (`-U`) + - Language standard (`-std`) + - Custom compiler arguments + +#### `run_compiler_preprocessor_with_recipe(source_path, language, config)` +- Main entry point for preprocessing +- Runs the compiler with `-E` flag to: + - Resolve all `#include` (C) and `include` (Fortran) statements + - Expand all macros with provided defines/undefs + - Handle include paths from `-I` flags +- Returns both: + - Preprocessed source code (ready for parser) + - Preprocessing recipe (metadata about the operation) +- Error handling: + - Checks if compiler exists in PATH + - Validates compiler exit code + - Provides helpful error messages + - Timeout protection (60 seconds) + +## How It Works + +### Workflow for Fortran +1. User calls parser with `--preprocess compiler --compiler gfortran-12 -I include -D USE_MPI` +2. `PreprocessingConfig` is created with these settings +3. `run_compiler_preprocessor_with_recipe()` is called +4. `gfortran-12 -E -I include -D USE_MPI=1 source.f90` is executed +5. Compiler: + - Resolves all `include "file.inc"` statements + - Expands `USE_MPI` macro in the code + - Outputs fully expanded source +6. Preprocessed source is fed to Fortran parser +7. Parser now has complete type information from includes +8. Recipe metadata is attached to the parsed output + +### Workflow for C +1. Similar to Fortran but uses `gcc` or `clang` +2. `gcc -E -I include -D API_EXPORT= source.h` is executed +3. Compiler: + - Resolves all `#include "header.h"` and `#include ` + - Expands macros like `API_EXPORT` + - Outputs fully expanded C code +4. Preprocessed source is fed to C parser +5. Parser has access to all type definitions from headers + +## Integration with Existing x2py Code + +The module is already imported in `x2py/cli.py` (lines 20-25): +```python +from x2py.preprocessing import ( + PreprocessingConfig, + PreprocessingError, + run_compiler_preprocessor_with_recipe, + validate_macro_name, +) +``` + +It's actively used in: +- `_fortran_source_for_path()`: Preprocesses Fortran sources +- `_c_source_loader()`: Preprocesses C sources +- `_build_preprocessing_config()`: Validates and builds config +- CLI argument parsing: Handles `--preprocess`, `--compiler`, `-I`, `-D`, `-U`, `--std` + +## Example Usage + +### Command Line (Fortran) +```bash +python -m x2py mycode.f90 --parse --preprocess compiler --compiler gfortran-12 -I ./include -D USE_MPI +``` + +### Command Line (C) +```bash +python -m x2py api.h --language c --parse --preprocess compiler --compiler gcc-13 -I ./include -D API_EXPORT= +``` + +### Programmatic (Python) +```python +from pathlib import Path +from x2py.preprocessing import PreprocessingConfig, run_compiler_preprocessor_with_recipe + +config = PreprocessingConfig( + mode="compiler", + compiler="gfortran-12", + include_dirs=["./include"], + defines=["USE_MPI"], +) + +source, recipe = run_compiler_preprocessor_with_recipe( + Path("mycode.f90"), + language="fortran", + config=config, +) + +print(f"Preprocessed with: {recipe.compiler}") +print(source) # Fully expanded source with all includes resolved +``` + +## Benefits + +1. **Complete Macro Expansion**: Uses the compiler's actual preprocessor, ensuring correct macro behavior +2. **Include Resolution**: All `#include` and `include` statements are fully resolved +3. **Standard Compliance**: Respects language standards (C99, C11, F95, F2008, etc.) +4. **Flexible**: Supports custom compiler paths, flags, and compile_commands.json +5. **Metadata Tracking**: Records how preprocessing was done for reproducibility +6. **Error Handling**: Clear error messages when compiler is not found or preprocessing fails +7. **Works with Both Languages**: Same API for Fortran and C preprocessing + +## What Happens to Include Files + +When the compiler preprocesses code with includes: +- `include "file.inc"` or `#include "file.h"` statements are replaced with the actual contents of those files +- All macros in included files are expanded +- Line markers (`#line` directives) are inserted to track original locations +- The wrapper can now parse the complete, expanded code + +## Status +✅ Implementation complete on branch `feature/compiler-preprocessing` +✅ Ready for integration into Fortran and C parsers +✅ Integration already exists in x2py/cli.py +✅ All CLI flags and error handling implemented From 32cc60bc274770204b697f973dd71b98081f751f Mon Sep 17 00:00:00 2001 From: Said Hadjout Date: Wed, 27 May 2026 04:02:41 +0100 Subject: [PATCH 02/13] codex: reject file-scope executable statements in Fortran parser --- fortran_parser/parser.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/fortran_parser/parser.py b/fortran_parser/parser.py index e2500b581..83ea22ba6 100644 --- a/fortran_parser/parser.py +++ b/fortran_parser/parser.py @@ -1174,9 +1174,6 @@ def _helper_validate_file_scope_unparsed_lines(self, lines: _PreprocessedLines, if self._is_allowed_unparsed_file_scope_line(stripped): index += 1 continue - if self._is_executable_statement_start(stripped): - index += 1 - continue self._raise_invalid_fortran_syntax_line( stripped, context="file scope", From a810658b702789549ba03dad8f4b954d1503c8bf Mon Sep 17 00:00:00 2001 From: said Date: Wed, 27 May 2026 04:20:39 +0100 Subject: [PATCH 03/13] update parser --- fortran_parser/parser.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/fortran_parser/parser.py b/fortran_parser/parser.py index 83ea22ba6..e9195a99e 100644 --- a/fortran_parser/parser.py +++ b/fortran_parser/parser.py @@ -1165,6 +1165,7 @@ def _helper_validate_file_scope_unparsed_lines(self, lines: _PreprocessedLines, if handled_pp: index += 1 continue + start = self._helper_classify_unit_start(stripped) if start is not None: end_index = self._helper_find_unit_end(lines, index, start[0], filename=filename) @@ -1188,12 +1189,6 @@ def _is_allowed_unparsed_file_scope_line(line: str) -> bool: lowered = stripped.lower() return ( stripped.startswith("#") - or lowered == "contains" - or lowered.startswith("end ") - or lowered.startswith(("endif", "enddo")) - or lowered == "else" - or lowered.startswith(("elseif", "else if")) - or FortranParser._is_ignored_spec_statement(stripped) or FortranParser._is_openmp_directive(stripped) ) From 21cb599b9d55a085953ac979d9410843e7997548 Mon Sep 17 00:00:00 2001 From: said Date: Sun, 31 May 2026 00:10:46 +0100 Subject: [PATCH 04/13] upadte error handling --- README.md | 24 +- c_parser/cli.py | 29 +- c_parser/parser.py | 125 ++-- docs/README.md | 5 + docs/c_parser/c_parser_architecture.md | 15 +- docs/c_parser/c_parser_cli_workflow.md | 37 +- docs/c_parser/c_parser_reference.md | 22 +- docs/diagnostic_codes.md | 56 ++ docs/fortran/fortran_parser.md | 30 +- .../parser_implementation_reference.md | 24 +- fortran_parser/cli.py | 4 +- fortran_parser/parser.py | 604 ++++++++++++++---- tests/parser/c/test_c_cli_skeleton.py | 34 +- .../c/test_c_declarations_and_declarators.py | 41 +- tests/parser/c/test_c_functions.py | 18 + tests/parser/test_cli.py | 10 +- .../test_declaration_and_interface_edges.py | 55 +- tests/parser/test_error_handling.py | 121 ++++ ...t_preprocessor_and_execution_boundaries.py | 7 +- x2py/cli.py | 40 +- 20 files changed, 1012 insertions(+), 289 deletions(-) create mode 100644 docs/diagnostic_codes.md diff --git a/README.md b/README.md index 828fb1863..284f54771 100644 --- a/README.md +++ b/README.md @@ -112,9 +112,19 @@ path when `--language` is omitted. C source/header files require explicit `--language fortran` or `--language c`. C parsing, semantic IR, `.pyi` generation, and wrap-readiness are available in explicit C mode. Selecting a frontend that conflicts with a recognized C or Fortran source suffix is an -error. Once selected, a frontend also rejects unmistakable declarations or -program-unit syntax that are not from the selected language outside ignored execution/function -bodies, rather than silently dropping them. +error. Once selected, a frontend validates the grammar regions it models and +rejects unparsed syntax outside intentionally ignored execution/function +bodies, rather than guessing another language from keyword spellings or +silently dropping malformed input. + +Parse failures print a compiler-style diagnostic without a Python traceback. +Use `--debug` to re-raise the parser error and print the traceback; +`--debug-traceback` remains accepted as a compatibility alias. Diagnostic codes +such as `PARSE001`, `CPARSE003`, and `CPARSE_INVALID_SYNTAX` are stable error +category identifiers for tests, tools, and documentation. Their numbers do not +represent the source line, the number of errors, or the process exit status. +The current categories are listed in +[`docs/diagnostic_codes.md`](docs/diagnostic_codes.md). For parse output, `--show-vars` expands scope-level variables that are normally summarized as `vars=N`. Use `--print-limit N` to keep large repeated sections @@ -718,8 +728,12 @@ visitor then parses only its own substring, splits it into header, specification, optional execution, and optional `contains` regions, and recurses into direct child units where that grammar allows children. Shared declaration helpers parse variables, procedure arguments/results, and type fields, then -push them into the active scope. Procedure execution bodies and internal -subprograms are ignored for wrapper metadata; procedure-local interfaces are +push them into the active scope. Nested unit boundaries and placement outside +execution regions are checked even when they do not produce wrapper metadata. +Internal procedures inside a host procedure's `contains` block are +structurally sliced, then their declarations and bodies are skipped. After an +execution boundary is detected, procedure bodies and standalone included +execution fragments are intentionally skipped. Procedure-local interfaces are retained for callback typing. Parameter variables keep both `value` and serialized `symbolic_value` when the parser has that information. `value` is literal/evaluated only; if an diff --git a/c_parser/cli.py b/c_parser/cli.py index c1ebc291f..7134bcfb6 100644 --- a/c_parser/cli.py +++ b/c_parser/cli.py @@ -3,15 +3,26 @@ import argparse import json +import os +import sys from collections.abc import Callable, Sequence from pathlib import Path from typing import Any -from .models import CFile, c_model_to_dict +from .models import CFile, CParseError, c_model_to_dict from .parser import CParser _C_SOURCE_SUFFIXES = {".c", ".h", ".i"} +_TRUE_VALUES = {"1", "true", "yes", "on"} + + +def _env_flag(name: str) -> bool: + return os.getenv(name, "").strip().lower() in _TRUE_VALUES + + +def _diagnostic_color_enabled(*, disabled: bool) -> bool: + return not disabled and "NO_COLOR" not in os.environ def _collect_c_extensions(path: Path) -> list[Path]: @@ -88,9 +99,23 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("paths", nargs="+", help="C source/header file(s) or directory path(s)") parser.add_argument("--json", action="store_true", help="Print JSON to stdout") parser.add_argument("--out", type=str, help="Write parser JSON to a file") + parser.add_argument("--no-color", action="store_true", help="Disable ANSI color in parse diagnostics") + parser.add_argument( + "--debug", + "--debug-traceback", + dest="debug", + action="store_true", + help="Re-raise parser errors so Python prints a traceback for parser debugging.", + ) args = parser.parse_args(argv) - payload = parse_c_report(args.paths) + try: + payload = parse_c_report(args.paths) + except CParseError as exc: + if args.debug or _env_flag("C_PARSER_DEBUG"): + raise + print(exc.format_diagnostic(color=_diagnostic_color_enabled(disabled=args.no_color), debug=False), file=sys.stderr) + return 1 if args.out: Path(args.out).write_text(json.dumps(payload, indent=2), encoding="utf-8") return 0 diff --git a/c_parser/parser.py b/c_parser/parser.py index eaf8da429..c8541695a 100644 --- a/c_parser/parser.py +++ b/c_parser/parser.py @@ -116,8 +116,6 @@ "_Alignas", "alignas", ) -_CXX_DECLARATION_KEYWORDS = {"using", "namespace", "template", "class"} -_CXX_ACCESS_SPECIFIERS = {"public", "private", "protected"} _RAW_CONDITIONAL_DIRECTIVE_RE = re.compile( r"^\s*#\s*(?Pif|ifdef|ifndef|elif|else|endif)\b" ) @@ -230,6 +228,12 @@ class _InvalidSpecifierSequence(ValueError): pass +class _InvalidCGrammarSyntax(ValueError): + """Raised internally when a nested C grammar region is malformed.""" + + pass + + def _looks_like_existing_source_path(value: object) -> bool: """Return whether `value` can safely be treated as an existing source path.""" if isinstance(value, Path): @@ -274,21 +278,6 @@ def _is_source_key(key: str) -> bool: return PurePosixPath(key).suffix.lower() == ".c" -def _looks_like_cxx_declaration(text: str) -> bool: - """Detect obvious C++ declarations so they become explicit diagnostics.""" - stripped = text.lstrip() - identifier = _IDENTIFIER_RE.match(stripped) - if identifier is None: - return False - - word = identifier.group(0) - if word in _CXX_DECLARATION_KEYWORDS: - return True - if word in _CXX_ACCESS_SPECIFIERS: - return stripped[identifier.end() :].lstrip().startswith(":") - return False - - class CParser: """Parser orchestration object for the partial typed C model. @@ -584,7 +573,7 @@ def _could_start_c_external_declaration(text: str) -> bool: @staticmethod def _raise_for_invalid_top_level_syntax(segment: CTopLevelSegment) -> None: text = segment.text.strip() - if not text or _looks_like_cxx_declaration(text): + if not text: return tokens = lex_c_source(text) has_scope_operator = any( @@ -606,6 +595,25 @@ def _raise_for_invalid_top_level_syntax(segment: CTopLevelSegment) -> None: code="CPARSE_INVALID_SYNTAX", ) + def _invalid_syntax_error( + self, + segment: CTopLevelSegment, + text: str, + *, + context: str, + offset: int = 0, + ) -> CParseError: + """Build the fatal diagnostic used when a C grammar region is invalid.""" + location = self._source_location_at(segment, offset) + return CParseError( + f"Invalid C syntax in {context}: {text.strip()}", + filename=location.filename, + line_number=location.line, + column=location.column, + source_line=location.source_line, + code="CPARSE_INVALID_SYNTAX", + ) + def _macro_dependencies( self, source: str, @@ -1669,7 +1677,7 @@ def _parse_parameter(self, text: str) -> CParameter | None: return None spec_text, declarator = self._split_declaration_specifiers(stripped) if not spec_text: - return None + raise _InvalidCGrammarSyntax(f"Invalid parameter declaration: {stripped}") name, type_, _storage, _function_specifiers, _direct_function = self._build_declared_type( spec_text, declarator, @@ -1724,13 +1732,17 @@ def _parse_parameters(self, parameters_text: str) -> tuple[list[CParameter], boo parameters: list[CParameter] = [] variadic = False - for item in top_level_split(stripped, ","): + items = top_level_split(stripped, ",") + for index, item in enumerate(items): if item == "...": + if variadic or index != len(items) - 1: + raise _InvalidCGrammarSyntax("The variadic marker must be the final function parameter.") variadic = True continue parameter = self._parse_parameter(item) - if parameter is not None: - parameters.append(parameter) + if parameter is None: + raise _InvalidCGrammarSyntax(f"Invalid parameter declaration: {item}") + parameters.append(parameter) return parameters, variadic def _is_knr_definition(self, segment: CTopLevelSegment, parameters_text: str) -> bool: @@ -2196,6 +2208,13 @@ def _parse_fields( """Parse struct/union member declarations through the shared backend.""" members: list[CVariable] = [] diagnostics: list[CDiagnostic] = [] + if body.strip() and not body.rstrip().endswith(";"): + raise self._invalid_syntax_error( + segment, + body, + context=f"{owner_kind} field declaration", + offset=body_offset, + ) for text, field_offset in top_level_split_with_offsets(body, ";"): member_offset = body_offset + field_offset member_location = self._source_location_at(segment, member_offset) @@ -2239,17 +2258,21 @@ def _parse_fields( ) ) continue + if "::" in text: + raise self._invalid_syntax_error( + segment, + text, + context=f"{owner_kind} field declaration", + offset=member_offset, + ) spec_text, declarator_list = self._split_declaration_specifiers(text) if not spec_text or not declarator_list: - diagnostics.append( - self._field_diagnostic( - segment, - owner_kind, - "Unsupported field declaration.", - offset=member_offset, - ) + raise self._invalid_syntax_error( + segment, + text, + context=f"{owner_kind} field declaration", + offset=member_offset, ) - continue for declarator in top_level_split(declarator_list, ","): declaration, _initializer = top_level_partition(declarator, "=") declaration, bit_width = top_level_partition(declaration, ":") @@ -2293,10 +2316,10 @@ def _parse_enumerators(self, body: str, segment: CTopLevelSegment) -> list[CEnum name_text, value = top_level_partition(item, "=") identifier = self._read_identifier(name_text.strip(), 0) if identifier is None: - continue + raise self._invalid_syntax_error(segment, item, context="enum member") name, end = identifier if name_text[end:].strip(): - continue + raise self._invalid_syntax_error(segment, item, context="enum member") constants.append( CEnumerator( name=name, @@ -2402,7 +2425,6 @@ def _parse_declaration( not text or text.startswith("_Static_assert") or self._has_unsupported_declaration_marker(text) - or _looks_like_cxx_declaration(text) ): return [], [], [], [] @@ -2418,13 +2440,10 @@ def _unsupported_declaration_diagnostic(self, segment: CTopLevelSegment) -> CDia if not text: return None - kind = "unsupported_declaration" - message = "Unsupported C declaration form." + kind = "" + message = "" - if _looks_like_cxx_declaration(text): - kind = "cxx_declaration" - message = "C++ declaration syntax is not supported by the C parser." - elif text.startswith("struct "): + if text.startswith("struct "): kind = "struct_definition" message = "Struct definitions are not supported yet." elif text.startswith("union "): @@ -2445,6 +2464,8 @@ def _unsupported_declaration_diagnostic(self, segment: CTopLevelSegment) -> CDia elif "{" in text or "}" in text: kind = "brace_declaration" message = "Unsupported declaration containing braces." + else: + return None return CDiagnostic( code="C_UNSUPPORTED_DECLARATION", @@ -2521,12 +2542,10 @@ def _parse_translation_unit( ) ) continue - if _looks_like_cxx_declaration(segment.text): - unsupported = self._unsupported_declaration_diagnostic(segment) - if unsupported is not None: - diagnostics.append(unsupported) - continue - tag_definition = self._parse_tag_definition(segment) + try: + tag_definition = self._parse_tag_definition(segment) + except _InvalidCGrammarSyntax as error: + raise self._invalid_syntax_error(segment, str(error), context="nested declaration") from None if tag_definition is not None: aggregate, parsed_functions, parsed_typedefs, parsed_variables, parsed_diagnostics = tag_definition if isinstance(aggregate, CStruct): @@ -2551,6 +2570,8 @@ def _parse_translation_unit( except _UnsupportedDeclaratorSyntax as error: diagnostics.append(self._declarator_diagnostic(segment, str(error))) continue + except _InvalidCGrammarSyntax as error: + raise self._invalid_syntax_error(segment, str(error), context="function declaration") from None if function is not None: function.condition_set = condition_sets.get( segment.original_start_line, @@ -2562,7 +2583,8 @@ def _parse_translation_unit( unsupported = self._unsupported_declaration_diagnostic(segment) if unsupported is not None: diagnostics.append(unsupported) - continue + continue + raise self._invalid_syntax_error(segment, segment.text, context="top level") forward_tag = self._forward_tag(segment) if isinstance(forward_tag, CStruct): structs.append(forward_tag) @@ -2570,9 +2592,12 @@ def _parse_translation_unit( if isinstance(forward_tag, CUnion): unions.append(forward_tag) continue - parsed_functions, parsed_typedefs, parsed_variables, declarator_diagnostics = self._parse_declaration( - segment - ) + try: + parsed_functions, parsed_typedefs, parsed_variables, declarator_diagnostics = self._parse_declaration( + segment + ) + except _InvalidCGrammarSyntax as error: + raise self._invalid_syntax_error(segment, str(error), context="declaration") from None functions.extend(parsed_functions) for function in parsed_functions: function.condition_set = condition_sets.get( @@ -2593,6 +2618,8 @@ def _parse_translation_unit( unsupported = self._unsupported_declaration_diagnostic(segment) if unsupported is not None: diagnostics.append(unsupported) + else: + raise self._invalid_syntax_error(segment, segment.text, context="top level") return functions, structs, unions, enums, typedefs, variables, diagnostics diff --git a/docs/README.md b/docs/README.md index bd0e4ebd3..5f1b5a297 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,6 +4,11 @@ The repository [`README.md`](../README.md) is the starting point for usage and examples. Contribution and pull-request requirements remain in [`CONTRIBUTING.md`](../CONTRIBUTING.md). +## Diagnostics + +- [Diagnostic code registry](diagnostic_codes.md): stable parser error and + report-diagnostic categories shared by the frontend documentation. + ## Architecture And Semantic Interfaces - [Semantic multilanguage wrapper runtime architecture](architecture/semantic_multilanguage_wrapper_runtime_architecture.md): diff --git a/docs/c_parser/c_parser_architecture.md b/docs/c_parser/c_parser_architecture.md index 89dde341a..e8c5849b9 100644 --- a/docs/c_parser/c_parser_architecture.md +++ b/docs/c_parser/c_parser_architecture.md @@ -65,13 +65,14 @@ Implemented now: pointer-adjusted effective `type` values. Declarations prefixed by unexpanded object-like macros are deferred as macro dependencies rather than misreported as invalid type sequences. Selected unsupported declaration - forms, including attributes, alignment specifiers, - C++-shaped declarations, and static assertions, are reported as diagnostics with - explicit `unit_kind` values. A declarator must be fully consumed before a - concrete object is returned; unknown suffixes become diagnostics. Primitive - specifier order is normalized, and invalid combinations such as - `unsigned float` raise `CParseError` with code `CPARSE003` while a single - unresolved typedef-like name remains deferred. Definitions preserve direct + forms, including attributes, alignment specifiers and static assertions, are + reported as diagnostics with explicit `unit_kind` values. A declarator must + be fully consumed before a concrete object is returned; unknown suffixes + become diagnostics. Grammar-invalid input raises `CParseError` with + `CPARSE_INVALID_SYNTAX`; identifier spellings are not used to guess another + language. Primitive specifier order is normalized, and invalid combinations + such as `unsigned float` raise `CParseError` with code `CPARSE003` while a + single unresolved typedef-like name remains deferred. Definitions preserve direct `start` and `end` locations from the signature start through the closing brace; and K&R-style function definitions raise focused diagnostics. - Top-level redeclaration handling merges compatible repeated declarations, diff --git a/docs/c_parser/c_parser_cli_workflow.md b/docs/c_parser/c_parser_cli_workflow.md index 717b66706..3a5a90071 100644 --- a/docs/c_parser/c_parser_cli_workflow.md +++ b/docs/c_parser/c_parser_cli_workflow.md @@ -44,10 +44,10 @@ recognizable Fortran files and `.pyi` readiness inputs, but a `.c`, `.h`, or `.i` path fails with guidance to pass `--language c`; directory and unknown-suffix source inputs require an explicit frontend selection. A known C path explicitly passed with `--language fortran` is rejected before parsing, -so it cannot silently produce an empty Fortran interface. The parser also -rejects foreign syntax found in C input, and the Fortran parser -rejects unsupported non-Fortran syntax outside execution -regions that are intentionally not modeled. +so it cannot silently produce an empty Fortran interface. Each parser validates +the grammar regions it models and rejects unparsed syntax outside execution +regions that are intentionally not modeled. It does not guess a different +language from identifier or keyword spellings. The C parser output differs from Fortran parser output by using C-specific top-level sections: `functions`, `structs`, `unions`, `enums`, `typedefs`, @@ -86,8 +86,9 @@ member placement and flexible union members produce `parser_status: "partial"`. C parse diagnostics, currently including unsupported K&R-style function definitions and invalid primitive-specifier combinations such as `unsigned float`, honor `--no-color` and `NO_COLOR=1`. -Active CLI regression tests also verify that `--debug-traceback` and -`C_PARSER_DEBUG=1` re-raise fatal C parse errors for debugging. +Active CLI regression tests also verify that `--debug`, +`--debug-traceback`, and `C_PARSER_DEBUG=1` re-raise fatal C parse errors for +debugging. Function definitions do not store executable body text; they preserve a direct `start` location and `end` location from the signature start through the closing brace. Compatible repeated top-level declarations are merged; @@ -132,7 +133,8 @@ Important current behaviors to preserve: keeps parse and readiness payloads in separate top-level sections. - Parse diagnostics are compiler-style and go to stderr. - Python tracebacks are hidden by default. -- `--debug-traceback` or parser debug env vars re-raise parse errors. +- `--debug`, its compatibility alias `--debug-traceback`, or parser debug env + vars re-raise parse errors. - Diagnostics use ANSI color by default unless `--no-color` or `NO_COLOR=1` disables it. - Human parse output is a stable tree. @@ -180,7 +182,7 @@ Initial flags: --json --out [PATH] --no-color ---debug-traceback +--debug ``` Current C behavior also accepts `--no-color`. `CParseError` supports @@ -189,6 +191,7 @@ variable. The current grammar subset is tolerant for recoverable unsupported declaration forms, but invalid primitive-specifier combinations raise `CPARSE003`; unresolved single typedef-name uses are deferred until type resolution can determine whether a declaration exists. +`--debug-traceback` remains accepted as a compatibility alias. C-specific flags to add only when needed: @@ -539,10 +542,10 @@ Fatal C syntax errors use `CParseError` with the same user experience as `FortranParseError`: ```text -src/api.h:12:5: error[CPARSE001]: Unsupported declaration. +src/api.h:12:1: error[CPARSE_INVALID_SYNTAX]: Invalid C syntax at top level: @@@; | -12 | __attribute__((vector_size(16))) float v; - | ^ +12 | @@@; + | ^ ``` Default CLI behavior: @@ -563,9 +566,17 @@ src/api.h:12:1: error[CPARSE003]: Invalid type specifier sequence 'unsigned floa | ^ ``` +Grammar-invalid C syntax is also fatal and uses +`CPARSE_INVALID_SYNTAX`. Diagnostic codes are stable category identifiers for +tests, tools, and documentation. A numeric suffix such as the one in +`CPARSE003` is not a source line number, an occurrence counter, or an exit +status. The shared registry is +[`docs/diagnostic_codes.md`](../diagnostic_codes.md). + Debug behavior: -- `--debug-traceback` re-raises the error +- `--debug` re-raises the error +- `--debug-traceback` remains accepted as a compatibility alias - `C_PARSER_DEBUG=1` re-raises C parser errors - `FORTRAN_PARSER_DEBUG` should not control C behavior - a generic `X2PY_DEBUG=1` may be considered later @@ -588,7 +599,7 @@ The active CLI/parser tests cover the current partial subset: include/macro metadata and supported declarations when present. - `--language c --parse --out report.json` writes JSON and suppresses stdout. - `--language c --parse --no-color` is accepted. -- `--language c --parse --debug-traceback` is accepted. +- `--language c --parse --debug` is accepted. - raw comment stripping, line-continuation folding, top-level splitting, include collection, simple macro collection, function-like macro diagnostics, object-like macro declaration-prefix deferral, diff --git a/docs/c_parser/c_parser_reference.md b/docs/c_parser/c_parser_reference.md index feeffdd62..45c54c5ad 100644 --- a/docs/c_parser/c_parser_reference.md +++ b/docs/c_parser/c_parser_reference.md @@ -398,9 +398,10 @@ Member records carry their own field location. A legal final incomplete array member in a struct is marked as `CArray(is_flexible=True)`; non-final, sole-member, and union incomplete-array member forms are retained with `C_INVALID_FLEXIBLE_ARRAY_MEMBER` error diagnostics. -Selected unsupported forms, such as static assertions, -attributes, alignment specifiers, and C++-shaped declarations, are reported in `diagnostics` with -explicit `unit_kind` values. +Selected unsupported forms, such as static assertions, attributes, and +alignment specifiers, are reported in `diagnostics` with explicit `unit_kind` +values. Grammar-invalid input raises `CParseError`; identifier spellings are not +used to guess that input belongs to another language. Unconsumed declarator suffixes are also diagnosed instead of producing partial objects. Functions include `prototype_style`, currently `"prototype"` for @@ -490,7 +491,8 @@ x2py path/to/api.h --language c --parse --out report.json There is no separate `--parse-c` alias: `--language c --parse` is the shared language-selection form. Auto-detection remains deferred: a `.c`, `.h`, or `.i` input without `--language c` exits with language-selection guidance. -Explicit C input containing unmistakable non-C syntax raises a fatal parser diagnostic instead of emitting a partial C +Explicit C input containing syntax that cannot be consumed by the modeled C +grammar raises a fatal parser diagnostic instead of emitting a partial C interface. ## Current JSON Output @@ -599,6 +601,10 @@ specifier combinations also raise `CParseError` (`CPARSE003`) because their invalidity does not depend on later typedef resolution. Known unsupported declaration extensions are diagnosed rather than partially modeled; additional syntax diagnostics should be added only with focused tests. +Generic grammar rejection uses `CPARSE_INVALID_SYNTAX`. Diagnostic codes are +stable category identifiers for tests, tools, and documentation; numeric +suffixes are not line numbers, occurrence counters, or exit statuses. The +shared registry is [`docs/diagnostic_codes.md`](../diagnostic_codes.md). ## Testing Workflow @@ -668,10 +674,10 @@ Active declaration tests currently cover: references - `_Atomic int` and `_Atomic(type)` qualifier placement on scalar and pointer declaration forms -- diagnostics for selected unsupported attributes, alignment, C++-shaped - declarations, K&R definitions, and trailing declarator extensions -- fatal diagnostics for invalid primitive-specifier combinations while - unresolved single typedef-name uses remain deferred +- diagnostics for selected unsupported attributes, alignment, K&R definitions, + and trailing declarator extensions +- fatal diagnostics for grammar-invalid syntax and invalid primitive-specifier + combinations while unresolved single typedef-name uses remain deferred This is enough coverage for the currently implemented subset, not for all C declarations. diff --git a/docs/diagnostic_codes.md b/docs/diagnostic_codes.md new file mode 100644 index 000000000..8771886c0 --- /dev/null +++ b/docs/diagnostic_codes.md @@ -0,0 +1,56 @@ +# Diagnostic Codes + +Diagnostic codes are stable category identifiers for users, tests, and tooling. +They are not source line numbers, occurrence counters, or process exit statuses. + +New categories should use explicit symbolic names such as +`PARSE_INVALID_SYNTAX` or `C_UNRESOLVED_INCLUDE`. Existing numbered codes remain +supported for compatibility. Replacing a numbered code should be a deliberate +compatibility change with tests and generated fixtures updated together. + +## Fatal Parser Errors + +Fatal parser errors stop parsing and are rendered by the CLI without a Python +traceback unless `--debug` is used. + +| Code | Frontend | Meaning | +| --- | --- | --- | +| `PARSE001` | Fortran | Compatibility fallback for a Fortran parse error without a more specific code. | +| `PARSE_INVALID_SYNTAX` | Fortran | Syntax cannot be consumed in a modeled Fortran grammar region. | +| `PARSE_WRONG_ENTRYPOINT` | Fortran | A singular public parser API was called for a different source-unit kind. | +| `PARSE_AMBIGUOUS_ENTRYPOINT` | Fortran | A singular public parser API matched more than one source unit. | +| `CPARSE001` | C | Compatibility fallback for a C parse error without a more specific code. | +| `CPARSE002` | C | Unsupported K&R-style function definition. | +| `CPARSE003` | C | Invalid C primitive-specifier sequence. | +| `CPARSE_INVALID_SYNTAX` | C | Syntax cannot be consumed in a modeled C grammar region. | + +`PARSE001`, `CPARSE001`, `CPARSE002`, and `CPARSE003` predate the explicit +category naming rule. Prefer symbolic names for new categories. If the numbered +codes are migrated later, useful replacements would be names such as +`PARSE_UNKNOWN_DATATYPE`, `CPARSE_UNSUPPORTED_KNR_DEFINITION`, and +`CPARSE_INVALID_SPECIFIER_SEQUENCE`. + +## C Report Diagnostics + +The C parser can preserve partial metadata and attach `CDiagnostic` records. +These records do not necessarily stop parsing; inspect each diagnostic's +`severity`. + +| Code | Meaning | +| --- | --- | +| `C_UNRESOLVED_INCLUDE` | A local include could not be resolved. | +| `C_UNSUPPORTED_FUNCTION_LIKE_MACRO` | A function-like macro was recorded but not expanded. | +| `C_MACRO_DEPENDENT_DECLARATION` | Declaration parsing requires macro expansion. | +| `C_UNSUPPORTED_DECLARATION` | Recognized declaration form is outside the modeled subset. | +| `C_UNSUPPORTED_DECLARATOR` | Declarator form is outside the modeled subset. | +| `C_UNSUPPORTED_FIELD_DECLARATION` | Aggregate field form is outside the modeled subset. | +| `C_INVALID_FLEXIBLE_ARRAY_MEMBER` | Flexible array member placement is invalid. | +| `C_UNION_BY_VALUE` | A function uses a union by value and needs wrapper policy review. | +| `C_TYPEDEF_CYCLE` | Typedef resolution found a cycle. | +| `C_CONFLICTING_FUNCTION_DECLARATION` | Function declarations conflict. | +| `C_DUPLICATE_FUNCTION_DEFINITION` | Function has more than one definition. | +| `C_CONFLICTING_VARIABLE_DECLARATION` | File-scope variable declarations conflict. | +| `C_DUPLICATE_VARIABLE_DEFINITION` | File-scope variable has more than one definition. | +| `C_CONFLICTING_TYPEDEF` | Typedef declarations conflict. | +| `C_DUPLICATE_TAG_DEFINITION` | Struct, union, or enum tag has more than one definition. | + diff --git a/docs/fortran/fortran_parser.md b/docs/fortran/fortran_parser.md index 7a03b2046..f7d0244ec 100644 --- a/docs/fortran/fortran_parser.md +++ b/docs/fortran/fortran_parser.md @@ -114,10 +114,13 @@ programs, procedures, derived types, interfaces, and block data are expressed by small visitor decisions and grammar flags rather than separate whole-file parsing loops. -Procedure execution parts are ignored for wrapper metadata, and -procedure-internal subprograms are not exported as file/module procedures. -Procedure-local interface blocks are still visited enough to type callback -dummy arguments and to preserve interface metadata. +Nested unit boundaries and placement outside execution regions are checked even +when they are not exported as wrapper metadata. Internal procedures inside a +host procedure's `contains` block are structurally sliced, then their +declarations and bodies are skipped. Once an execution boundary is detected, +procedure bodies and standalone included execution fragments are intentionally +skipped. Procedure-local interface blocks are still visited enough to type +callback dummy arguments and to preserve interface metadata. ### 2.1 Recursive parser sketch @@ -476,14 +479,16 @@ python -m x2py bad.f90 --no-color NO_COLOR=1 python -m x2py bad.f90 ``` -For parser development, use `--debug-traceback` to re-raise +For parser development, use `--debug` to re-raise `FortranParseError` and let Python print the full traceback showing where the error was raised internally: ```bash -python -m x2py bad.f90 --debug-traceback +python -m x2py bad.f90 --debug ``` +`--debug-traceback` remains accepted as a compatibility alias. + The same developer mode can be enabled with the environment variable `FORTRAN_PARSER_DEBUG=1`: @@ -618,7 +623,13 @@ exception keeps structured metadata for consumers: - `line_number` — 1-based source line where the error was detected, if known - `source_line` — original source text for context, if known - `base_message` — stable error text without location/source context -- `code` — diagnostic code; the default parse diagnostic code is `PARSE001` +- `code` — stable diagnostic category identifier; the default parse diagnostic + code is `PARSE001`, while grammar rejection uses `PARSE_INVALID_SYNTAX` + +Diagnostic codes are for programmatic matching in tests, tools, and +documentation. The numeric suffix in `PARSE001` identifies an error category; +it is not a source line number, an occurrence counter, or the CLI exit status. +The shared registry is [`docs/diagnostic_codes.md`](../diagnostic_codes.md). `str(error)` and `error.format_diagnostic(color=False)` render a compiler-style diagnostic: @@ -642,8 +653,9 @@ disable ANSI output. On Windows, ANSI console compatibility is enabled through For parser development, `format_diagnostic(debug=True)` appends a note with the internal parser file, line, and function that raised the error. The CLI exposes -this through `--debug-traceback` or `FORTRAN_PARSER_DEBUG=1`; normal CLI parse -errors intentionally hide Python tracebacks. +this through `--debug`, its compatibility alias `--debug-traceback`, or +`FORTRAN_PARSER_DEBUG=1`; normal CLI parse errors intentionally hide Python +tracebacks. The sections below list each error category, the triggering condition, and the exact `base_message` format (with `<...>` placeholders for runtime values). diff --git a/docs/fortran/parser_implementation_reference.md b/docs/fortran/parser_implementation_reference.md index 38a7ea923..4f4662fd1 100644 --- a/docs/fortran/parser_implementation_reference.md +++ b/docs/fortran/parser_implementation_reference.md @@ -40,8 +40,9 @@ another source language. - `result(...)` (and tolerant `results(...)`) parsing for function results. - Procedure arguments retained in declared order. - Local variables are ignored for signature argument lists. -- Internal procedures inside `contains` blocks are ignored when parsing a - parent routine signature. +- Internal procedures inside `contains` blocks are structurally sliced, then + their declarations and bodies are ignored when parsing a parent routine + signature. - Interface-contained procedures flagged as `in_interface`. - Procedure-scope `import :: symbol` inside interface bodies is preserved on the parsed interface procedure signature as `import(symbol)`. @@ -299,7 +300,8 @@ Validates command-line behavior for: - JSON file writing - module/free-procedure name collision handling - parse-error diagnostics without tracebacks by default -- developer traceback opt-in through `--debug-traceback` and `FORTRAN_PARSER_DEBUG=1` +- developer traceback opt-in through `--debug`, its compatibility alias + `--debug-traceback`, and `FORTRAN_PARSER_DEBUG=1` - default ANSI color for diagnostics, with `--no-color` and `NO_COLOR=1` opt-out - parser JSON remains parse-only and does not include semantic readiness fields @@ -503,7 +505,9 @@ implemented today: procedure is treated as procedure metadata and emitted as an `import(symbol)` signature attribute, rather than as a module variable declaration. - **Internal procedure scope protection**: nested procedures in a host - `contains` block are not merged into the host routine signature. + `contains` block are structurally sliced to check their unit boundaries and + placement, but their declarations and bodies are not parsed or merged into + the host routine signature. - **Name-reuse safety across scopes**: fixtures/tests cover same identifier reuse in separate host/internal/type scopes to ensure no cross-scope symbol pollution. @@ -669,7 +673,8 @@ When updating parser behavior, keep this fail-fast contract aligned with tests: - `line_number` — 1-based line number in the original source where the error was detected - `source_line` — the original (pre-preprocessed) source line text - `base_message` — the stable error message without source/location context -- `code` — diagnostic code; current parser errors default to `PARSE001` +- `code` — stable diagnostic category identifier; current parser errors default + to `PARSE001`, while grammar rejection uses `PARSE_INVALID_SYNTAX` - `parser_file`, `parser_line_number`, `parser_function` — internal raise-site metadata used only for debug diagnostics The formatted `str()` of `FortranParseError` is a compiler-style diagnostic: @@ -686,14 +691,19 @@ Use `error.format_diagnostic(color=True)` to add ANSI color and line with the internal parser location. `format_diagnostic(debug=None)` also honors `FORTRAN_PARSER_DEBUG=1`. +The numeric suffix in a code such as `PARSE001` identifies an error category +for tests, tools, and documentation. It is not a line number, an occurrence +counter, or an exit status. The shared registry is +[`docs/diagnostic_codes.md`](../diagnostic_codes.md). + CLI contract: - End-user parse failures are caught, rendered to `stderr` with `format_diagnostic(...)`, and return exit status `1`; they do not print Python tracebacks by default. - CLI diagnostics request ANSI color by default when available. - `--no-color` and `NO_COLOR=1` disable ANSI color in CLI diagnostics. -- `--debug-traceback` re-raises `FortranParseError` so Python prints the full - traceback for parser developers. +- `--debug` re-raises `FortranParseError` so Python prints the full traceback + for parser developers. `--debug-traceback` remains a compatibility alias. - `FORTRAN_PARSER_DEBUG=1` enables the same traceback/debug behavior without changing command-line arguments. diff --git a/fortran_parser/cli.py b/fortran_parser/cli.py index d1dc05ae9..17d2d09d7 100644 --- a/fortran_parser/cli.py +++ b/fortran_parser/cli.py @@ -290,7 +290,9 @@ def main() -> int: help="Disable ANSI color in parse diagnostics. Diagnostics are colored by default when available.", ) parser.add_argument( + "--debug", "--debug-traceback", + dest="debug", action="store_true", help="Re-raise parser errors so Python prints a traceback for parser debugging. " "Can also be enabled with FORTRAN_PARSER_DEBUG=1.", @@ -305,7 +307,7 @@ def main() -> int: report = _parse_paths(args.paths) semantic = _semantic_report(args.paths) if (args.semantics or args.pyi) else None except FortranParseError as exc: - if args.debug_traceback or _env_flag("FORTRAN_PARSER_DEBUG"): + if args.debug or _env_flag("FORTRAN_PARSER_DEBUG"): raise print(exc.format_diagnostic(color=_diagnostic_color_enabled(disabled=args.no_color), debug=False), file=sys.stderr) return 1 diff --git a/fortran_parser/parser.py b/fortran_parser/parser.py index e9195a99e..90273c938 100644 --- a/fortran_parser/parser.py +++ b/fortran_parser/parser.py @@ -3,6 +3,7 @@ import re import ast +from copy import deepcopy from pathlib import Path from dataclasses import dataclass, replace @@ -60,7 +61,8 @@ its direct children. The contained procedure is dispatched to `visit_procedure_unit`, which creates a procedure scope and visits only its specification part; the execution part and internal subprograms are ignored for -wrapper metadata. +wrapper metadata. Internal subprogram boundaries are still sliced so malformed +unit structure is rejected before their contents are skipped. Scoping follows the same recursion. A helper that parses `integer :: n` or `real :: x(n)` receives a `_ParserScope` argument. The shared declaration parser @@ -562,6 +564,9 @@ def visit_source_unit( return self.visit_interface_unit(unit, parent_scope=parent_scope, filename=filename) if unit.kind == "procedure": return self.visit_procedure_unit(unit, parent_scope=parent_scope, filename=filename) + if unit.kind == "enum": + self._helper_validate_enum_unit(unit, filename=filename) + return None return None def visit_module_unit( @@ -581,6 +586,8 @@ def visit_module_unit( self._helper_visit_spec_part(scope, parts.specification, filename=filename) child_units = self._helper_slice_child_units(unit.lines[1:-1], parent_scope=scope, filename=filename) + self._helper_validate_child_unit_regions(unit, parts, child_units, filename=filename) + self._helper_validate_contains_lines(scope, parts.contains, filename=filename) self._helper_validate_sibling_units(child_units, parent_scope=scope, filename=filename) signatures = [ self.visit_procedure_unit(child, parent_scope=scope, filename=filename) @@ -597,6 +604,11 @@ def visit_module_unit( for child in child_units if child.kind == "interface" ] + self._helper_validate_ignored_child_units( + [child for child in child_units if child.kind == "enum"], + parent_scope=scope, + filename=filename, + ) module.procedures.extend(sig for sig in signatures if sig.module and sig.module.lower() == module.name.lower() and not sig.in_interface) module.derived_types.extend(dtype for dtype in types if dtype.module and dtype.module.lower() == module.name.lower()) module.interfaces.extend(iface for iface in interfaces if iface.module and iface.module.lower() == module.name.lower()) @@ -621,6 +633,8 @@ def visit_submodule_unit( self._helper_visit_spec_part(scope, parts.specification, filename=filename) child_units = self._helper_slice_child_units(unit.lines[1:-1], parent_scope=scope, filename=filename) + self._helper_validate_child_unit_regions(unit, parts, child_units, filename=filename) + self._helper_validate_contains_lines(scope, parts.contains, filename=filename) self._helper_validate_sibling_units(child_units, parent_scope=scope, filename=filename) signatures = [ self.visit_procedure_unit(child, parent_scope=scope, filename=filename) @@ -637,6 +651,11 @@ def visit_submodule_unit( for child in child_units if child.kind == "interface" ] + self._helper_validate_ignored_child_units( + [child for child in child_units if child.kind == "enum"], + parent_scope=scope, + filename=filename, + ) submodule.procedures.extend(sig for sig in signatures if sig.module and sig.module.lower() == submodule.name.lower() and not sig.in_interface) submodule.derived_types.extend(dtype for dtype in types if dtype.module and dtype.module.lower() == submodule.name.lower()) submodule.interfaces.extend(iface for iface in interfaces if iface.module and iface.module.lower() == submodule.name.lower()) @@ -658,6 +677,16 @@ def visit_program_unit( scope = self._helper_scope_for_model("program", program, parent=parent_scope) parts = self._helper_split_unit_parts(unit, self._helper_unit_grammar("program"), filename=filename) self._helper_visit_spec_part(scope, parts.specification, filename=filename) + child_units = self._helper_nonexecution_child_units(unit, parent_scope=scope, filename=filename) + self._helper_validate_child_unit_regions(unit, parts, child_units, filename=filename) + self._helper_validate_contains_lines(scope, parts.contains, filename=filename) + self._helper_validate_ignored_child_units( + child_units, + parent_scope=scope, + filename=filename, + unit=unit, + parts=parts, + ) self._validate_variable_declarations( program.variables, owner_kind="program", @@ -681,6 +710,8 @@ def visit_block_data_source_unit( scope = self._helper_scope_for_model("block_data", block_data, parent=parent_scope) parts = self._helper_split_unit_parts(unit, self._helper_unit_grammar("block_data"), filename=filename) self._helper_visit_spec_part(scope, parts.specification, filename=filename) + child_units = self._helper_slice_child_units(unit.lines[1:-1], parent_scope=scope, filename=filename) + self._helper_validate_child_unit_regions(unit, parts, child_units, filename=filename) self._validate_variable_declarations( block_data.variables, owner_kind="block data", @@ -715,6 +746,8 @@ def visit_derived_type_unit( lineno=lineno, source_line=source_line, ) + child_units = self._helper_slice_child_units(unit.lines[1:-1], parent_scope=scope, filename=filename) + self._helper_validate_child_unit_regions(unit, parts, child_units, filename=filename) self._validate_derived_type_fields(dtype, filename) return dtype @@ -732,13 +765,22 @@ def visit_interface_unit( raise FortranParseError("Expected interface unit.", filename=filename, line_number=header[1], source_line=header[2]) interface = FortranInterface(name=interface_name, module=parent_scope.module_owner) scope = self._helper_scope_for_model("interface", interface, parent=parent_scope) + parts = self._helper_split_unit_parts(unit, self._helper_unit_grammar("interface"), filename=filename) + self._helper_validate_interface_lines(scope, parts.specification, filename=filename) child_units = self._helper_slice_child_units( unit.lines[1:-1], parent_scope=scope, - allowed_kinds={"procedure"}, filename=filename, ) for child in child_units: + if child.kind != "procedure": + self._raise_invalid_fortran_syntax_line( + child.lines[0][0] if child.lines else child.kind, + context=f"interface '{scope.name or ''}'", + filename=filename, + lineno=child.start_line, + source_line=child.lines[0][2] if child.lines else None, + ) sig = self.visit_procedure_unit(child, parent_scope=scope, filename=filename, in_interface=True) self._add_interface_attribute(sig, interface.name) interface.procedures.append(sig) @@ -778,7 +820,17 @@ def visit_procedure_unit( scope = self._helper_scope_for_model("procedure", proc_state["signature"], parent=parent_scope, state=proc_state) parts = self._helper_split_unit_parts(unit, self._helper_unit_grammar("procedure"), filename=filename) self._helper_visit_spec_part(scope, parts.specification, filename=filename) - self._helper_apply_local_interface_declarations(proc_state, unit, scope, filename=filename) + child_units = self._helper_nonexecution_child_units(unit, parent_scope=scope, filename=filename) + self._helper_validate_child_unit_regions(unit, parts, child_units, filename=filename) + self._helper_validate_contains_lines(scope, parts.contains, filename=filename) + self._helper_validate_ignored_child_units( + [child for child in child_units if child.kind != "interface"], + parent_scope=scope, + filename=filename, + unit=unit, + parts=parts, + ) + self._helper_apply_local_interface_declarations(proc_state, unit, parts, scope, filename=filename) return self._finalize_proc(proc_state) # ------------------------------------------------------------------ @@ -897,7 +949,6 @@ def _helper_prepare_source_units( """ lines = self._preprocessed_lines(code, filename) lines = self._helper_select_active_preprocessor_lines(lines, macro_defines) - self._helper_validate_unit_headers(lines, filename) root_scope = _ParserScope(kind="file", name=None) units = self._helper_slice_child_units(lines, parent_scope=root_scope, filename=filename) self._helper_validate_file_scope_unparsed_lines(lines, filename) @@ -918,8 +969,8 @@ def _collect_interface_source_units( lines, root_scope, _all_units = self._helper_prepare_source_units(code, filename) interfaces: list[tuple[_SourceUnit, _ParserScope]] = [] - def collect(scope: _ParserScope, source_lines: _PreprocessedLines) -> None: - for child in self._helper_slice_child_units(source_lines, parent_scope=scope, filename=filename): + def collect(scope: _ParserScope, child_units: list[_SourceUnit]) -> None: + for child in child_units: if child.kind == "interface": interfaces.append((child, scope)) continue @@ -930,18 +981,24 @@ def collect(scope: _ParserScope, source_lines: _PreprocessedLines) -> None: parent=scope, module_owner=child.name, ) - collect(child_scope, child.lines[1:-1]) + collect( + child_scope, + self._helper_nonexecution_child_units(child, parent_scope=child_scope, filename=filename), + ) continue - if child.kind in {"procedure", "program", "block_data"}: + if child.kind in {"procedure", "program"}: child_scope = _ParserScope( kind=child.kind, name=child.name, parent=scope, module_owner=scope.module_owner, ) - collect(child_scope, child.lines[1:-1]) + collect( + child_scope, + self._helper_nonexecution_child_units(child, parent_scope=child_scope, filename=filename), + ) - collect(root_scope, lines) + collect(root_scope, self._helper_slice_child_units(lines, parent_scope=root_scope, filename=filename)) return interfaces def _collect_derived_type_source_units( @@ -953,8 +1010,8 @@ def _collect_derived_type_source_units( lines, root_scope, _all_units = self._helper_prepare_source_units(code, filename) types: list[tuple[_SourceUnit, _ParserScope]] = [] - def collect(scope: _ParserScope, source_lines: _PreprocessedLines) -> None: - for child in self._helper_slice_child_units(source_lines, parent_scope=scope, filename=filename): + def collect(scope: _ParserScope, child_units: list[_SourceUnit]) -> None: + for child in child_units: if child.kind == "derived_type": types.append((child, scope)) continue @@ -965,18 +1022,24 @@ def collect(scope: _ParserScope, source_lines: _PreprocessedLines) -> None: parent=scope, module_owner=child.name if child.kind in {"module", "submodule"} else scope.module_owner, ) - collect(child_scope, child.lines[1:-1]) + collect( + child_scope, + self._helper_nonexecution_child_units(child, parent_scope=child_scope, filename=filename), + ) continue - if child.kind in {"procedure", "block_data"}: + if child.kind == "procedure": child_scope = _ParserScope( kind=child.kind, name=child.name, parent=scope, module_owner=scope.module_owner, ) - collect(child_scope, child.lines[1:-1]) + collect( + child_scope, + self._helper_nonexecution_child_units(child, parent_scope=child_scope, filename=filename), + ) - collect(root_scope, lines) + collect(root_scope, self._helper_slice_child_units(lines, parent_scope=root_scope, filename=filename)) return types def _helper_select_active_preprocessor_lines( @@ -1094,48 +1157,41 @@ def _handle_procedure_preprocessor_line( def _procedure_preprocessor_condition_set(pp_condition_stack: list[tuple[int, int]]) -> frozenset[str]: return frozenset(f"g{group_id}:b{branch_id}" for group_id, branch_id in pp_condition_stack) - def _helper_validate_unit_headers(self, lines: _PreprocessedLines, filename: str | None) -> None: - """Validate recognizable unit headers before slicing hides bad ones. - - The slicer intentionally ignores lines that are not valid starts. This - helper preserves diagnostics for malformed headers whose first keyword - still shows the user's intent. - - Example: - ``module :: bad_mod`` is not a valid module unit and therefore is - not returned by `_helper_slice_child_units`; this helper raises the - same explicit "malformed module header" error before parsing - continues. - """ - for line, lineno, source_line in lines: - stripped = line.strip() - if not stripped: - continue - self._parse_module_header(stripped, filename, lineno=lineno, source_line=source_line) - if stripped.lower().startswith("end "): - continue - if re.match(r"^module\s+procedure\s*::", stripped, flags=re.IGNORECASE): - continue - if not ( - stripped.lower().startswith("module procedure") - or self._looks_like_procedure_header(stripped) - ): - continue - if self._parse_procedure_header( + def _helper_validate_possible_unit_header( + self, + line: str, + *, + filename: str | None, + lineno: int | None, + source_line: str | None, + ) -> None: + """Validate a line that lexically resembles a source-unit header.""" + stripped = line.strip() + self._parse_module_header(stripped, filename, lineno=lineno, source_line=source_line) + if stripped.lower().startswith("end "): + return + if re.match(r"^module\s+procedure\s*::", stripped, flags=re.IGNORECASE): + return + if not ( + stripped.lower().startswith("module procedure") + or self._looks_like_procedure_header(stripped) + ): + return + if self._parse_procedure_header( + stripped, + None, + False, + filename=filename, + lineno=lineno, + source_line=source_line, + ) is None: + self._raise_if_unparsed_procedure_header( stripped, - None, - False, + in_interface=False, filename=filename, lineno=lineno, source_line=source_line, - ) is None: - self._raise_if_unparsed_procedure_header( - stripped, - in_interface=False, - filename=filename, - lineno=lineno, - source_line=source_line, - ) + ) def _helper_validate_file_scope_unparsed_lines(self, lines: _PreprocessedLines, filename: str | None) -> None: """Reject any non-Fortran syntax outside recognized unit bodies. @@ -1166,12 +1222,23 @@ def _helper_validate_file_scope_unparsed_lines(self, lines: _PreprocessedLines, index += 1 continue + self._helper_validate_possible_unit_header( + stripped, + filename=filename, + lineno=lineno, + source_line=source_line, + ) start = self._helper_classify_unit_start(stripped) if start is not None: end_index = self._helper_find_unit_end(lines, index, start[0], filename=filename) if end_index is not None: index = end_index + 1 continue + if self._is_executable_statement_start(stripped): + # A standalone include fragment can contain executable lines + # without an enclosing procedure. Once execution starts, the + # remaining fragment is intentionally opaque to this parser. + return if self._is_allowed_unparsed_file_scope_line(stripped): index += 1 continue @@ -1186,10 +1253,10 @@ def _helper_validate_file_scope_unparsed_lines(self, lines: _PreprocessedLines, @staticmethod def _is_allowed_unparsed_file_scope_line(line: str) -> bool: stripped = line.strip() - lowered = stripped.lower() return ( stripped.startswith("#") or FortranParser._is_openmp_directive(stripped) + or _REGEX["include"].match(stripped) ) @staticmethod @@ -1216,6 +1283,7 @@ def _helper_slice_child_units( parent_scope: _ParserScope, allowed_kinds: set[str] | None = None, filename: str | None = None, + skip_execution_region: bool = False, ) -> list[_SourceUnit]: """Slice direct child units from a parent source substring. @@ -1237,6 +1305,7 @@ def _helper_slice_child_units( pp_condition_stack: list[tuple[int, int]] = [] pp_active_stack: list[bool] = [] pp_group_counter = 0 + region = "specification" while index < len(lines): line, lineno, _ = lines[index] stripped = line.strip() @@ -1251,6 +1320,16 @@ def _helper_slice_child_units( if handled_pp: index += 1 continue + if skip_execution_region: + if self._is_contains_transition(stripped): + region = "contains" + index += 1 + continue + if region == "specification" and self._is_executable_statement_start(stripped): + region = "execution" + if region == "execution": + index += 1 + continue if parent_scope.kind == "interface" and re.match(r"^module\s+procedure\b", line.strip(), re.IGNORECASE): index += 1 continue @@ -1265,12 +1344,6 @@ def _helper_slice_child_units( end_index = self._helper_find_unit_end(lines, index, kind, filename=filename) if end_index is None: - if kind == "derived_type": - # A `type :: name` statement without a matching `end type` - # is treated as a declaration-like line for compatibility - # with existing tolerant parser behavior. - index += 1 - continue if kind == "interface" and (lines[index][2] or "").strip().lower().startswith("end interface"): index += 1 continue @@ -1326,8 +1399,8 @@ def _helper_find_unit_end( """ start = self._helper_classify_unit_start(lines[start_index][0]) start_name = start[1] if start is not None else None - stack: list[tuple[str, str | None, int | None, str | None]] = [ - (kind, start_name, lines[start_index][1], lines[start_index][2]) + stack: list[tuple[str, str | None, int | None, str | None, str]] = [ + (kind, start_name, lines[start_index][1], lines[start_index][2], "specification") ] idx = start_index + 1 while idx < len(lines): @@ -1336,34 +1409,57 @@ def _helper_find_unit_end( if not line: idx += 1 continue - start = self._helper_classify_unit_start(line) - current_kind, current_name, current_line, current_source = stack[-1] + current_kind, current_name, current_line, current_source, current_region = stack[-1] if current_kind == "interface" and re.match(r"^module\s+procedure\b", line, re.IGNORECASE): idx += 1 continue - if start is not None and self._helper_has_unit_end_ahead(lines, idx, start[0]): - nested_kind, _ = start - stack.append((nested_kind, start[1], lineno, source_line)) - idx += 1 - continue closes_current, end_name = self._helper_parse_unit_end(current_kind, line) if closes_current: - if current_kind != "procedure" and end_name and current_name and end_name.lower() != current_name.lower(): + if end_name and current_name and end_name.lower() != current_name.lower(): + if current_kind == "procedure" and self._helper_has_preferred_unit_end_ahead( + lines, + idx, + current_kind, + current_name, + ): + idx += 1 + continue label = self._helper_unit_label(current_kind) - raise FortranParseError( - f"Mismatched end {label} name '{end_name}' for {label} '{current_name}'.", - filename=filename, - line_number=lineno, - source_line=source_line, - ) + if current_kind != "procedure": + raise FortranParseError( + f"Mismatched end {label} name '{end_name}' for {label} '{current_name}'.", + filename=filename, + line_number=lineno, + source_line=source_line, + ) stack.pop() if not stack: return idx idx += 1 continue - for open_kind, open_name, open_line, open_source in reversed(stack): + grammar = self._helper_unit_grammar(current_kind) + if self._is_contains_transition(line) and grammar.has_contains_part: + stack[-1] = (current_kind, current_name, current_line, current_source, "contains") + idx += 1 + continue + if current_region == "specification" and grammar.has_execution_part and self._is_executable_statement_start(line): + stack[-1] = (current_kind, current_name, current_line, current_source, "execution") + idx += 1 + continue + if current_region == "execution": + idx += 1 + continue + + start = self._helper_classify_unit_start(line) + if start is not None and self._helper_has_unit_end_ahead(lines, idx, start[0]): + nested_kind, _ = start + stack.append((nested_kind, start[1], lineno, source_line, "specification")) + idx += 1 + continue + + for open_kind, open_name, open_line, open_source, _open_region in reversed(stack): closes_open, end_name = self._helper_parse_unit_end(open_kind, line) if not closes_open: continue @@ -1393,16 +1489,30 @@ def _helper_has_unit_end_ahead(self, lines: _PreprocessedLines, start_index: int """ start = self._helper_classify_unit_start(lines[start_index][0]) start_name = start[1] if start is not None else None + if self._helper_has_preferred_unit_end_ahead(lines, start_index, kind, start_name): + return True + if kind != "procedure": + return False for idx in range(start_index + 1, len(lines)): - matched, end_name = self._helper_parse_unit_end(kind, lines[idx][0]) - if not matched: - continue - if kind != "procedure" and start_name and end_name and end_name.lower() != start_name.lower(): - continue + matched, _end_name = self._helper_parse_unit_end(kind, lines[idx][0]) if matched: return True return False + def _helper_has_preferred_unit_end_ahead( + self, + lines: _PreprocessedLines, + start_index: int, + kind: str, + start_name: str | None, + ) -> bool: + """Return whether an exact or unnamed terminator exists later.""" + for idx in range(start_index + 1, len(lines)): + matched, end_name = self._helper_parse_unit_end(kind, lines[idx][0]) + if matched and (not start_name or not end_name or end_name.lower() == start_name.lower()): + return True + return False + def _helper_split_unit_parts( self, unit: _SourceUnit, @@ -1439,10 +1549,23 @@ def _helper_split_unit_parts( index += 1 continue if self._is_contains_transition(stripped): + if not grammar.has_contains_part: + self._raise_invalid_fortran_syntax_line( + stripped, + context=f"{self._helper_unit_label(grammar.kind)} '{unit.name or ''}'", + filename=filename, + lineno=body[index][1], + source_line=body[index][2], + ) region = "contains" index += 1 continue + if grammar.kind == "interface" and re.match(r"^module\s+procedure\b", stripped, re.IGNORECASE): + specification.append(body[index]) + index += 1 + continue + start = self._helper_classify_unit_start(stripped) if start is not None: child_kind, _ = start @@ -1450,6 +1573,8 @@ def _helper_split_unit_parts( if child_end is not None: index = child_end + 1 continue + if grammar.kind == "interface" and child_kind == "procedure": + break if ( region == "specification" @@ -1474,6 +1599,247 @@ def _helper_split_unit_parts( footer=footer, ) + def _helper_child_unit_region( + self, + unit: _SourceUnit, + parts: _UnitParts, + child: _SourceUnit, + ) -> str: + """Return the grammar region containing one direct child unit.""" + child_line = child.start_line + if child_line is None: + return "specification" + contains_line = self._helper_direct_contains_line(unit, filename=None) + if contains_line is not None and child_line > contains_line: + return "contains" + execution_line = next( + (lineno for _line, lineno, _source_line in parts.execution if lineno is not None), + None, + ) + if execution_line is not None and child_line >= execution_line: + return "execution" + return "specification" + + def _helper_nonexecution_child_units( + self, + unit: _SourceUnit, + *, + parent_scope: _ParserScope, + filename: str | None, + ) -> list[_SourceUnit]: + """Return direct nested units outside an intentionally skipped execution part.""" + grammar = self._helper_unit_grammar(unit.kind) + child_units = self._helper_slice_child_units( + unit.lines[1:-1], + parent_scope=parent_scope, + filename=filename, + skip_execution_region=grammar.has_execution_part, + ) + if not grammar.has_execution_part: + return child_units + parts = self._helper_split_unit_parts(unit, grammar, filename=filename) + return [ + child + for child in child_units + if self._helper_child_unit_region(unit, parts, child) != "execution" + ] + + def _helper_direct_contains_line( + self, + unit: _SourceUnit, + *, + filename: str | None, + ) -> int | None: + """Return the direct `contains` transition, skipping nested child units.""" + body = unit.lines[1:-1] + index = 0 + while index < len(body): + line, lineno, _source_line = body[index] + stripped = line.strip() + if self._is_contains_transition(stripped): + return lineno + start = self._helper_classify_unit_start(stripped) + if start is not None: + child_end = self._helper_find_unit_end(body, index, start[0], filename=filename) + if child_end is not None: + index = child_end + 1 + continue + index += 1 + return None + + def _helper_validate_child_unit_regions( + self, + unit: _SourceUnit, + parts: _UnitParts, + child_units: list[_SourceUnit], + *, + filename: str | None, + ) -> None: + """Reject child units that occur outside their parent's grammar region.""" + allowed = { + "module": { + "specification": {"derived_type", "interface", "enum"}, + "contains": {"procedure"}, + }, + "submodule": { + "specification": {"derived_type", "interface", "enum"}, + "contains": {"procedure"}, + }, + "program": { + "specification": {"derived_type", "interface", "enum"}, + "contains": {"procedure"}, + }, + "procedure": { + "specification": {"derived_type", "interface", "enum"}, + "contains": {"procedure"}, + }, + "derived_type": { + "specification": set(), + "contains": set(), + }, + "block_data": { + "specification": set(), + "contains": set(), + }, + "enum": { + "specification": set(), + "contains": set(), + }, + } + grammar_regions = allowed.get(unit.kind, {}) + for child in child_units: + region = self._helper_child_unit_region(unit, parts, child) + if region == "execution": + continue + if child.kind in grammar_regions.get(region, set()): + continue + self._raise_invalid_fortran_syntax_line( + child.lines[0][0] if child.lines else child.kind, + context=( + f"{self._helper_unit_label(unit.kind)} '{unit.name or ''}' " + f"{region} part" + ), + filename=filename, + lineno=child.start_line, + source_line=child.lines[0][2] if child.lines else None, + ) + + def _helper_validate_contains_lines( + self, + scope: _ParserScope, + lines: _PreprocessedLines, + *, + filename: str | None, + ) -> None: + """Validate non-child lines left in a `contains` region.""" + for line, lineno, source_line in lines: + stripped = line.strip() + if not stripped or stripped.startswith("#") or _REGEX["include"].match(stripped): + continue + if self._helper_is_valid_contains_alternative_line(scope, stripped): + continue + self._helper_validate_possible_unit_header( + stripped, + filename=filename, + lineno=lineno, + source_line=source_line, + ) + self._raise_invalid_fortran_syntax_line( + stripped, + context=f"{self._helper_unit_label(scope.kind)} '{scope.name or ''}' contains part", + filename=filename, + lineno=lineno, + source_line=source_line, + ) + + def _helper_is_valid_contains_alternative_line(self, scope: _ParserScope, line: str) -> bool: + """Accept syntax from an unselected raw-preprocessor specification alternative.""" + scratch_scope = deepcopy(scope) + try: + self._helper_visit_spec_part(scratch_scope, [(line, None, None)], filename=None) + except FortranParseError: + return False + return True + + def _helper_validate_interface_lines( + self, + scope: _ParserScope, + lines: _PreprocessedLines, + *, + filename: str | None, + ) -> None: + """Validate interface statements that are not nested procedure bodies.""" + for line, lineno, source_line in lines: + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + if re.match(r"^module\s+procedure\s*(?:::)?\s*[A-Za-z_]\w*(?:\s*,\s*[A-Za-z_]\w*)*\s*$", stripped, re.IGNORECASE): + continue + if re.match(r"^procedure(?:\s*\([^)]*\))?(?:\s*,\s*[^:]*)?\s*::\s*[A-Za-z_]\w*(?:\s*,\s*[A-Za-z_]\w*)*\s*$", stripped, re.IGNORECASE): + continue + self._helper_validate_possible_unit_header( + stripped, + filename=filename, + lineno=lineno, + source_line=source_line, + ) + self._raise_invalid_fortran_syntax_line( + stripped, + context=f"interface '{scope.name or ''}'", + filename=filename, + lineno=lineno, + source_line=source_line, + ) + + def _helper_validate_enum_unit(self, unit: _SourceUnit, *, filename: str | None) -> None: + """Validate an interoperability enum block without exporting metadata.""" + parts = self._helper_split_unit_parts(unit, self._helper_unit_grammar("enum"), filename=filename) + for line, lineno, source_line in parts.specification: + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + match = re.match(r"^enumerator\s*(?:::)?\s*(?P.+)$", stripped, re.IGNORECASE) + if match and all( + re.match(r"^[A-Za-z_]\w*(?:\s*=\s*.+)?$", item.strip()) + for item in split_csv(match.group("items")) + ): + continue + self._raise_invalid_fortran_syntax_line( + stripped, + context="enum specification part", + filename=filename, + lineno=lineno, + source_line=source_line, + ) + child_units = self._helper_slice_child_units(unit.lines[1:-1], parent_scope=_ParserScope(kind="enum", name=unit.name), filename=filename) + self._helper_validate_child_unit_regions(unit, parts, child_units, filename=filename) + + def _helper_validate_ignored_child_units( + self, + child_units: list[_SourceUnit], + *, + parent_scope: _ParserScope, + filename: str | None, + unit: _SourceUnit | None = None, + parts: _UnitParts | None = None, + ) -> None: + """Check or skip nested units that are intentionally omitted from metadata.""" + for child in child_units: + if unit is not None and parts is not None: + if self._helper_child_unit_region(unit, parts, child) == "execution": + continue + if child.kind == "procedure": + # The slicer has already checked the nested unit boundary and + # the caller has checked its grammar region. Internal procedure + # declarations and bodies do not affect wrapper metadata. + continue + elif child.kind == "interface": + self.visit_interface_unit(child, parent_scope=parent_scope, filename=filename) + elif child.kind == "derived_type": + self.visit_derived_type_unit(child, parent_scope=parent_scope, filename=filename) + elif child.kind == "enum": + self._helper_validate_enum_unit(child, filename=filename) + def _helper_validate_sibling_units( self, units: list[_SourceUnit], @@ -1573,7 +1939,7 @@ def _helper_unit_grammar(self, kind: str) -> _UnitGrammar: has_contains_part=True, declaration_role="type_field", ), - "interface": _UnitGrammar(kind="interface", has_contains_part=True), + "interface": _UnitGrammar(kind="interface"), "block_data": _UnitGrammar(kind="block_data", declaration_role="module_variable"), "file": _UnitGrammar(kind="file", has_contains_part=True), } @@ -2193,7 +2559,13 @@ def _helper_visit_module_like_spec_line( return if _REGEX["derived_type"].match(stripped): - return + parsed_type = self._parse_derived_type_start(stripped) + raise FortranParseError( + f"Missing end derived type for derived type '{parsed_type[0] if parsed_type else ''}'.", + filename=filename, + line_number=lineno, + source_line=source_line, + ) if "::" in stripped: left, right = [x.strip() for x in stripped.split("::", 1)] @@ -2344,7 +2716,12 @@ def _helper_visit_type_spec_line(self, line: str, scope: _ParserScope, filename: raise FortranParseError("Derived-type specification scope is missing a target model.", filename=filename) stripped = line.strip() if re.match(r"^type\s*::\s*\w+$", stripped, re.IGNORECASE): - return + raise FortranParseError( + f"Missing end derived type for derived type '{stripped.split('::', 1)[1].strip()}'.", + filename=filename, + line_number=lineno, + source_line=source_line, + ) if stripped.lower() in {"sequence", "private"}: return if self._is_openmp_declarative_directive(stripped): @@ -2366,8 +2743,6 @@ def _helper_visit_type_spec_line(self, line: str, scope: _ParserScope, filename: if parsed: return if "::" not in stripped and not self._looks_like_declaration_or_spec(stripped): - if self._is_executable_statement_start(stripped): - return self._raise_invalid_fortran_syntax_line( stripped, context=f"type '{dtype.name}' specification part", @@ -2410,18 +2785,21 @@ def _parse_derived_type_contains_line( dtype.generic_bindings.append({"name": lhs, "targets": rhs, "attrs": attrs}) return - if self._looks_like_declaration_or_spec(line): - raise FortranParseError( - f"Unsupported or malformed type-bound declaration in type '{dtype.name}': {line.strip()}", - filename=filename, - line_number=lineno, - source_line=source_line, - ) + if re.match(r"^final\s*::\s*[A-Za-z_]\w*(?:\s*,\s*[A-Za-z_]\w*)*\s*$", line, re.IGNORECASE): + return + + raise FortranParseError( + f"Unsupported or malformed type-bound declaration in type '{dtype.name}': {line.strip()}", + filename=filename, + line_number=lineno, + source_line=source_line, + ) def _helper_apply_local_interface_declarations( self, proc_state: dict, unit: _SourceUnit, + parts: _UnitParts, scope: _ParserScope, *, filename: str | None, @@ -2444,34 +2822,14 @@ def _helper_apply_local_interface_declarations( parent_scope=scope, allowed_kinds={"interface"}, filename=filename, + skip_execution_region=True, ) for interface_unit in interface_units: - interface_scope = _ParserScope( - kind="interface", - name=interface_unit.name, - parent=scope, - module_owner=scope.module_owner, - ) - for child in self._helper_slice_child_units( - interface_unit.lines[1:-1], - parent_scope=interface_scope, - allowed_kinds={"procedure"}, - filename=filename, - ): - header = child.lines[0] if child.lines else None - if header is None: - continue - parsed = self._parse_procedure_header( - header[0].strip(), - scope.module_owner, - True, - filename=filename, - lineno=header[1], - source_line=header[2], - ) - if parsed is None: - continue - name = parsed["signature"].name + if self._helper_child_unit_region(unit, parts, interface_unit) == "execution": + continue + interface = self.visit_interface_unit(interface_unit, parent_scope=scope, filename=filename) + for signature in interface.procedures: + name = signature.name if self._proc_scope_symbol_is_declared(proc_state, name): key = self._scope_key(name) else: @@ -2479,8 +2837,8 @@ def _helper_apply_local_interface_declarations( proc_state, name, filename=filename, - line_number=header[1], - source_line=header[2], + line_number=interface_unit.start_line, + source_line=interface_unit.lines[0][2] if interface_unit.lines else None, ) arg = self._proc_scope_get_symbol(proc_state, key) if arg is not None and arg.base_type == "unknown": diff --git a/tests/parser/c/test_c_cli_skeleton.py b/tests/parser/c/test_c_cli_skeleton.py index 2a9b25e6b..13890273b 100644 --- a/tests/parser/c/test_c_cli_skeleton.py +++ b/tests/parser/c/test_c_cli_skeleton.py @@ -300,7 +300,7 @@ def test_cli_c_invalid_primitive_specifier_sequence_is_fatal(tmp_path: Path): assert "\x1b[" not in res.stderr -def test_cli_c_debug_traceback_reraises_parse_errors(tmp_path: Path): +def test_cli_c_debug_reraises_parse_errors(tmp_path: Path): header = tmp_path / "invalid_specifiers.h" header.write_text("unsigned float value;\n", encoding="utf-8") cmd = [ @@ -311,7 +311,7 @@ def test_cli_c_debug_traceback_reraises_parse_errors(tmp_path: Path): "--language", "c", "--parse", - "--debug-traceback", + "--debug", ] res = subprocess.run(cmd, capture_output=True, text=True) @@ -394,6 +394,36 @@ def test_c_parser_module_entrypoint_and_compatibility_exports(tmp_path: Path): assert c_utils.__all__ == () +def test_c_parser_module_formats_parse_errors_without_traceback(tmp_path: Path): + header = tmp_path / "invalid.h" + header.write_text("@@@;\n", encoding="utf-8") + + result = subprocess.run( + [sys.executable, "-m", "c_parser", str(header), "--no-color"], + capture_output=True, + text=True, + ) + + assert result.returncode == 1 + assert "error[CPARSE_INVALID_SYNTAX]" in result.stderr + assert "Traceback" not in result.stderr + + +def test_c_parser_module_debug_reraises_parse_errors(tmp_path: Path): + header = tmp_path / "invalid.h" + header.write_text("@@@;\n", encoding="utf-8") + + result = subprocess.run( + [sys.executable, "-m", "c_parser", str(header), "--debug"], + capture_output=True, + text=True, + ) + + assert result.returncode == 1 + assert "Traceback" in result.stderr + assert "CParseError" in result.stderr + + def test_x2py_c_compiler_source_loader_drives_semantics_and_readiness(tmp_path: Path, monkeypatch): header = tmp_path / "api.h" header.write_text("API(int) add(int a, int b);\n", encoding="utf-8") diff --git a/tests/parser/c/test_c_declarations_and_declarators.py b/tests/parser/c/test_c_declarations_and_declarators.py index e40c54c27..f9b291233 100644 --- a/tests/parser/c/test_c_declarations_and_declarators.py +++ b/tests/parser/c/test_c_declarations_and_declarators.py @@ -571,30 +571,35 @@ def test_unimplemented_declaration_extensions_are_diagnosed_not_partially_modele @pytest.mark.parametrize( "source", [ - "using size_type = int;\n", - "using namespace api;\n", "namespace api { int run(void); }\n", - "namespace api = other;\n", - "template T identity(T value);\n", - "class widget;\n", "public:\n", ], ) -def test_cxx_declaration_shapes_are_diagnosed_not_partially_modeled(source): - from c_parser import parse_c_file +def test_non_c_top_level_grammar_is_rejected_without_language_guessing(source): + from c_parser import CParseError, parse_c_file - parsed = parse_c_file(source, filename="cxx_shapes.h") + with pytest.raises(CParseError, match="Invalid C syntax") as exc_info: + parse_c_file(source, filename="invalid_top_level.h") - assert parsed.functions == [] - assert parsed.structs == [] - assert parsed.unions == [] - assert parsed.enums == [] - assert parsed.typedefs == [] - assert parsed.variables == [] - assert [ - (diagnostic.code, diagnostic.unit_kind, diagnostic.location.line) - for diagnostic in parsed.diagnostics - ] == [("C_UNSUPPORTED_DECLARATION", "cxx_declaration", 1)] + assert exc_info.value.code == "CPARSE_INVALID_SYNTAX" + + +@pytest.mark.parametrize( + ("source", "name", "type_name"), + [ + ("class widget;\n", "widget", "class"), + ("namespace api = other;\n", "api", "namespace"), + ("using size_type = value;\n", "size_type", "using"), + ], +) +def test_identifier_spelling_does_not_trigger_foreign_language_detection(source, name, type_name): + from c_parser import CTypedef, parse_c_file + + parsed = parse_c_file(source, filename="identifier_spelling.h") + + assert [variable.name for variable in parsed.variables] == [name] + assert isinstance(parsed.variables[0].type, CTypedef) + assert parsed.variables[0].type.name == type_name def test_braced_and_designated_initializer_declarations_preserve_source_text(): diff --git a/tests/parser/c/test_c_functions.py b/tests/parser/c/test_c_functions.py index 98f5d8932..df37d328a 100644 --- a/tests/parser/c/test_c_functions.py +++ b/tests/parser/c/test_c_functions.py @@ -165,6 +165,24 @@ def test_c_parser_ignores_invalid_syntax_inside_function_body(): assert [function.name for function in parsed.functions] == ["run"] +@pytest.mark.parametrize( + "source", + [ + "struct bad { @@@; };\n", + "enum bad { OK, @@@ };\n", + "int run(@@@);\n", + "int run(int first, ..., int last);\n", + ], +) +def test_c_parser_rejects_invalid_nested_grammar_units(source): + from c_parser import CParseError, parse_c_file + + with pytest.raises(CParseError, match="Invalid C syntax") as exc_info: + parse_c_file(source, filename="invalid_nested.h") + + assert exc_info.value.code == "CPARSE_INVALID_SYNTAX" + + def test_control_flow_conditions_inside_function_body_do_not_look_like_knr_definitions(): from c_parser import parse_c_file diff --git a/tests/parser/test_cli.py b/tests/parser/test_cli.py index 1a3193098..f292eb796 100644 --- a/tests/parser/test_cli.py +++ b/tests/parser/test_cli.py @@ -193,7 +193,7 @@ def test_cli_formats_parse_errors_without_traceback(tmp_path: Path): assert "2 | weirdtype :: x" in res.stderr -def test_cli_debug_traceback_flag_reraises_parse_errors(tmp_path: Path): +def test_cli_debug_flag_reraises_parse_errors(tmp_path: Path): f90 = tmp_path / "bad.f90" f90.write_text( """subroutine bad(x) @@ -203,7 +203,7 @@ def test_cli_debug_traceback_flag_reraises_parse_errors(tmp_path: Path): encoding="utf-8", ) - cmd = [sys.executable, "-m", "x2py", str(f90), "--parse", "--debug-traceback"] + cmd = [sys.executable, "-m", "x2py", str(f90), "--parse", "--debug"] res = subprocess.run(cmd, capture_output=True, text=True) assert res.returncode == 1 @@ -853,7 +853,7 @@ def test_x2py_cli_rejects_invalid_stage_combinations(extra_args, message): assert message in res.stderr -def test_fortran_parser_cli_debug_traceback_flag_reraises_parse_errors(tmp_path: Path): +def test_fortran_parser_cli_debug_flag_reraises_parse_errors(tmp_path: Path): f90 = tmp_path / "bad.f90" f90.write_text( """subroutine bad(x) @@ -863,7 +863,7 @@ def test_fortran_parser_cli_debug_traceback_flag_reraises_parse_errors(tmp_path: encoding="utf-8", ) - cmd = [sys.executable, "-m", "fortran_parser", str(f90), "--debug-traceback"] + cmd = [sys.executable, "-m", "fortran_parser", str(f90), "--debug"] res = subprocess.run(cmd, capture_output=True, text=True) assert res.returncode == 1 @@ -993,6 +993,6 @@ def fail_parse(_paths, _preprocessing): assert x2py_cli.main() == 1 assert "x2py: error: invalid generated interface" in capsys.readouterr().err - monkeypatch.setattr(sys, "argv", ["x2py", str(source), "--parse", "--debug-traceback"]) + monkeypatch.setattr(sys, "argv", ["x2py", str(source), "--parse", "--debug"]) with pytest.raises(ValueError, match="invalid generated interface"): x2py_cli.main() diff --git a/tests/parser/test_declaration_and_interface_edges.py b/tests/parser/test_declaration_and_interface_edges.py index 5c02bdea6..8dd533547 100644 --- a/tests/parser/test_declaration_and_interface_edges.py +++ b/tests/parser/test_declaration_and_interface_edges.py @@ -324,39 +324,31 @@ def test_local_compile_time_arithmetic_is_folded_for_shapes_and_parameters(): ] assert sig.variables["one"].value == "1" -def test_type_contains_ignores_executable_like_lines_and_rejects_bad_declarations(): - ok_code = """ -module type_contains_ok_mod +def test_type_contains_accepts_bindings_and_rejects_other_lines(): + valid_code = """ +module type_contains_valid_mod type :: state contains - call ignored_statement() + procedure :: update + final :: destroy end type state -end module type_contains_ok_mod +end module type_contains_valid_mod """ - bad_code = """ + + parsed = parse_fortran_file(valid_code, filename="type_contains_valid.f90") + assert parsed.modules[0].derived_types[0].methods == ["update"] + + for invalid_line in ("call ignored_statement()", "!$omp declare target", "integer, public :: bad_binding"): + code = f""" module type_contains_bad_mod type :: state contains -!$omp declare target + {invalid_line} end type state end module type_contains_bad_mod """ - comma_bad_code = """ -module type_contains_comma_bad_mod - type :: state - contains - integer, public :: bad_binding - end type state -end module type_contains_comma_bad_mod -""" - - parsed = parse_fortran_file(ok_code, filename="type_contains_ok.f90") - assert parsed.modules[0].derived_types[0].methods == [] - - with pytest.raises(FortranParseError, match="Unsupported or malformed type-bound declaration"): - parse_fortran_file(bad_code, filename="type_contains_omp.f90") - with pytest.raises(FortranParseError, match="Unsupported or malformed type-bound declaration"): - parse_fortran_file(comma_bad_code, filename="type_contains_comma.f90") + with pytest.raises(FortranParseError, match="Unsupported or malformed type-bound declaration"): + parse_fortran_file(code, filename="type_contains_bad.f90") def test_malformed_type_bound_declaration_raises(): code = """ @@ -387,10 +379,8 @@ def test_type_field_spec_variants_and_empty_entities_from_public_source(): code = """ module type_field_edges_mod type :: state - type :: nested_marker sequence private - call ignored_in_type_spec() integer :: first, , second end type state end module type_field_edges_mod @@ -400,6 +390,19 @@ def test_type_field_spec_variants_and_empty_entities_from_public_source(): assert [field.name for field in dtype.fields] == ["first", "second"] +@pytest.mark.parametrize("invalid_line", ["type :: nested_marker", "call invalid_in_type_spec()"]) +def test_type_field_specification_rejects_invalid_nested_syntax(invalid_line): + code = f""" +module type_field_invalid_mod + type :: state + {invalid_line} + end type state +end module type_field_invalid_mod +""" + + with pytest.raises(FortranParseError): + parse_fortran_file(code, filename="type_field_invalid.f90") + def test_module_like_declaration_edges_from_program_and_module_sources(): module_code = """ module module_spec_edges_mod @@ -411,6 +414,8 @@ def test_module_like_declaration_edges_from_program_and_module_sources(): program_code = """ program type_stmt_program type :: local_state + integer :: marker + end type local_state integer :: kept end program type_stmt_program """ diff --git a/tests/parser/test_error_handling.py b/tests/parser/test_error_handling.py index 68f621586..4c1ca5cba 100644 --- a/tests/parser/test_error_handling.py +++ b/tests/parser/test_error_handling.py @@ -739,6 +739,18 @@ def test_slicer_reports_mismatched_end_unit_name(): parse_fortran_file(code, filename="mismatch_module.f90") +def test_slicer_accepts_mismatched_procedure_end_name_without_preferred_alternative(): + parsed = parse_fortran_file( + """ +subroutine expected_name() +end subroutine alternate_name +""", + filename="mismatch_procedure_raw_alternative.f90", + ) + + assert parsed.procedures[0].name == "expected_name" + + def test_slicer_reports_missing_end_unit(): code = """ module missing_end @@ -821,6 +833,115 @@ def test_fortran_parser_ignores_invalid_syntax_after_execution_boundary(): assert parsed.procedures[0].name == "ignored_body" +def test_fortran_parser_skips_standalone_include_fragment_after_execution_boundary(): + parsed = parse_fortran_file( + """ +include 'fragment.inc' +if (enabled) then + @@@ +else + @@@ +endif +""", + filename="fragment.inc", + ) + + assert parsed.procedures == [] + + +def test_fortran_parser_skips_balanced_internal_procedure_contents(): + parsed = parse_fortran_file( + """ +subroutine host() +contains + subroutine nested() + @@@ + end subroutine nested +end subroutine host +""", + filename="ignored_internal_body.f90", + ) + + assert parsed.procedures[0].name == "host" + + +def test_fortran_parser_rejects_unterminated_internal_procedure_unit(): + with pytest.raises(FortranParseError, match="Missing end procedure"): + parse_fortran_file( + """ +subroutine host() +contains + subroutine nested() +end subroutine host +""", + filename="unterminated_internal_unit.f90", + ) + + +def test_fortran_parser_skips_nested_unit_like_lines_after_execution_boundary(): + parsed = parse_fortran_file( + """ +subroutine host() + call begin_work() + interface + subroutine ignored() + @@@ + end subroutine ignored + end interface +end subroutine host +""", + filename="ignored_nested_execution.f90", + ) + + assert parsed.procedures[0].name == "host" + + +def test_fortran_parser_skips_unterminated_unit_like_lines_after_execution_boundary(): + parsed = parse_fortran_file( + """ +subroutine host() + call begin_work() + subroutine ignored() +end subroutine host +""", + filename="ignored_unterminated_nested_execution.f90", + ) + + assert parsed.procedures[0].name == "host" + + +def test_fortran_parser_rejects_malformed_enum_subunit(): + with pytest.raises(FortranParseError, match="Invalid Fortran syntax") as exc_info: + parse_fortran_file( + """ +module invalid_enum_mod + enum, bind(c) + enumerator :: valid = 1 + @@@ + end enum +end module invalid_enum_mod +""", + filename="invalid_enum.f90", + ) + + assert exc_info.value.code == "PARSE_INVALID_SYNTAX" + + +def test_fortran_parser_rejects_subunit_inside_block_data(): + with pytest.raises(FortranParseError, match="Invalid Fortran syntax") as exc_info: + parse_fortran_file( + """ +block data invalid_block + interface + end interface +end block data invalid_block +""", + filename="invalid_block_data.f90", + ) + + assert exc_info.value.code == "PARSE_INVALID_SYNTAX" + + def test_invalid_syntax_guard_preserves_valid_semicolon_separated_fortran_statements(): parsed = parse_fortran_file( """ diff --git a/tests/parser/test_preprocessor_and_execution_boundaries.py b/tests/parser/test_preprocessor_and_execution_boundaries.py index b98b797b5..e05a63d7e 100644 --- a/tests/parser/test_preprocessor_and_execution_boundaries.py +++ b/tests/parser/test_preprocessor_and_execution_boundaries.py @@ -302,8 +302,9 @@ def test_implicit_mapping_parameter_noise_and_assignment_lines_do_not_break_proc assert sig.arguments[0].name == "x" assert sig.arguments[0].base_type == "real" -def test_stray_end_unit_lines_are_ignored_by_public_file_parse(): - parsed = parse_fortran_file( +def test_stray_end_unit_lines_are_rejected_by_public_file_parse(): + with pytest.raises(FortranParseError, match="Invalid Fortran syntax") as exc_info: + parse_fortran_file( """ end module stray_mod end submodule stray_submod @@ -316,4 +317,4 @@ def test_stray_end_unit_lines_are_ignored_by_public_file_parse(): filename="stray_ends.f90", ) - assert [proc.name for proc in parsed.procedures] == ["kept"] + assert exc_info.value.code == "PARSE_INVALID_SYNTAX" diff --git a/x2py/cli.py b/x2py/cli.py index 81945275c..ec2c59abc 100644 --- a/x2py/cli.py +++ b/x2py/cli.py @@ -27,6 +27,10 @@ _TRUE_VALUES = {"1", "true", "yes", "on"} _FORTRAN_SOURCE_SUFFIXES = {".f", ".for", ".ftn", ".f77", ".f90", ".f95", ".f03", ".f08"} _C_SOURCE_SUFFIXES = {".c", ".h", ".i"} +_SOURCE_SUFFIXES_BY_LANGUAGE = { + "fortran": _FORTRAN_SOURCE_SUFFIXES, + "c": _C_SOURCE_SUFFIXES, +} def _env_flag(name: str) -> bool: @@ -94,21 +98,27 @@ def _resolve_language( requested: str | None, parser: argparse.ArgumentParser, ) -> str: + def language_for_suffix(suffix: str) -> str | None: + return next( + ( + language + for language, suffixes in _SOURCE_SUFFIXES_BY_LANGUAGE.items() + if suffix in suffixes + ), + None, + ) + if requested is not None: for raw in paths: path = Path(raw) if path.is_dir(): continue suffix = path.suffix.lower() - if requested == "fortran" and suffix in _C_SOURCE_SUFFIXES: - parser.error( - f"C input {path} is incompatible with --language fortran; " - "pass --language c. Use --help for examples." - ) - if requested == "c" and suffix in _FORTRAN_SOURCE_SUFFIXES: + detected = language_for_suffix(suffix) + if detected is not None and detected != requested: parser.error( - f"Fortran input {path} is incompatible with --language c; " - "pass --language fortran. Use --help for examples." + f"{detected.capitalize()} input {path} is incompatible with --language {requested}; " + f"pass --language {detected}. Use --help for examples." ) return requested @@ -610,7 +620,13 @@ def main() -> int: parser.add_argument("--json", action="store_true", help="Print JSON to stdout") parser.add_argument("--out", nargs="?", const="", type=str, help="Write stage output to file (optional explicit output filename)") parser.add_argument("--no-color", action="store_true", help="Disable ANSI color in parse diagnostics") - parser.add_argument("--debug-traceback", action="store_true", help="Re-raise parser errors for debug") + parser.add_argument( + "--debug", + "--debug-traceback", + dest="debug", + action="store_true", + help="Re-raise parser errors so Python prints a traceback for parser debugging", + ) args = parser.parse_args() args.language = _resolve_language(args.paths, args.language, parser) preprocessing = _build_preprocessing_config(args, parser) @@ -649,17 +665,17 @@ def main() -> int: readiness_payload = _wrap_readiness_report(args.paths, preprocessing, language=args.language) if args.wrap_readiness else None _attach_wrap_readiness(semantic_payload, readiness_payload) except CParseError as exc: - if args.debug_traceback or _env_flag("C_PARSER_DEBUG"): + if args.debug or _env_flag("C_PARSER_DEBUG"): raise print(exc.format_diagnostic(color=_diagnostic_color_enabled(disabled=args.no_color), debug=False), file=sys.stderr) return 1 except FortranParseError as exc: - if args.debug_traceback or _env_flag("FORTRAN_PARSER_DEBUG"): + if args.debug or _env_flag("FORTRAN_PARSER_DEBUG"): raise print(exc.format_diagnostic(color=_diagnostic_color_enabled(disabled=args.no_color), debug=False), file=sys.stderr) return 1 except (SyntaxError, ValueError) as exc: - if args.debug_traceback or _env_flag("X2PY_DEBUG"): + if args.debug or _env_flag("X2PY_DEBUG"): raise print(f"x2py: error: {exc}", file=sys.stderr) return 1 From 1649ad8a551798fa97c8b66af5810f7dd6f450dc Mon Sep 17 00:00:00 2001 From: said Date: Sun, 31 May 2026 00:21:55 +0100 Subject: [PATCH 05/13] fix issues --- tests/parser/c/fixtures/stb/stb_ds.json | 72 ++++++++++++------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/tests/parser/c/fixtures/stb/stb_ds.json b/tests/parser/c/fixtures/stb/stb_ds.json index 2613585bc..65c97b266 100644 --- a/tests/parser/c/fixtures/stb/stb_ds.json +++ b/tests/parser/c/fixtures/stb/stb_ds.json @@ -7944,8 +7944,8 @@ "unit_name": null }, { - "code": "C_UNSUPPORTED_DECLARATION", - "message": "C++ declaration syntax is not supported by the C parser.", + "code": "C_UNSUPPORTED_DECLARATOR", + "message": "Unsupported declarator syntax after parsed type layers: ' static T * stbds_arrgrowf_wrapper(T *a, size_t elemsize, size_t addlen, size_t min_cap)'.", "severity": "warning", "location": { "filename": "stb/stb_ds.h", @@ -7953,12 +7953,12 @@ "column": 1, "source_line": "template static T * stbds_arrgrowf_wrapper(T *a, size_t elemsize, size_t addlen, size_t min_cap) {" }, - "unit_kind": "cxx_declaration", + "unit_kind": "declarator", "unit_name": null }, { - "code": "C_UNSUPPORTED_DECLARATION", - "message": "C++ declaration syntax is not supported by the C parser.", + "code": "C_UNSUPPORTED_DECLARATOR", + "message": "Unsupported declarator syntax after parsed type layers: ' static T * stbds_hmget_key_wrapper(T *a, size_t elemsize, void *key, size_t keysize, int mode)'.", "severity": "warning", "location": { "filename": "stb/stb_ds.h", @@ -7966,12 +7966,12 @@ "column": 1, "source_line": "template static T * stbds_hmget_key_wrapper(T *a, size_t elemsize, void *key, size_t keysize, int mode) {" }, - "unit_kind": "cxx_declaration", + "unit_kind": "declarator", "unit_name": null }, { - "code": "C_UNSUPPORTED_DECLARATION", - "message": "C++ declaration syntax is not supported by the C parser.", + "code": "C_UNSUPPORTED_DECLARATOR", + "message": "Unsupported declarator syntax after parsed type layers: ' static T * stbds_hmget_key_ts_wrapper(T *a, size_t elemsize, void *key, size_t keysize, ptrdiff_t *temp, int mode)'.", "severity": "warning", "location": { "filename": "stb/stb_ds.h", @@ -7979,12 +7979,12 @@ "column": 1, "source_line": "template static T * stbds_hmget_key_ts_wrapper(T *a, size_t elemsize, void *key, size_t keysize, ptrdiff_t *temp, int mode) {" }, - "unit_kind": "cxx_declaration", + "unit_kind": "declarator", "unit_name": null }, { - "code": "C_UNSUPPORTED_DECLARATION", - "message": "C++ declaration syntax is not supported by the C parser.", + "code": "C_UNSUPPORTED_DECLARATOR", + "message": "Unsupported declarator syntax after parsed type layers: ' static T * stbds_hmput_default_wrapper(T *a, size_t elemsize)'.", "severity": "warning", "location": { "filename": "stb/stb_ds.h", @@ -7992,12 +7992,12 @@ "column": 1, "source_line": "template static T * stbds_hmput_default_wrapper(T *a, size_t elemsize) {" }, - "unit_kind": "cxx_declaration", + "unit_kind": "declarator", "unit_name": null }, { - "code": "C_UNSUPPORTED_DECLARATION", - "message": "C++ declaration syntax is not supported by the C parser.", + "code": "C_UNSUPPORTED_DECLARATOR", + "message": "Unsupported declarator syntax after parsed type layers: ' static T * stbds_hmput_key_wrapper(T *a, size_t elemsize, void *key, size_t keysize, int mode)'.", "severity": "warning", "location": { "filename": "stb/stb_ds.h", @@ -8005,12 +8005,12 @@ "column": 1, "source_line": "template static T * stbds_hmput_key_wrapper(T *a, size_t elemsize, void *key, size_t keysize, int mode) {" }, - "unit_kind": "cxx_declaration", + "unit_kind": "declarator", "unit_name": null }, { - "code": "C_UNSUPPORTED_DECLARATION", - "message": "C++ declaration syntax is not supported by the C parser.", + "code": "C_UNSUPPORTED_DECLARATOR", + "message": "Unsupported declarator syntax after parsed type layers: ' static T * stbds_hmdel_key_wrapper(T *a, size_t elemsize, void *key, size_t keysize, size_t keyoffset, int mode)'.", "severity": "warning", "location": { "filename": "stb/stb_ds.h", @@ -8018,7 +8018,7 @@ "column": 1, "source_line": "template static T * stbds_hmdel_key_wrapper(T *a, size_t elemsize, void *key, size_t keysize, size_t keyoffset, int mode){" }, - "unit_kind": "cxx_declaration", + "unit_kind": "declarator", "unit_name": null }, { @@ -13614,8 +13614,8 @@ "unit_name": null }, { - "code": "C_UNSUPPORTED_DECLARATION", - "message": "C++ declaration syntax is not supported by the C parser.", + "code": "C_UNSUPPORTED_DECLARATOR", + "message": "Unsupported declarator syntax after parsed type layers: ' static T * stbds_arrgrowf_wrapper(T *a, size_t elemsize, size_t addlen, size_t min_cap)'.", "severity": "warning", "location": { "filename": "stb/stb_ds.h", @@ -13623,12 +13623,12 @@ "column": 1, "source_line": "template static T * stbds_arrgrowf_wrapper(T *a, size_t elemsize, size_t addlen, size_t min_cap) {" }, - "unit_kind": "cxx_declaration", + "unit_kind": "declarator", "unit_name": null }, { - "code": "C_UNSUPPORTED_DECLARATION", - "message": "C++ declaration syntax is not supported by the C parser.", + "code": "C_UNSUPPORTED_DECLARATOR", + "message": "Unsupported declarator syntax after parsed type layers: ' static T * stbds_hmget_key_wrapper(T *a, size_t elemsize, void *key, size_t keysize, int mode)'.", "severity": "warning", "location": { "filename": "stb/stb_ds.h", @@ -13636,12 +13636,12 @@ "column": 1, "source_line": "template static T * stbds_hmget_key_wrapper(T *a, size_t elemsize, void *key, size_t keysize, int mode) {" }, - "unit_kind": "cxx_declaration", + "unit_kind": "declarator", "unit_name": null }, { - "code": "C_UNSUPPORTED_DECLARATION", - "message": "C++ declaration syntax is not supported by the C parser.", + "code": "C_UNSUPPORTED_DECLARATOR", + "message": "Unsupported declarator syntax after parsed type layers: ' static T * stbds_hmget_key_ts_wrapper(T *a, size_t elemsize, void *key, size_t keysize, ptrdiff_t *temp, int mode)'.", "severity": "warning", "location": { "filename": "stb/stb_ds.h", @@ -13649,12 +13649,12 @@ "column": 1, "source_line": "template static T * stbds_hmget_key_ts_wrapper(T *a, size_t elemsize, void *key, size_t keysize, ptrdiff_t *temp, int mode) {" }, - "unit_kind": "cxx_declaration", + "unit_kind": "declarator", "unit_name": null }, { - "code": "C_UNSUPPORTED_DECLARATION", - "message": "C++ declaration syntax is not supported by the C parser.", + "code": "C_UNSUPPORTED_DECLARATOR", + "message": "Unsupported declarator syntax after parsed type layers: ' static T * stbds_hmput_default_wrapper(T *a, size_t elemsize)'.", "severity": "warning", "location": { "filename": "stb/stb_ds.h", @@ -13662,12 +13662,12 @@ "column": 1, "source_line": "template static T * stbds_hmput_default_wrapper(T *a, size_t elemsize) {" }, - "unit_kind": "cxx_declaration", + "unit_kind": "declarator", "unit_name": null }, { - "code": "C_UNSUPPORTED_DECLARATION", - "message": "C++ declaration syntax is not supported by the C parser.", + "code": "C_UNSUPPORTED_DECLARATOR", + "message": "Unsupported declarator syntax after parsed type layers: ' static T * stbds_hmput_key_wrapper(T *a, size_t elemsize, void *key, size_t keysize, int mode)'.", "severity": "warning", "location": { "filename": "stb/stb_ds.h", @@ -13675,12 +13675,12 @@ "column": 1, "source_line": "template static T * stbds_hmput_key_wrapper(T *a, size_t elemsize, void *key, size_t keysize, int mode) {" }, - "unit_kind": "cxx_declaration", + "unit_kind": "declarator", "unit_name": null }, { - "code": "C_UNSUPPORTED_DECLARATION", - "message": "C++ declaration syntax is not supported by the C parser.", + "code": "C_UNSUPPORTED_DECLARATOR", + "message": "Unsupported declarator syntax after parsed type layers: ' static T * stbds_hmdel_key_wrapper(T *a, size_t elemsize, void *key, size_t keysize, size_t keyoffset, int mode)'.", "severity": "warning", "location": { "filename": "stb/stb_ds.h", @@ -13688,7 +13688,7 @@ "column": 1, "source_line": "template static T * stbds_hmdel_key_wrapper(T *a, size_t elemsize, void *key, size_t keysize, size_t keyoffset, int mode){" }, - "unit_kind": "cxx_declaration", + "unit_kind": "declarator", "unit_name": null }, { From 01615d0d27ad07567e30d9351f1d12da8f5f92a2 Mon Sep 17 00:00:00 2001 From: said Date: Sun, 31 May 2026 04:26:31 +0100 Subject: [PATCH 06/13] update errors --- README.md | 7 +- c_parser/models.py | 2 +- c_parser/parser.py | 8 +- docs/c_parser/c_parser_architecture.md | 6 +- docs/c_parser/c_parser_cli_workflow.md | 14 ++-- docs/c_parser/c_parser_reference.md | 13 +-- docs/diagnostic_codes.md | 49 +++++++---- docs/fortran/fortran_parser.md | 82 ++++++------------- .../parser_implementation_reference.md | 14 ++-- fortran_parser/models.py | 2 +- fortran_parser/parser.py | 78 +++++++++++++++--- .../errors/invalid_type_specifiers.h.json | 2 +- tests/parser/c/test_c_cli_skeleton.py | 2 +- .../c/test_c_declarations_and_declarators.py | 4 +- tests/parser/c/test_c_public_api_skeleton.py | 4 +- .../errors/err_duplicate_argument_name.json | 2 +- .../err_duplicate_declaration_procedure.json | 2 +- .../err_duplicate_field_derived_type.json | 2 +- .../errors/err_duplicate_parameter.json | 2 +- .../err_duplicate_procedure_global.json | 2 +- .../err_duplicate_procedure_module.json | 2 +- .../errors/err_duplicate_variable_module.json | 2 +- .../err_implicit_none_undeclared_arg.json | 2 +- .../err_implicit_none_undeclared_result.json | 2 +- ..._parameter_without_type_implicit_none.json | 2 +- .../errors/err_result_shadows_argument.json | 2 +- .../errors/err_unknown_function_result.json | 2 +- .../errors/err_unknown_type_derived_type.json | 2 +- .../errors/err_unknown_type_module.json | 2 +- .../errors/err_unknown_type_procedure.json | 2 +- tests/parser/test_cli.py | 6 +- tests/parser/test_error_handling.py | 6 +- 32 files changed, 184 insertions(+), 145 deletions(-) diff --git a/README.md b/README.md index 284f54771..f9ad1c663 100644 --- a/README.md +++ b/README.md @@ -120,10 +120,9 @@ silently dropping malformed input. Parse failures print a compiler-style diagnostic without a Python traceback. Use `--debug` to re-raise the parser error and print the traceback; `--debug-traceback` remains accepted as a compatibility alias. Diagnostic codes -such as `PARSE001`, `CPARSE003`, and `CPARSE_INVALID_SYNTAX` are stable error -category identifiers for tests, tools, and documentation. Their numbers do not -represent the source line, the number of errors, or the process exit status. -The current categories are listed in +such as `PARSE_UNSUPPORTED_DECLARATION`, `CPARSE_INVALID_SPECIFIER_SEQUENCE`, +and `CPARSE_INVALID_SYNTAX` are stable, explicit error-category identifiers for +tests, tools, and documentation. The current categories are listed in [`docs/diagnostic_codes.md`](docs/diagnostic_codes.md). For parse output, `--show-vars` expands scope-level variables that are normally diff --git a/c_parser/models.py b/c_parser/models.py index 47d344d93..e5e2d8b55 100644 --- a/c_parser/models.py +++ b/c_parser/models.py @@ -106,7 +106,7 @@ def c_model_to_dict(obj: Any, _seen: set[int] | None = None) -> Any: class CParseError(ValueError): """C parser error with compiler-style diagnostic rendering support.""" - default_code = "CPARSE001" + default_code = "CPARSE_ERROR" def __init__( self, diff --git a/c_parser/parser.py b/c_parser/parser.py index c8541695a..7c0ded12c 100644 --- a/c_parser/parser.py +++ b/c_parser/parser.py @@ -1191,7 +1191,7 @@ def _invalid_specifier_error( line_number=location.line, column=location.column, source_line=location.source_line, - code="CPARSE003", + code="CPARSE_INVALID_SPECIFIER_SEQUENCE", ) def _atomic_type_specifier_parts(self, spec_text: str) -> tuple[str, str] | None: @@ -1815,7 +1815,7 @@ def _raise_for_unsupported_old_style_definitions( line_number=mapping.line if mapping is not None else index + 1, column=max(line.find(name_match.group(0)) + 1, 1), source_line=source_line, - code="CPARSE002", + code="CPARSE_UNSUPPORTED_KNR_DEFINITION", ) if stripped.endswith(";"): saw_old_style_declaration = True @@ -1833,7 +1833,7 @@ def _raise_for_unsupported_old_style_definitions( line_number=mapping.line if mapping is not None else index + 1, column=max(line.find(name_match.group(0)) + 1, 1), source_line=source_line, - code="CPARSE002", + code="CPARSE_UNSUPPORTED_KNR_DEFINITION", ) def _prototype_style(self, parameters_text: str) -> str: @@ -1869,7 +1869,7 @@ def _parse_function(self, segment: CTopLevelSegment) -> CFunction | None: line_number=segment.original_start_line, column=segment.original_start_column, source_line=segment.original_source_line, - code="CPARSE002", + code="CPARSE_UNSUPPORTED_KNR_DEFINITION", ) return self._function_from_type( name, diff --git a/docs/c_parser/c_parser_architecture.md b/docs/c_parser/c_parser_architecture.md index e8c5849b9..f72896544 100644 --- a/docs/c_parser/c_parser_architecture.md +++ b/docs/c_parser/c_parser_architecture.md @@ -71,7 +71,8 @@ Implemented now: become diagnostics. Grammar-invalid input raises `CParseError` with `CPARSE_INVALID_SYNTAX`; identifier spellings are not used to guess another language. Primitive specifier order is normalized, and invalid combinations - such as `unsigned float` raise `CParseError` with code `CPARSE003` while a + such as `unsigned float` raise `CParseError` with code + `CPARSE_INVALID_SPECIFIER_SEQUENCE` while a single unresolved typedef-like name remains deferred. Definitions preserve direct `start` and `end` locations from the signature start through the closing brace; and K&R-style function definitions raise focused diagnostics. @@ -284,7 +285,8 @@ Current and planned responsibilities: helper methods. Function models record prototype-style versus unspecified empty parameter lists, function definitions preserve start/end locations, K&R-style definitions are rejected with `CParseError`, and invalid - primitive-specifier combinations are rejected with `CPARSE003`. Array and + primitive-specifier combinations are rejected with + `CPARSE_INVALID_SPECIFIER_SEQUENCE`. Array and function parameters preserve `declared_type` while effective `type` uses C parameter adjustment. Raw declarations beginning with an object-like macro name are retained as macro-dependent diagnostics. diff --git a/docs/c_parser/c_parser_cli_workflow.md b/docs/c_parser/c_parser_cli_workflow.md index 3a5a90071..ffff2a2c0 100644 --- a/docs/c_parser/c_parser_cli_workflow.md +++ b/docs/c_parser/c_parser_cli_workflow.md @@ -189,7 +189,7 @@ Current C behavior also accepts `--no-color`. `CParseError` supports compiler-style diagnostic formatting and the `C_PARSER_DEBUG` environment variable. The current grammar subset is tolerant for recoverable unsupported declaration forms, but invalid primitive-specifier combinations raise -`CPARSE003`; unresolved single typedef-name uses are deferred until type +`CPARSE_INVALID_SPECIFIER_SEQUENCE`; unresolved single typedef-name uses are deferred until type resolution can determine whether a declaration exists. `--debug-traceback` remains accepted as a compatibility alias. @@ -561,16 +561,14 @@ Invalid primitive-specifier combinations that are independent of symbol resolution are fatal: ```text -src/api.h:12:1: error[CPARSE003]: Invalid type specifier sequence 'unsigned float'. +src/api.h:12:1: error[CPARSE_INVALID_SPECIFIER_SEQUENCE]: Invalid type specifier sequence 'unsigned float'. 12 | unsigned float value; | ^ ``` Grammar-invalid C syntax is also fatal and uses -`CPARSE_INVALID_SYNTAX`. Diagnostic codes are stable category identifiers for -tests, tools, and documentation. A numeric suffix such as the one in -`CPARSE003` is not a source line number, an occurrence counter, or an exit -status. The shared registry is +`CPARSE_INVALID_SYNTAX`. Diagnostic codes are stable, explicit category +identifiers for tests, tools, and documentation. The shared registry is [`docs/diagnostic_codes.md`](../diagnostic_codes.md). Debug behavior: @@ -649,8 +647,8 @@ Completed order: 12. Replaced generic type references and declaration-kind tags with concrete `CType` subclasses, `CComposedType` components, and concrete declaration objects. -13. Added order-insensitive primitive specifier validation and `CPARSE003` - errors for invalid primitive combinations while retaining unresolved +13. Added order-insensitive primitive specifier validation and + `CPARSE_INVALID_SPECIFIER_SEQUENCE` errors for invalid primitive combinations while retaining unresolved typedef-name references for later resolution. 14. Added field-level source locations, flexible array member classification and invalid-use diagnostics, plus explicit bit-field regression coverage. diff --git a/docs/c_parser/c_parser_reference.md b/docs/c_parser/c_parser_reference.md index 45c54c5ad..3b154e6a4 100644 --- a/docs/c_parser/c_parser_reference.md +++ b/docs/c_parser/c_parser_reference.md @@ -71,7 +71,8 @@ Implemented: - raw `#undef` directive provenance in macro metadata - concrete primitive `CType` objects, pointer/array composition, and concrete qualifier objects -- order-insensitive primitive specifier matching with `CPARSE003` errors for +- order-insensitive primitive specifier matching with + `CPARSE_INVALID_SPECIFIER_SEQUENCE` errors for invalid combinations such as `unsigned float` - recursive declarator extraction for parenthesized pointer/array precedence - nameless `CFunctionType` signatures for function pointer typedefs and @@ -357,8 +358,8 @@ type component they qualify. `_Atomic int value;` is stored with a `CAtomic` qualifier; `_Atomic(int) value;` is represented the same way, while `_Atomic(int *) value;` qualifies the pointer component. Equivalent primitive orderings, such as `int unsigned` and `double long`, map to the same concrete type while -invalid combinations, such as `unsigned float`, raise `CParseError` with -code `CPARSE003`. A single unresolved typedef-name use remains a `CTypedef` +invalid combinations, such as `unsigned float`, raise `CParseError` with code +`CPARSE_INVALID_SPECIFIER_SEQUENCE`. A single unresolved typedef-name use remains a `CTypedef` until resolution can establish whether a matching declaration exists. Nested declarators are `CComposedType` objects whose `components` are read @@ -597,13 +598,13 @@ non-fatal metadata diagnostics, such as unresolved local includes or macros that affect declarations but were recorded rather than expanded. K&R-style function definitions now raise `CParseError` because the current function parser only models prototype-style declarations and definitions. Invalid primitive -specifier combinations also raise `CParseError` (`CPARSE003`) because their +specifier combinations also raise `CParseError` +(`CPARSE_INVALID_SPECIFIER_SEQUENCE`) because their invalidity does not depend on later typedef resolution. Known unsupported declaration extensions are diagnosed rather than partially modeled; additional syntax diagnostics should be added only with focused tests. Generic grammar rejection uses `CPARSE_INVALID_SYNTAX`. Diagnostic codes are -stable category identifiers for tests, tools, and documentation; numeric -suffixes are not line numbers, occurrence counters, or exit statuses. The +stable, explicit category identifiers for tests, tools, and documentation. The shared registry is [`docs/diagnostic_codes.md`](../diagnostic_codes.md). ## Testing Workflow diff --git a/docs/diagnostic_codes.md b/docs/diagnostic_codes.md index 8771886c0..90dee7ffb 100644 --- a/docs/diagnostic_codes.md +++ b/docs/diagnostic_codes.md @@ -3,10 +3,8 @@ Diagnostic codes are stable category identifiers for users, tests, and tooling. They are not source line numbers, occurrence counters, or process exit statuses. -New categories should use explicit symbolic names such as -`PARSE_INVALID_SYNTAX` or `C_UNRESOLVED_INCLUDE`. Existing numbered codes remain -supported for compatibility. Replacing a numbered code should be a deliberate -compatibility change with tests and generated fixtures updated together. +Categories use explicit symbolic names such as `PARSE_INVALID_SYNTAX` and +`C_UNRESOLVED_INCLUDE`. The name describes the failure class directly. ## Fatal Parser Errors @@ -15,21 +13,43 @@ traceback unless `--debug` is used. | Code | Frontend | Meaning | | --- | --- | --- | -| `PARSE001` | Fortran | Compatibility fallback for a Fortran parse error without a more specific code. | +| `PARSE_ERROR` | Fortran | Fallback for a manually constructed or defensive Fortran parse error without a narrower category. | | `PARSE_INVALID_SYNTAX` | Fortran | Syntax cannot be consumed in a modeled Fortran grammar region. | | `PARSE_WRONG_ENTRYPOINT` | Fortran | A singular public parser API was called for a different source-unit kind. | | `PARSE_AMBIGUOUS_ENTRYPOINT` | Fortran | A singular public parser API matched more than one source unit. | -| `CPARSE001` | C | Compatibility fallback for a C parse error without a more specific code. | -| `CPARSE002` | C | Unsupported K&R-style function definition. | -| `CPARSE003` | C | Invalid C primitive-specifier sequence. | +| `PARSE_EXPECTED_UNIT` | Fortran | An internal unit visitor received the wrong source-unit kind. | +| `PARSE_MISSING_UNIT_END` | Fortran | A source unit has no closing statement. | +| `PARSE_MISMATCHED_UNIT_END` | Fortran | A named source-unit closing statement does not match its opener. | +| `PARSE_UNEXPECTED_UNIT_END` | Fortran | A closing statement appears while another nested unit is active. | +| `PARSE_DUPLICATE_UNIT` | Fortran | A scope contains duplicate named source units of the same kind. | +| `PARSE_DUPLICATE_PROCEDURE` | Fortran | A scope contains duplicate procedure names. | +| `PARSE_MALFORMED_HEADER` | Fortran | A module or procedure header is unsupported or malformed. | +| `PARSE_UNSUPPORTED_RESULT_TYPE` | Fortran | A function header contains an unsupported result-type prefix. | +| `PARSE_DUPLICATE_DECLARATION` | Fortran | A procedure symbol is declared more than once. | +| `PARSE_UNKNOWN_PARAMETER_TYPE` | Fortran | A `PARAMETER` symbol has no declared type where one is required. | +| `PARSE_DUPLICATE_PARAMETER` | Fortran | A procedure contains duplicate `PARAMETER` declarations. | +| `PARSE_DUPLICATE_SYMBOL` | Fortran | A file or project scope contains a duplicate symbol. | +| `PARSE_UNSUPPORTED_OPENMP_DIRECTIVE` | Fortran | A modeled specification region contains an unsupported OpenMP directive. | +| `PARSE_MISSING_DERIVED_TYPE_END` | Fortran | A derived-type declaration has no matching closing statement. | +| `PARSE_EXECUTABLE_IN_SPECIFICATION` | Fortran | An executable statement appears in a non-executable specification region. | +| `PARSE_UNSUPPORTED_DECLARATION` | Fortran | A declaration-shaped line uses an unsupported datatype form. | +| `PARSE_UNSUPPORTED_TYPE_BOUND_DECLARATION` | Fortran | A derived-type `contains` region has an unsupported binding declaration. | +| `PARSE_UNRESOLVED_ARGUMENT_TYPE` | Fortran | A defensive invariant could not apply a declared argument type. | +| `PARSE_UNKNOWN_FUNCTION_RESULT_TYPE` | Fortran | A function result has no resolvable datatype. | +| `PARSE_IMPLICIT_NONE_UNDECLARED_SYMBOL` | Fortran | `implicit none` requires a missing argument or result declaration. | +| `PARSE_MISSING_FUNCTION_RESULT` | Fortran | A defensive invariant found a function without a result variable. | +| `PARSE_RESULT_SHADOWS_ARGUMENT` | Fortran | A function result name shadows an argument. | +| `PARSE_DUPLICATE_VARIABLE` | Fortran | A module-like scope contains conflicting duplicate variable declarations. | +| `PARSE_UNKNOWN_VARIABLE_TYPE` | Fortran | A module variable still has an unknown datatype after parsing. | +| `PARSE_DUPLICATE_FIELD` | Fortran | A derived type contains duplicate fields. | +| `PARSE_UNKNOWN_FIELD_TYPE` | Fortran | A derived-type field still has an unknown datatype after parsing. | +| `PARSE_DUPLICATE_ARGUMENT` | Fortran | A procedure argument list repeats a name. | +| `PARSE_INTERNAL_STATE` | Fortran | A defensive internal parser invariant was violated. | +| `CPARSE_ERROR` | C | Fallback for a manually constructed or defensive C parse error without a narrower category. | +| `CPARSE_UNSUPPORTED_KNR_DEFINITION` | C | Unsupported K&R-style function definition. | +| `CPARSE_INVALID_SPECIFIER_SEQUENCE` | C | Invalid C primitive-specifier sequence. | | `CPARSE_INVALID_SYNTAX` | C | Syntax cannot be consumed in a modeled C grammar region. | -`PARSE001`, `CPARSE001`, `CPARSE002`, and `CPARSE003` predate the explicit -category naming rule. Prefer symbolic names for new categories. If the numbered -codes are migrated later, useful replacements would be names such as -`PARSE_UNKNOWN_DATATYPE`, `CPARSE_UNSUPPORTED_KNR_DEFINITION`, and -`CPARSE_INVALID_SPECIFIER_SEQUENCE`. - ## C Report Diagnostics The C parser can preserve partial metadata and attach `CDiagnostic` records. @@ -53,4 +73,3 @@ These records do not necessarily stop parsing; inspect each diagnostic's | `C_DUPLICATE_VARIABLE_DEFINITION` | File-scope variable has more than one definition. | | `C_CONFLICTING_TYPEDEF` | Typedef declarations conflict. | | `C_DUPLICATE_TAG_DEFINITION` | Struct, union, or enum tag has more than one definition. | - diff --git a/docs/fortran/fortran_parser.md b/docs/fortran/fortran_parser.md index f7d0244ec..732bfc89e 100644 --- a/docs/fortran/fortran_parser.md +++ b/docs/fortran/fortran_parser.md @@ -464,7 +464,7 @@ python -m x2py tests/data/fortran/errors/err_duplicate_argument_name.f90 Example diagnostic shape: ```text -tests/data/fortran/errors/err_duplicate_argument_name.f90:1:1: error[PARSE001]: Duplicate argument name 'x' in procedure 'dup'. +tests/data/fortran/errors/err_duplicate_argument_name.f90:1:1: error[PARSE_DUPLICATE_ARGUMENT]: Duplicate argument name 'x' in procedure 'dup'. | 1 | subroutine dup(x, y, x) | ^ @@ -623,19 +623,19 @@ exception keeps structured metadata for consumers: - `line_number` — 1-based source line where the error was detected, if known - `source_line` — original source text for context, if known - `base_message` — stable error text without location/source context -- `code` — stable diagnostic category identifier; the default parse diagnostic - code is `PARSE001`, while grammar rejection uses `PARSE_INVALID_SYNTAX` +- `code` — stable, explicit diagnostic category identifier; manually + constructed fallback errors use `PARSE_ERROR`, while grammar rejection uses + `PARSE_INVALID_SYNTAX` Diagnostic codes are for programmatic matching in tests, tools, and -documentation. The numeric suffix in `PARSE001` identifies an error category; -it is not a source line number, an occurrence counter, or the CLI exit status. -The shared registry is [`docs/diagnostic_codes.md`](../diagnostic_codes.md). +documentation. The category name states the failure class directly. The shared +registry is [`docs/diagnostic_codes.md`](../diagnostic_codes.md). `str(error)` and `error.format_diagnostic(color=False)` render a compiler-style diagnostic: ```text -::1: error[PARSE001]: +::1: error[]: | | | ^ @@ -682,7 +682,7 @@ end subroutine bad Example error: ``` -bad.f90:2:1: error[PARSE001]: Unknown or unsupported datatype declaration for procedure 'bad': weirdtype :: x +bad.f90:2:1: error[PARSE_UNSUPPORTED_DECLARATION]: Unknown or unsupported datatype declaration for procedure 'bad': weirdtype :: x | 2 | weirdtype :: x | ^ @@ -722,7 +722,7 @@ end subroutine dup Example error: ``` -dup.f90:3:1: error[PARSE001]: Duplicate declaration of symbol 'x' in procedure 'dup'. +dup.f90:3:1: error[PARSE_DUPLICATE_DECLARATION]: Duplicate declaration of symbol 'x' in procedure 'dup'. | 3 | integer :: x | ^ @@ -780,7 +780,7 @@ end subroutine work Example error: ``` -dup.f90:5:1: error[PARSE001]: Duplicate procedure name 'work' in global scope. +dup.f90:5:1: error[PARSE_DUPLICATE_PROCEDURE]: Duplicate procedure name 'work' in global scope. | 5 | subroutine work(n) | ^ @@ -806,62 +806,28 @@ end subroutine dup Example error: ``` -dup_arg.f90:1:1: error[PARSE001]: Duplicate argument name 'x' in procedure 'dup'. +dup_arg.f90:1:1: error[PARSE_DUPLICATE_ARGUMENT]: Duplicate argument name 'x' in procedure 'dup'. | 1 | subroutine dup(x, y, x) | ^ ``` -### 6.5 Star-kind in modern source +### 6.5 Star-kind declarations -Triggered when a legacy `type*N` (e.g. `real*8`) declaration appears in a file -with a modern Fortran extension (`.f90`, `.f95`, `.f03`, `.f08`). - -``` -Unsupported Fortran 77 star-kind declaration '*' in modern source ''. -``` - -Example: +Legacy `type*N` declarations, such as `real*8`, are accepted in both fixed-form +and modern-extension files. The parser preserves the kind metadata: ```fortran -subroutine bad(x) +subroutine accepted(x) real*8 :: x -end subroutine bad +end subroutine accepted ``` -Example error (file `bad.f90`): - -``` -bad.f90:2:1: error[PARSE001]: Unsupported Fortran 77 star-kind declaration 'real*8' in modern source 'bad.f90'. - | -2 | real*8 :: x - | ^ -``` - -### 6.6 Fortran 77 syntax in a `.f77` source file - -Triggered when modern constructs (`module`, `contains`, `interface`, -`class(...)`) appear in a file with extension `.f77`. +### 6.6 Source-form metadata -``` -Unsupported syntax for Fortran 77 source '': -``` - -Example: - -```fortran - module bad_module - end module bad_module -``` - -Example error (file `legacy.f77`): - -``` -legacy.f77:1:1: error[PARSE001]: Unsupported syntax for Fortran 77 source 'legacy.f77': module bad_module - | -1 | module bad_module - | ^ -``` +The parser records source-form metadata from the filename and lexer, but does +not reject a construct solely because a `.f77` suffix was used. Grammar-region +validation still applies after preprocessing. ### 6.7 Implicit none — undeclared argument or result @@ -892,7 +858,7 @@ end subroutine foo Example error: ``` -implicit_none.f90:1:1: error[PARSE001]: Argument 'y' in procedure 'foo' has no type declaration (implicit none is active). +implicit_none.f90:1:1: error[PARSE_IMPLICIT_NONE_UNDECLARED_SYMBOL]: Argument 'y' in procedure 'foo' has no type declaration (implicit none is active). | 1 | subroutine foo(x, y) | ^ @@ -919,7 +885,7 @@ end function f Example error: ``` -bad.f90:1:1: error[PARSE001]: Unknown datatype for function result 'res' in procedure 'f'. +bad.f90:1:1: error[PARSE_UNKNOWN_FUNCTION_RESULT_TYPE]: Unknown datatype for function result 'res' in procedure 'f'. | 1 | function f(x) result(res) | ^ @@ -965,7 +931,7 @@ Example: Example error: ``` -legacy.f:4:1: error[PARSE001]: Unknown datatype for PARAMETER symbol 'zero' in procedure 'cst'. +legacy.f:4:1: error[PARSE_UNKNOWN_PARAMETER_TYPE]: Unknown datatype for PARAMETER symbol 'zero' in procedure 'cst'. | 4 | parameter ( zero = 0.0e+0 ) | ^ @@ -992,7 +958,7 @@ end function f Example error: ``` -shadow.f90:1:1: error[PARSE001]: Function result variable 'res' in function 'f' shadows an argument name. +shadow.f90:1:1: error[PARSE_RESULT_SHADOWS_ARGUMENT]: Function result variable 'res' in function 'f' shadows an argument name. | 1 | function f(res) result(res) | ^ diff --git a/docs/fortran/parser_implementation_reference.md b/docs/fortran/parser_implementation_reference.md index 4f4662fd1..347cd983c 100644 --- a/docs/fortran/parser_implementation_reference.md +++ b/docs/fortran/parser_implementation_reference.md @@ -314,7 +314,7 @@ stage separation, and `--semantics --wrap-readiness`. Dedicated tests for the error handling system: - `FortranParseError` attribute presence (`filename`, `line_number`, `source_line`, `base_message`, `code`) -- Compiler-style diagnostic formatting with `PARSE001`, source line context, and caret marker +- Compiler-style diagnostic formatting with explicit categories, source line context, and caret marker - ANSI color formatting and environment-variable debug activation - Error raised for all error categories in all scopes: procedures, modules, derived types, interfaces - Line number accuracy @@ -673,14 +673,15 @@ When updating parser behavior, keep this fail-fast contract aligned with tests: - `line_number` — 1-based line number in the original source where the error was detected - `source_line` — the original (pre-preprocessed) source line text - `base_message` — the stable error message without source/location context -- `code` — stable diagnostic category identifier; current parser errors default - to `PARSE001`, while grammar rejection uses `PARSE_INVALID_SYNTAX` +- `code` — stable, explicit diagnostic category identifier; manually + constructed fallback errors use `PARSE_ERROR`, while grammar rejection uses + `PARSE_INVALID_SYNTAX` - `parser_file`, `parser_line_number`, `parser_function` — internal raise-site metadata used only for debug diagnostics The formatted `str()` of `FortranParseError` is a compiler-style diagnostic: ```text -::1: error[PARSE001]: +::1: error[]: | | | ^ @@ -691,9 +692,8 @@ Use `error.format_diagnostic(color=True)` to add ANSI color and line with the internal parser location. `format_diagnostic(debug=None)` also honors `FORTRAN_PARSER_DEBUG=1`. -The numeric suffix in a code such as `PARSE001` identifies an error category -for tests, tools, and documentation. It is not a line number, an occurrence -counter, or an exit status. The shared registry is +The category name identifies the failure class for tests, tools, and +documentation. The shared registry is [`docs/diagnostic_codes.md`](../diagnostic_codes.md). CLI contract: diff --git a/fortran_parser/models.py b/fortran_parser/models.py index 4e2cd3420..7219f87c6 100644 --- a/fortran_parser/models.py +++ b/fortran_parser/models.py @@ -136,7 +136,7 @@ def _enable_windows_ansi() -> None: # pragma: no cover - Windows-only console s class FortranParseError(ValueError): """Parser error with compiler-style diagnostic rendering support.""" - default_code = "PARSE001" + default_code = "PARSE_ERROR" def __init__( self, diff --git a/fortran_parser/parser.py b/fortran_parser/parser.py index 90273c938..69ec219ac 100644 --- a/fortran_parser/parser.py +++ b/fortran_parser/parser.py @@ -580,7 +580,7 @@ def visit_module_unit( header = unit.lines[0] module = self._parse_module_header(header[0].strip(), filename, lineno=header[1], source_line=header[2]) if module is None: # pragma: no cover - slicer only dispatches module units with module headers. - raise FortranParseError("Expected module unit.", filename=filename, line_number=header[1], source_line=header[2]) + raise FortranParseError("Expected module unit.", filename=filename, line_number=header[1], source_line=header[2], code="PARSE_EXPECTED_UNIT") scope = self._helper_scope_for_model("module", module, parent=parent_scope) parts = self._helper_split_unit_parts(unit, self._helper_unit_grammar("module"), filename=filename) self._helper_visit_spec_part(scope, parts.specification, filename=filename) @@ -627,7 +627,7 @@ def visit_submodule_unit( header = unit.lines[0] submodule = self._parse_submodule_header(header[0].strip(), filename) if submodule is None: # pragma: no cover - slicer only dispatches submodule units with submodule headers. - raise FortranParseError("Expected submodule unit.", filename=filename, line_number=header[1], source_line=header[2]) + raise FortranParseError("Expected submodule unit.", filename=filename, line_number=header[1], source_line=header[2], code="PARSE_EXPECTED_UNIT") scope = self._helper_scope_for_model("submodule", submodule, parent=parent_scope) parts = self._helper_split_unit_parts(unit, self._helper_unit_grammar("submodule"), filename=filename) self._helper_visit_spec_part(scope, parts.specification, filename=filename) @@ -673,7 +673,7 @@ def visit_program_unit( header = unit.lines[0] program = self._parse_program_header(header[0].strip(), filename) if program is None: # pragma: no cover - slicer only dispatches program units with program headers. - raise FortranParseError("Expected program unit.", filename=filename, line_number=header[1], source_line=header[2]) + raise FortranParseError("Expected program unit.", filename=filename, line_number=header[1], source_line=header[2], code="PARSE_EXPECTED_UNIT") scope = self._helper_scope_for_model("program", program, parent=parent_scope) parts = self._helper_split_unit_parts(unit, self._helper_unit_grammar("program"), filename=filename) self._helper_visit_spec_part(scope, parts.specification, filename=filename) @@ -706,7 +706,7 @@ def visit_block_data_source_unit( header = unit.lines[0] block_data = self._parse_block_data_header(header[0].strip(), filename) if block_data is None: # pragma: no cover - slicer only dispatches block-data units with block-data headers. - raise FortranParseError("Expected block data unit.", filename=filename, line_number=header[1], source_line=header[2]) + raise FortranParseError("Expected block data unit.", filename=filename, line_number=header[1], source_line=header[2], code="PARSE_EXPECTED_UNIT") scope = self._helper_scope_for_model("block_data", block_data, parent=parent_scope) parts = self._helper_split_unit_parts(unit, self._helper_unit_grammar("block_data"), filename=filename) self._helper_visit_spec_part(scope, parts.specification, filename=filename) @@ -731,7 +731,7 @@ def visit_derived_type_unit( header = unit.lines[0] dtype = self._init_derived_type(header[0].strip(), current_module=parent_scope.module_owner) if dtype is None: # pragma: no cover - slicer only dispatches derived-type units with type headers. - raise FortranParseError("Expected derived-type unit.", filename=filename, line_number=header[1], source_line=header[2]) + raise FortranParseError("Expected derived-type unit.", filename=filename, line_number=header[1], source_line=header[2], code="PARSE_EXPECTED_UNIT") scope = self._helper_scope_for_model("derived_type", dtype, parent=parent_scope) parts = self._helper_split_unit_parts(unit, self._helper_unit_grammar("derived_type"), filename=filename) self._helper_visit_spec_part(scope, parts.specification, filename=filename) @@ -762,7 +762,7 @@ def visit_interface_unit( header = unit.lines[0] starts_interface, interface_name = self._parse_interface_header(header[0].strip()) if not starts_interface: # pragma: no cover - slicer only dispatches interface units with interface headers. - raise FortranParseError("Expected interface unit.", filename=filename, line_number=header[1], source_line=header[2]) + raise FortranParseError("Expected interface unit.", filename=filename, line_number=header[1], source_line=header[2], code="PARSE_EXPECTED_UNIT") interface = FortranInterface(name=interface_name, module=parent_scope.module_owner) scope = self._helper_scope_for_model("interface", interface, parent=parent_scope) parts = self._helper_split_unit_parts(unit, self._helper_unit_grammar("interface"), filename=filename) @@ -812,7 +812,7 @@ def visit_procedure_unit( lineno=header[1], source_line=header[2], ) - raise FortranParseError("Expected procedure unit.", filename=filename, line_number=header[1], source_line=header[2]) + raise FortranParseError("Expected procedure unit.", filename=filename, line_number=header[1], source_line=header[2], code="PARSE_EXPECTED_UNIT") proc_state["filename"] = filename proc_state["header_lineno"] = header[1] proc_state["header_source_line"] = header[2] @@ -1361,6 +1361,7 @@ def _helper_slice_child_units( filename=filename, line_number=lineno, source_line=lines[index][2], + code="PARSE_MISSING_UNIT_END", ) end_line = lines[end_index][1] @@ -1432,6 +1433,7 @@ def _helper_find_unit_end( filename=filename, line_number=lineno, source_line=source_line, + code="PARSE_MISMATCHED_UNIT_END", ) stack.pop() if not stack: @@ -1470,6 +1472,7 @@ def _helper_find_unit_end( filename=filename, line_number=lineno, source_line=source_line, + code="PARSE_UNEXPECTED_UNIT_END", ) idx += 1 return None @@ -1884,12 +1887,14 @@ def _helper_validate_sibling_units( filename=filename, line_number=unit.start_line, source_line=unit.lines[0][2] if unit.lines else None, + code="PARSE_DUPLICATE_PROCEDURE", ) raise FortranParseError( f"Duplicate {unit.kind.replace('_', ' ')} name '{unit.name}' in {parent_scope.kind} scope.", filename=filename, line_number=unit.start_line, source_line=unit.lines[0][2] if unit.lines else None, + code="PARSE_DUPLICATE_UNIT", ) seen.setdefault(key, []).append(unit) @@ -2089,6 +2094,7 @@ def _parse_module_header( filename=filename, line_number=lineno, source_line=source_line, + code="PARSE_MALFORMED_HEADER", ) return None return FortranModule(name=module_match.group("name"), filename=filename) @@ -2220,6 +2226,7 @@ def _parse_procedure_header( filename=filename, line_number=lineno, source_line=source_line, + code="PARSE_UNSUPPORTED_RESULT_TYPE", ) if parsed_prefix: result.base_type, result.kind = parsed_prefix @@ -2261,6 +2268,7 @@ def _raise_if_unparsed_procedure_header( filename=filename, line_number=lineno, source_line=source_line, + code="PARSE_MALFORMED_HEADER", ) if FortranParser._looks_like_procedure_header(stripped): raise FortranParseError( @@ -2268,6 +2276,7 @@ def _raise_if_unparsed_procedure_header( filename=filename, line_number=lineno, source_line=source_line, + code="PARSE_MALFORMED_HEADER", ) @staticmethod @@ -2380,6 +2389,7 @@ def _proc_scope_mark_declared_symbol( filename=filename, line_number=line_number, source_line=source_line, + code="PARSE_DUPLICATE_DECLARATION", ) proc_state["typed_symbols"].add(key) return key @@ -2425,6 +2435,7 @@ def _proc_scope_add_local_parameter( filename=filename, line_number=line_number, source_line=source_line, + code="PARSE_UNKNOWN_PARAMETER_TYPE", ) if key in proc_state["local_params"]: raise FortranParseError( @@ -2432,6 +2443,7 @@ def _proc_scope_add_local_parameter( filename=filename, line_number=line_number, source_line=source_line, + code="PARSE_DUPLICATE_PARAMETER", ) proc_state["local_params"][key] = value if register_implicit_if_missing and not self._proc_scope_symbol_is_declared(proc_state, key): @@ -2449,7 +2461,11 @@ def _insert_unique_scope_symbol( filename: str | None = None, ) -> None: if key in scope: - raise FortranParseError(f"Duplicate symbol '{key}' in {label}.", filename=filename) + raise FortranParseError( + f"Duplicate symbol '{key}' in {label}.", + filename=filename, + code="PARSE_DUPLICATE_SYMBOL", + ) scope[key] = value # ------------------------------------------------------------------ @@ -2531,7 +2547,11 @@ def _helper_visit_module_like_spec_line( """ target = scope.model if target is None: # pragma: no cover - internal helper misuse. - raise FortranParseError("Module-like specification scope is missing a target model.", filename=filename) + raise FortranParseError( + "Module-like specification scope is missing a target model.", + filename=filename, + code="PARSE_INTERNAL_STATE", + ) stripped = line.strip() lower = stripped.lower() @@ -2542,6 +2562,7 @@ def _helper_visit_module_like_spec_line( filename=filename, line_number=lineno, source_line=source_line, + code="PARSE_UNSUPPORTED_OPENMP_DIRECTIVE", ) if scope.kind == "module": @@ -2565,6 +2586,7 @@ def _helper_visit_module_like_spec_line( filename=filename, line_number=lineno, source_line=source_line, + code="PARSE_MISSING_DERIVED_TYPE_END", ) if "::" in stripped: @@ -2594,6 +2616,7 @@ def _helper_visit_module_like_spec_line( filename=filename, line_number=lineno, source_line=source_line, + code="PARSE_EXECUTABLE_IN_SPECIFICATION", ) return @@ -2622,6 +2645,7 @@ def _helper_visit_module_like_spec_line( filename=filename, line_number=lineno, source_line=source_line, + code="PARSE_UNSUPPORTED_DECLARATION", ) def _helper_visit_procedure_spec_line(self, line: str, proc_state: dict, filename: str | None = None, lineno: int | None = None, source_line: str | None = None) -> None: @@ -2646,6 +2670,7 @@ def _helper_visit_procedure_spec_line(self, line: str, proc_state: dict, filenam filename=filename, line_number=lineno, source_line=source_line, + code="PARSE_UNSUPPORTED_OPENMP_DIRECTIVE", ) if self._handle_proc_implicit_line(stripped, proc_state): return @@ -2713,7 +2738,11 @@ def _helper_visit_type_spec_line(self, line: str, scope: _ParserScope, filename: """ dtype = scope.model if dtype is None: # pragma: no cover - internal helper misuse. - raise FortranParseError("Derived-type specification scope is missing a target model.", filename=filename) + raise FortranParseError( + "Derived-type specification scope is missing a target model.", + filename=filename, + code="PARSE_INTERNAL_STATE", + ) stripped = line.strip() if re.match(r"^type\s*::\s*\w+$", stripped, re.IGNORECASE): raise FortranParseError( @@ -2721,6 +2750,7 @@ def _helper_visit_type_spec_line(self, line: str, scope: _ParserScope, filename: filename=filename, line_number=lineno, source_line=source_line, + code="PARSE_MISSING_DERIVED_TYPE_END", ) if stripped.lower() in {"sequence", "private"}: return @@ -2730,6 +2760,7 @@ def _helper_visit_type_spec_line(self, line: str, scope: _ParserScope, filename: filename=filename, line_number=lineno, source_line=source_line, + code="PARSE_UNSUPPORTED_OPENMP_DIRECTIVE", ) parsed = self._helper_parse_declaration_line( stripped, @@ -2755,6 +2786,7 @@ def _helper_visit_type_spec_line(self, line: str, scope: _ParserScope, filename: filename=filename, line_number=lineno, source_line=source_line, + code="PARSE_UNSUPPORTED_DECLARATION", ) def _parse_derived_type_contains_line( @@ -2793,6 +2825,7 @@ def _parse_derived_type_contains_line( filename=filename, line_number=lineno, source_line=source_line, + code="PARSE_UNSUPPORTED_TYPE_BOUND_DECLARATION", ) def _helper_apply_local_interface_declarations( @@ -2991,7 +3024,11 @@ def _helper_push_declaration_to_scope( if role == "procedure_symbol": proc_state = scope.state if proc_state is None: # pragma: no cover - internal helper misuse. - raise FortranParseError("Procedure declaration scope is missing state.", filename=filename) + raise FortranParseError( + "Procedure declaration scope is missing state.", + filename=filename, + code="PARSE_INTERNAL_STATE", + ) if meta["base_type"] == "procedure" and meta["kind"] in proc_state.get("imports", set()): meta["kind"] = None for entity in split_csv(right): @@ -3019,7 +3056,11 @@ def _helper_push_declaration_to_scope( target = scope.model if target is None: # pragma: no cover - internal helper misuse. - raise FortranParseError("Declaration scope is missing a target model.", filename=filename) + raise FortranParseError( + "Declaration scope is missing a target model.", + filename=filename, + code="PARSE_INTERNAL_STATE", + ) for entity in split_csv(right): initializer = entity.split("=", 1)[1].strip() if "=" in entity else None @@ -3347,6 +3388,7 @@ def _handle_unknown_proc_declaration( filename=filename, line_number=lineno, source_line=source_line, + code="PARSE_UNSUPPORTED_DECLARATION", ) # ------------------------------------------------------------------ @@ -3385,6 +3427,7 @@ def _finalize_proc(self, state: dict) -> FortranProcedureSignature: raise FortranParseError( f"Failed to resolve declared argument '{arg.name}' in procedure '{sig.name}'.", filename=filename, + code="PARSE_UNRESOLVED_ARGUMENT_TYPE", ) local_resolver = _CompileTimeResolver(local_params) for arg in sig.arguments: @@ -3438,6 +3481,7 @@ def _finalize_proc(self, state: dict) -> FortranProcedureSignature: raise FortranParseError( f"Unknown datatype for function result '{sig.result.name}' in procedure '{sig.name}'.", filename=filename, + code="PARSE_UNKNOWN_FUNCTION_RESULT_TYPE", ) if sig.kind == "function": self._validate_function_result(sig, filename) @@ -3455,16 +3499,19 @@ def _validate_all_args_declared(sig: FortranProcedureSignature, filename: str | raise FortranParseError( f"Argument '{arg.name}' in procedure '{sig.name}' has no type declaration (implicit none is active).", filename=filename, + code="PARSE_IMPLICIT_NONE_UNDECLARED_SYMBOL", ) if sig.kind == "function" and sig.result and sig.result.base_type == "unknown": if explicit_result: raise FortranParseError( f"Unknown datatype for function result '{sig.result.name}' in procedure '{sig.name}'.", filename=filename, + code="PARSE_UNKNOWN_FUNCTION_RESULT_TYPE", ) raise FortranParseError( f"Function result '{sig.result.name}' in procedure '{sig.name}' has no type declaration (implicit none is active).", filename=filename, + code="PARSE_IMPLICIT_NONE_UNDECLARED_SYMBOL", ) @staticmethod @@ -3473,6 +3520,7 @@ def _validate_function_result(sig: FortranProcedureSignature, filename: str | No raise FortranParseError( f"Function '{sig.name}' has no result variable.", filename=filename, + code="PARSE_MISSING_FUNCTION_RESULT", ) result_name = sig.result.name.lower() func_name = sig.name.lower() @@ -3481,6 +3529,7 @@ def _validate_function_result(sig: FortranProcedureSignature, filename: str | No raise FortranParseError( f"Function result variable '{sig.result.name}' in function '{sig.name}' shadows an argument name.", filename=filename, + code="PARSE_RESULT_SHADOWS_ARGUMENT", ) @staticmethod @@ -3508,6 +3557,7 @@ def _validate_variable_declarations( raise FortranParseError( f"Duplicate variable '{var.name}' in {owner_kind} '{display_name}'.", filename=filename, + code="PARSE_DUPLICATE_VARIABLE", ) continue # pragma: no cover - exact duplicate declarations are invalid Fortran and tolerated defensively. seen[key] = var @@ -3548,6 +3598,7 @@ def _apply_module_visibility(module: FortranModule, filename: str | None) -> Non raise FortranParseError( f"Unknown type for variable '{var.name}' in module '{module.name}'.", filename=filename, + code="PARSE_UNKNOWN_VARIABLE_TYPE", ) @staticmethod @@ -3558,12 +3609,14 @@ def _validate_derived_type_fields(dtype: FortranDerivedType, filename: str | Non raise FortranParseError( f"Duplicate field '{f.name}' in derived type '{dtype.name}'.", filename=filename, + code="PARSE_DUPLICATE_FIELD", ) seen.add(f.name.lower()) if f.base_type == "unknown": # pragma: no cover - unknown type fields raise before finalization. raise FortranParseError( f"Unknown type for field '{f.name}' in derived type '{dtype.name}'.", filename=filename, + code="PARSE_UNKNOWN_FIELD_TYPE", ) @staticmethod @@ -3583,6 +3636,7 @@ def _validate_no_duplicate_arg_names( filename=filename, line_number=line_number, source_line=source_line, + code="PARSE_DUPLICATE_ARGUMENT", ) seen.add(key) diff --git a/tests/parser/c/fixtures/errors/invalid_type_specifiers.h.json b/tests/parser/c/fixtures/errors/invalid_type_specifiers.h.json index a28fe9b60..b3cf99b38 100644 --- a/tests/parser/c/fixtures/errors/invalid_type_specifiers.h.json +++ b/tests/parser/c/fixtures/errors/invalid_type_specifiers.h.json @@ -5,7 +5,7 @@ "Invalid type specifier sequence 'unsigned float'." ], "diagnostic_contains": [ - "error[CPARSE003]", + "error[CPARSE_INVALID_SPECIFIER_SEQUENCE]", "Invalid type specifier sequence 'unsigned float'.", "unsigned float value;" ] diff --git a/tests/parser/c/test_c_cli_skeleton.py b/tests/parser/c/test_c_cli_skeleton.py index 13890273b..a98005f1d 100644 --- a/tests/parser/c/test_c_cli_skeleton.py +++ b/tests/parser/c/test_c_cli_skeleton.py @@ -296,7 +296,7 @@ def test_cli_c_invalid_primitive_specifier_sequence_is_fatal(tmp_path: Path): res = subprocess.run(cmd, capture_output=True, text=True) assert res.returncode == 1 - assert "error[CPARSE003]: Invalid type specifier sequence 'unsigned float'." in res.stderr + assert "error[CPARSE_INVALID_SPECIFIER_SEQUENCE]: Invalid type specifier sequence 'unsigned float'." in res.stderr assert "\x1b[" not in res.stderr diff --git a/tests/parser/c/test_c_declarations_and_declarators.py b/tests/parser/c/test_c_declarations_and_declarators.py index f9b291233..711da5878 100644 --- a/tests/parser/c/test_c_declarations_and_declarators.py +++ b/tests/parser/c/test_c_declarations_and_declarators.py @@ -107,9 +107,9 @@ def test_invalid_primitive_specifier_sequences_raise_parse_errors(source, expect with pytest.raises(CParseError, match="Invalid type specifier sequence") as error: parse_c_file(source, filename="invalid_specifiers.h") - assert error.value.code == "CPARSE003" + assert error.value.code == "CPARSE_INVALID_SPECIFIER_SEQUENCE" assert ( - f"invalid_specifiers.h:1:{expected_column}: error[CPARSE003]" + f"invalid_specifiers.h:1:{expected_column}: error[CPARSE_INVALID_SPECIFIER_SEQUENCE]" in error.value.format_diagnostic(color=False) ) diff --git a/tests/parser/c/test_c_public_api_skeleton.py b/tests/parser/c/test_c_public_api_skeleton.py index b146871dd..6e6fc42b6 100644 --- a/tests/parser/c/test_c_public_api_skeleton.py +++ b/tests/parser/c/test_c_public_api_skeleton.py @@ -255,10 +255,10 @@ def test_c_parse_error_attributes_and_diagnostic_formatting(): assert err.line_number == 2 assert err.column == 5 assert err.base_message == "unexpected token" - assert err.code == "CPARSE001" + assert err.code == "CPARSE_ERROR" diagnostic = err.format_diagnostic(color=False, debug=True) - assert "bad.h:2:5: error[CPARSE001]: unexpected token" in diagnostic + assert "bad.h:2:5: error[CPARSE_ERROR]: unexpected token" in diagnostic assert "2 | int broken(;" in diagnostic assert "note: parser raised at" in diagnostic diff --git a/tests/parser/fortran/fixtures/errors/err_duplicate_argument_name.json b/tests/parser/fortran/fixtures/errors/err_duplicate_argument_name.json index f5a8a7ece..ec6dbdc14 100644 --- a/tests/parser/fortran/fixtures/errors/err_duplicate_argument_name.json +++ b/tests/parser/fortran/fixtures/errors/err_duplicate_argument_name.json @@ -5,7 +5,7 @@ "Duplicate argument name 'x' in procedure 'dup'." ], "diagnostic_contains": [ - "error[PARSE001]", + "error[PARSE_DUPLICATE_ARGUMENT]", "Duplicate argument name 'x' in procedure 'dup'.", "subroutine dup(x, y, x)" ] diff --git a/tests/parser/fortran/fixtures/errors/err_duplicate_declaration_procedure.json b/tests/parser/fortran/fixtures/errors/err_duplicate_declaration_procedure.json index 2cec23e8d..73cb9eee5 100644 --- a/tests/parser/fortran/fixtures/errors/err_duplicate_declaration_procedure.json +++ b/tests/parser/fortran/fixtures/errors/err_duplicate_declaration_procedure.json @@ -5,7 +5,7 @@ "Duplicate declaration of symbol 'x' in procedure 'dup'." ], "diagnostic_contains": [ - "error[PARSE001]", + "error[PARSE_DUPLICATE_DECLARATION]", "Duplicate declaration of symbol 'x' in procedure 'dup'.", "integer :: x" ] diff --git a/tests/parser/fortran/fixtures/errors/err_duplicate_field_derived_type.json b/tests/parser/fortran/fixtures/errors/err_duplicate_field_derived_type.json index a5256a829..d005c62a3 100644 --- a/tests/parser/fortran/fixtures/errors/err_duplicate_field_derived_type.json +++ b/tests/parser/fortran/fixtures/errors/err_duplicate_field_derived_type.json @@ -5,7 +5,7 @@ "Duplicate field 'x' in derived type 'point'." ], "diagnostic_contains": [ - "error[PARSE001]", + "error[PARSE_DUPLICATE_FIELD]", "Duplicate field 'x' in derived type 'point'.", "" ] diff --git a/tests/parser/fortran/fixtures/errors/err_duplicate_parameter.json b/tests/parser/fortran/fixtures/errors/err_duplicate_parameter.json index 173ff80f6..3a49cbc84 100644 --- a/tests/parser/fortran/fixtures/errors/err_duplicate_parameter.json +++ b/tests/parser/fortran/fixtures/errors/err_duplicate_parameter.json @@ -5,7 +5,7 @@ "Duplicate PARAMETER declaration of symbol 'n' in procedure 'dup_param'." ], "diagnostic_contains": [ - "error[PARSE001]", + "error[PARSE_DUPLICATE_PARAMETER]", "Duplicate PARAMETER declaration of symbol 'n' in procedure 'dup_param'.", "integer, parameter :: n = 10" ] diff --git a/tests/parser/fortran/fixtures/errors/err_duplicate_procedure_global.json b/tests/parser/fortran/fixtures/errors/err_duplicate_procedure_global.json index 69e06d29b..9f036d8b8 100644 --- a/tests/parser/fortran/fixtures/errors/err_duplicate_procedure_global.json +++ b/tests/parser/fortran/fixtures/errors/err_duplicate_procedure_global.json @@ -5,7 +5,7 @@ "Duplicate procedure name 'work' in global scope." ], "diagnostic_contains": [ - "error[PARSE001]", + "error[PARSE_DUPLICATE_PROCEDURE]", "Duplicate procedure name 'work' in global scope.", "subroutine work(n)" ] diff --git a/tests/parser/fortran/fixtures/errors/err_duplicate_procedure_module.json b/tests/parser/fortran/fixtures/errors/err_duplicate_procedure_module.json index e925f29b4..c2cc637c7 100644 --- a/tests/parser/fortran/fixtures/errors/err_duplicate_procedure_module.json +++ b/tests/parser/fortran/fixtures/errors/err_duplicate_procedure_module.json @@ -5,7 +5,7 @@ "Duplicate procedure name 'work' in module 'm'." ], "diagnostic_contains": [ - "error[PARSE001]", + "error[PARSE_DUPLICATE_PROCEDURE]", "Duplicate procedure name 'work' in module 'm'.", "subroutine work(n)" ] diff --git a/tests/parser/fortran/fixtures/errors/err_duplicate_variable_module.json b/tests/parser/fortran/fixtures/errors/err_duplicate_variable_module.json index e401d75fb..4096848cc 100644 --- a/tests/parser/fortran/fixtures/errors/err_duplicate_variable_module.json +++ b/tests/parser/fortran/fixtures/errors/err_duplicate_variable_module.json @@ -5,7 +5,7 @@ "Duplicate variable 'n' in module 'm'." ], "diagnostic_contains": [ - "error[PARSE001]", + "error[PARSE_DUPLICATE_VARIABLE]", "Duplicate variable 'n' in module 'm'.", "" ] diff --git a/tests/parser/fortran/fixtures/errors/err_implicit_none_undeclared_arg.json b/tests/parser/fortran/fixtures/errors/err_implicit_none_undeclared_arg.json index 8441129c6..e6fac08b3 100644 --- a/tests/parser/fortran/fixtures/errors/err_implicit_none_undeclared_arg.json +++ b/tests/parser/fortran/fixtures/errors/err_implicit_none_undeclared_arg.json @@ -5,7 +5,7 @@ "Argument 'y' in procedure 'foo' has no type declaration (implicit none is active)." ], "diagnostic_contains": [ - "error[PARSE001]", + "error[PARSE_IMPLICIT_NONE_UNDECLARED_SYMBOL]", "Argument 'y' in procedure 'foo' has no type declaration (implicit none is active).", "" ] diff --git a/tests/parser/fortran/fixtures/errors/err_implicit_none_undeclared_result.json b/tests/parser/fortran/fixtures/errors/err_implicit_none_undeclared_result.json index 4b8f31d49..4a55be7c6 100644 --- a/tests/parser/fortran/fixtures/errors/err_implicit_none_undeclared_result.json +++ b/tests/parser/fortran/fixtures/errors/err_implicit_none_undeclared_result.json @@ -5,7 +5,7 @@ "Function result 'f' in procedure 'f' has no type declaration (implicit none is active)." ], "diagnostic_contains": [ - "error[PARSE001]", + "error[PARSE_IMPLICIT_NONE_UNDECLARED_SYMBOL]", "Function result 'f' in procedure 'f' has no type declaration (implicit none is active).", "" ] diff --git a/tests/parser/fortran/fixtures/errors/err_parameter_without_type_implicit_none.json b/tests/parser/fortran/fixtures/errors/err_parameter_without_type_implicit_none.json index 5c31dc38b..341e701cc 100644 --- a/tests/parser/fortran/fixtures/errors/err_parameter_without_type_implicit_none.json +++ b/tests/parser/fortran/fixtures/errors/err_parameter_without_type_implicit_none.json @@ -5,7 +5,7 @@ "Unknown datatype for PARAMETER symbol 'zero' in procedure 'cst'." ], "diagnostic_contains": [ - "error[PARSE001]", + "error[PARSE_UNKNOWN_PARAMETER_TYPE]", "Unknown datatype for PARAMETER symbol 'zero' in procedure 'cst'.", "parameter ( zero = 0.0e+0 )" ] diff --git a/tests/parser/fortran/fixtures/errors/err_result_shadows_argument.json b/tests/parser/fortran/fixtures/errors/err_result_shadows_argument.json index 80e96b063..c3fabbb38 100644 --- a/tests/parser/fortran/fixtures/errors/err_result_shadows_argument.json +++ b/tests/parser/fortran/fixtures/errors/err_result_shadows_argument.json @@ -5,7 +5,7 @@ "Function result variable 'res' in function 'f' shadows an argument name." ], "diagnostic_contains": [ - "error[PARSE001]", + "error[PARSE_RESULT_SHADOWS_ARGUMENT]", "Function result variable 'res' in function 'f' shadows an argument name.", "" ] diff --git a/tests/parser/fortran/fixtures/errors/err_unknown_function_result.json b/tests/parser/fortran/fixtures/errors/err_unknown_function_result.json index a2081b6ee..1d087b74c 100644 --- a/tests/parser/fortran/fixtures/errors/err_unknown_function_result.json +++ b/tests/parser/fortran/fixtures/errors/err_unknown_function_result.json @@ -5,7 +5,7 @@ "Unknown datatype for function result 'res' in procedure 'f'." ], "diagnostic_contains": [ - "error[PARSE001]", + "error[PARSE_UNKNOWN_FUNCTION_RESULT_TYPE]", "Unknown datatype for function result 'res' in procedure 'f'.", "" ] diff --git a/tests/parser/fortran/fixtures/errors/err_unknown_type_derived_type.json b/tests/parser/fortran/fixtures/errors/err_unknown_type_derived_type.json index d6b483be3..b35d27028 100644 --- a/tests/parser/fortran/fixtures/errors/err_unknown_type_derived_type.json +++ b/tests/parser/fortran/fixtures/errors/err_unknown_type_derived_type.json @@ -5,7 +5,7 @@ "Unknown or unsupported datatype declaration in type 't': weirdtype :: x" ], "diagnostic_contains": [ - "error[PARSE001]", + "error[PARSE_UNSUPPORTED_DECLARATION]", "Unknown or unsupported datatype declaration in type 't': weirdtype :: x", "weirdtype :: x" ] diff --git a/tests/parser/fortran/fixtures/errors/err_unknown_type_module.json b/tests/parser/fortran/fixtures/errors/err_unknown_type_module.json index 1a038c85f..ec47a6201 100644 --- a/tests/parser/fortran/fixtures/errors/err_unknown_type_module.json +++ b/tests/parser/fortran/fixtures/errors/err_unknown_type_module.json @@ -5,7 +5,7 @@ "Unknown or unsupported datatype declaration in module 'm': weirdtype :: x" ], "diagnostic_contains": [ - "error[PARSE001]", + "error[PARSE_UNSUPPORTED_DECLARATION]", "Unknown or unsupported datatype declaration in module 'm': weirdtype :: x", "weirdtype :: x" ] diff --git a/tests/parser/fortran/fixtures/errors/err_unknown_type_procedure.json b/tests/parser/fortran/fixtures/errors/err_unknown_type_procedure.json index ca2b86795..6dccf0f9e 100644 --- a/tests/parser/fortran/fixtures/errors/err_unknown_type_procedure.json +++ b/tests/parser/fortran/fixtures/errors/err_unknown_type_procedure.json @@ -5,7 +5,7 @@ "Unknown or unsupported datatype declaration for procedure 'bad': weirdtype :: x" ], "diagnostic_contains": [ - "error[PARSE001]", + "error[PARSE_UNSUPPORTED_DECLARATION]", "Unknown or unsupported datatype declaration for procedure 'bad': weirdtype :: x", "weirdtype :: x" ] diff --git a/tests/parser/test_cli.py b/tests/parser/test_cli.py index f292eb796..363637896 100644 --- a/tests/parser/test_cli.py +++ b/tests/parser/test_cli.py @@ -189,7 +189,7 @@ def test_cli_formats_parse_errors_without_traceback(tmp_path: Path): assert res.returncode == 1 assert res.stdout == "" assert "Traceback" not in res.stderr - assert f"{f90}:2:1: error[PARSE001]:" in res.stderr + assert f"{f90}:2:1: error[PARSE_UNSUPPORTED_DECLARATION]:" in res.stderr assert "2 | weirdtype :: x" in res.stderr @@ -273,7 +273,7 @@ def test_cli_no_color_env_disables_default_ansi(tmp_path: Path): assert res.returncode == 1 assert "\033[" not in res.stderr - assert f"{f90}:2:1: error[PARSE001]:" in res.stderr + assert f"{f90}:2:1: error[PARSE_UNSUPPORTED_DECLARATION]:" in res.stderr @@ -563,7 +563,7 @@ def test_cli_fortran_rejects_embedded_c_declaration_outside_execution_body(tmp_p ) assert result.returncode == 1 - assert "PARSE001" in result.stderr + assert "PARSE_UNSUPPORTED_DECLARATION" in result.stderr assert "Unknown or unsupported datatype declaration" in result.stderr diff --git a/tests/parser/test_error_handling.py b/tests/parser/test_error_handling.py index 4c1ca5cba..f36e50c99 100644 --- a/tests/parser/test_error_handling.py +++ b/tests/parser/test_error_handling.py @@ -34,7 +34,7 @@ def test_parse_error_message_includes_filename_and_lineno(): parse_fortran_file(code, filename="myfile.f90") msg = str(exc_info.value) assert "myfile.f90:3:1" in msg - assert "error[PARSE001]" in msg + assert "error[PARSE_UNSUPPORTED_DECLARATION]" in msg def test_parse_error_message_includes_source_line(): @@ -73,7 +73,7 @@ def test_parse_error_formats_compiler_style_diagnostic(): parse_fortran_file(code, filename="myfile.f90") diagnostic = exc_info.value.format_diagnostic(color=False) - assert "myfile.f90:3:1: error[PARSE001]:" in diagnostic + assert "myfile.f90:3:1: error[PARSE_UNSUPPORTED_DECLARATION]:" in diagnostic assert "Unknown or unsupported datatype" in diagnostic assert "3 | weirdtype :: x" in diagnostic assert "| ^" in diagnostic @@ -779,7 +779,7 @@ def test_fortran_parser_rejects_invalid_non_fortran_syntax_outside_execution_bod ) as exc_info: parse_fortran_file(code, filename="mixed.f90") - assert exc_info.value.code in {"PARSE_INVALID_SYNTAX", "PARSE001"} + assert exc_info.value.code in {"PARSE_INVALID_SYNTAX", "PARSE_UNSUPPORTED_DECLARATION"} def test_fortran_parser_ignores_non_fortran_syntax_after_execution_boundary(): From 2a76e4f663fa241835b9faa30d07f8b84ac22790 Mon Sep 17 00:00:00 2001 From: said Date: Sun, 31 May 2026 07:02:15 +0100 Subject: [PATCH 07/13] update the preprocessing --- README.md | 52 +- c_parser/cli.py | 36 +- docs/c_parser/c_parser_architecture.md | 6 +- docs/c_parser/c_parser_cli_workflow.md | 71 +- docs/c_parser/c_parser_reference.md | 7 +- docs/diagnostic_codes.md | 16 + docs/fortran/fortran_parser.md | 24 +- .../parser_implementation_reference.md | 24 +- fortran_parser/parser.py | 104 +- semantics/c2ir.py | 42 + tests/parser/test_error_handling.py | 12 +- tests/parser/test_preprocessing_cli.py | 299 +++- ...t_preprocessor_and_execution_boundaries.py | 21 +- .../parser/test_procedure_and_type_parsing.py | 9 +- tests/parser/test_scope_handling.py | 2 +- tests/semantics/test_c2ir.py | 31 + x2py/c_type_probe.py | 4 +- x2py/cli.py | 81 +- x2py/fortran_type_probe.py | 4 +- x2py/preprocessing.py | 1310 +++++++++++++++-- x2py/preprocessing_test_example.md | 190 +-- 21 files changed, 1848 insertions(+), 497 deletions(-) diff --git a/README.md b/README.md index f9ad1c663..8e855b8d7 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,8 @@ Current handled coverage: The C frontend is currently parse-only. It supports: - Raw-source directive metadata for includes, simple macros, conditionals, and - pragmas. + pragmas. Raw mode records these facts but does not expand macros or select + conditional branches. - Compiler-assisted preprocessing through the shared CLI flags, with `#line` and GCC/Clang linemarker remapping back to original source locations. - Top-level variables, typedefs, function declarations/definitions, structs, @@ -66,8 +67,9 @@ The C frontend is currently parse-only. It supports: - Project include/index facts through `parse_c_project(...)`, with includes recorded non-recursively: only explicitly supplied files or files below an explicitly supplied directory are parsed. -- Raw mutually exclusive function alternatives preserved for later semantic - selection rather than collapsed into one signature. +- Compiler mode is the wrapper-facing path for macro-dependent APIs: it parses + one compiler-expanded translation unit and keeps mutually exclusive branches + separate across build configurations. The supported C subset continues through semantic IR conversion, `.pyi` generation, and wrap-readiness. @@ -152,7 +154,15 @@ python -m x2py path/to/c_src --language c --parse Fortran directories scan `.f`, `.for`, `.ftn`, `.f90`, `.f95`, `.f03`, `.f08`; C directories scan `.c`, `.h`, and `.i` files. -### Compiler preprocessing and target probes +### Compiler preprocessing, includes, and target probes + +Wrapper-facing source parsing should use compiler preprocessing whenever the +input contains C/CPP preprocessing. The selected compiler is authoritative for +macro expansion, `#if`/`#ifdef` branch selection, C `#include`, Fortran CPP +`#include`, predefined macros, `-D`/`-U`, include paths, target flags, and +sysroot behavior. Internal parser mode remains available for plain source, +already-preprocessed source, and focused parser tests; it does not evaluate CPP +branches. The shared compiler mode is: @@ -168,9 +178,37 @@ python -m x2py path/to/source.f90 --language fortran --parse \ ``` For C, `--language c --preprocess compiler` runs the exact compiler -preprocessor and parses stdout. C also supports `--compile-commands -build/compile_commands.json`; the matching entry supplies the compiler and -project flags. +preprocessor and parses stdout. C and Fortran can use `--compile-commands +build/compile_commands.json` when a matching entry supplies the compiler and +project flags. GCC-compatible C/Clang invocations use `-E -x c`; GNU Fortran +invocations use `-E -cpp`. Linemarkers are preserved so parser locations can be +mapped back to original files. For unsupported compiler families, use +`--preprocessor-adapter command-template --preprocess-template '...'`; the +minimum adapter contract is expanded source on stdout. + +Fortran native `include "file.inc"` is resolved after compiler CPP output and +before parsing. This is textual insertion into the current module, procedure, +interface, or execution scope; it is not the same as `use module_name`. Native +includes are resolved relative to the including file first, then configured +`-I` directories, and duplicate textual inclusion is preserved. Missing +includes and cycles are reported as preprocessing diagnostics. + +Preprocessing JSON records the exact recipe: compiler or adapter, argv, working +directory, include directories, defines, undefs, standard, extra compiler +arguments, included files, source mappings, diagnostics, and optional macro +metadata when the adapter output exposes it. System-header declarations are +classified private by default. Reachable project includes are public by +default; use `--include-exposure roots-only`, `--public-include`, and +`--private-include` to control wrapper export. Private declarations remain +available internally for type resolution. Public signatures that refer to +private C handle types can use private opaque classes rather than exposing data +members. + +Preprocessing failures print explicit categories such as +`PREPROCESSOR_NOT_FOUND`, `PREPROCESSOR_FAILED`, +`INVALID_COMPILER_ARGUMENTS`, `UNSUPPORTED_COMPILER_CAPABILITY`, +`PROVENANCE_UNAVAILABLE`, `INCLUDE_NOT_FOUND`, and `INCLUDE_CYCLE` without a +Python traceback. Pass `--debug` to re-raise and show the traceback. Target-dependent type facts are not hard-coded. They are probed with the same compiler path and target-relevant flags because results may change with ABI, diff --git a/c_parser/cli.py b/c_parser/cli.py index 7134bcfb6..3a333d31e 100644 --- a/c_parser/cli.py +++ b/c_parser/cli.py @@ -9,7 +9,7 @@ from pathlib import Path from typing import Any -from .models import CFile, CParseError, c_model_to_dict +from .models import CFile, CMacro, CParseError, CSourceLocation, c_model_to_dict from .parser import CParser @@ -44,6 +44,38 @@ def expand_c_paths(paths: list[str]) -> list[Path]: return sorted(set(expanded)) +def attach_preprocessing_recipe(parsed: CFile, preprocessing_recipe: dict[str, Any] | None) -> None: + """Attach compiler recipe side-channel facts to a parsed C file.""" + + parsed.preprocessing_recipe = preprocessing_recipe + if not preprocessing_recipe: + return + existing = {(macro.name, macro.source_location.filename if macro.source_location else None, macro.source_location.line if macro.source_location else None) for macro in parsed.macros} + for item in preprocessing_recipe.get("macros") or []: + if not isinstance(item, dict): + continue + name = item.get("name") + if not isinstance(name, str) or not name: + continue + location = CSourceLocation( + filename=item.get("path") if isinstance(item.get("path"), str) else None, + line=item.get("line") if isinstance(item.get("line"), int) else None, + column=1, + ) + key = (name, location.filename, location.line) + if key in existing: + continue + parsed.macros.append( + CMacro( + name=name, + value=item.get("value") if isinstance(item.get("value"), str) else None, + function_like=bool(item.get("function_like")), + source_location=location, + ) + ) + existing.add(key) + + def parse_c_report( paths: list[str], *, @@ -70,7 +102,7 @@ def parse_c_report( include_dirs=include_dirs, preprocessing=preprocessing, ) - parsed.preprocessing_recipe = preprocessing_recipe + attach_preprocessing_recipe(parsed, preprocessing_recipe) out[str(p)] = c_model_to_dict(parsed) return out diff --git a/docs/c_parser/c_parser_architecture.md b/docs/c_parser/c_parser_architecture.md index f72896544..4a39acfa3 100644 --- a/docs/c_parser/c_parser_architecture.md +++ b/docs/c_parser/c_parser_architecture.md @@ -323,9 +323,9 @@ parse_c_file(source_or_path, filename=None, macro_defines=None, include_dirs=Non parse_c_project(files, include_dirs=None, macro_defines=None, preprocessing="raw", encoding="utf-8") -> CProject ``` -`macro_defines` is reserved for future compiler-assisted preprocessing -configuration. It must not cause raw mode to evaluate C preprocessor -conditionals or expand macros inside x2py. +`macro_defines` is accepted for API compatibility only. Raw mode must not +evaluate C preprocessor conditionals or expand macros inside x2py. Compiler +mode receives already-expanded source from the shared preprocessing layer. Implemented companion class: diff --git a/docs/c_parser/c_parser_cli_workflow.md b/docs/c_parser/c_parser_cli_workflow.md index ffff2a2c0..4ee7ed677 100644 --- a/docs/c_parser/c_parser_cli_workflow.md +++ b/docs/c_parser/c_parser_cli_workflow.md @@ -208,24 +208,53 @@ Implemented shared preprocessing flags: --preprocess {internal,compiler} --compiler EXACT_EXECUTABLE --compile-commands PATH +--preprocessor-adapter {auto,gcc-compatible-c,gnu-fortran,command-template} +--preprocess-template TEMPLATE -I DIR / --include-dir DIR -D NAME[=VALUE] / --define NAME[=VALUE] -U NAME / --undef NAME --std STANDARD --compiler-arg ARG +--include-exposure {reachable-project,roots-only} +--public-include PATH_OR_PATTERN +--private-include PATH_OR_PATTERN ``` `--preprocess internal` is the default. For C it means the current raw directive metadata mode: x2py reads `.h`/`.c` input directly, records includes and macros from the file, parses ordinary visible declarations, and does not -expand macros or select `#if` branches. For Fortran it means the existing -internal preprocessing path, including simple macro branch selection through -`-D` and `-U`. +expand macros or select `#if` branches. For Fortran it means plain or +already-preprocessed parser input; internal mode does not evaluate CPP +branches through `-D` or `-U`. `--preprocess compiler` means x2py runs an external compiler/preprocessor and parses stdout. The user must pass an exact compiler executable with -`--compiler`, unless a C `--compile-commands` entry supplies it. Do not rely on -generic defaults when multiple compiler versions are installed. +`--compiler`, unless a `--compile-commands` entry or custom command template +supplies it. Do not rely on generic defaults when multiple compiler versions +are installed. GCC-compatible C and Clang use `-E -x c`; GNU Fortran uses +`-E -cpp`. Linemarkers are preserved so parser diagnostics can report original +source files after preprocessing. + +The compiler is authoritative for macro expansion, conditional branch +selection, C and Fortran CPP includes, predefined macros, command-line macro +flags, include paths, target flags, and sysroot behavior. Multiple build +configurations must be preprocessed and parsed separately. x2py does not merge +mutually exclusive CPP branches into one parser model. + +Fortran native `include "file.inc"` is handled after compiler CPP output +because GNU Fortran does not preprocess the contents of native INCLUDE files. +The preprocessing layer resolves these includes relative to the including file, +then `-I` directories, preserves duplicate textual inclusion, and reports +`INCLUDE_NOT_FOUND` or `INCLUDE_CYCLE` explicitly. + +Compiler preprocessing records a recipe with the adapter, compiler argv, +working directory, include directories, defines, undefs, standard, raw compiler +arguments, source mappings, included files, diagnostics, and optional macro +metadata. System headers are recorded but private by default. Reachable project +includes are public by default; `--include-exposure roots-only`, +`--public-include`, and `--private-include` control public wrapper export while +keeping private declarations available for type resolution. Private C handle +types may be emitted as opaque classes in `.pyi` output. Before this mechanism is treated as portable across supported toolchains, it needs substantial integration testing with multiple C and Fortran compiler @@ -294,16 +323,17 @@ python -m x2py include/vendor_api.h --language c --parse \ --compiler /opt/intel/oneapi/compiler/latest/bin/icx \ -D VENDOR_PUBLIC= -# C, project build database. The compiler and most flags come from the matching -# compile_commands.json entry for the input file. +# C or Fortran, project build database. The compiler and most flags come from +# the matching compile_commands.json entry for the input file. python -m x2py src/api.c --language c --parse \ --preprocess compiler \ --compile-commands build/compile_commands.json -# Fortran, current internal branch selection. -python -m x2py src/solver.F90 --parse \ - -D USE_MPI \ - -U DEBUG +# Unsupported compiler family through a command template. +python -m x2py include/vendor_api.h --language c --parse \ + --preprocess compiler \ + --preprocessor-adapter command-template \ + --preprocess-template 'vendor-cc --preprocess {include_dirs} {defines} {source}' # Fortran, compiler-assisted preprocessing with an exact versioned executable. python -m x2py src/solver.F90 --parse \ @@ -326,25 +356,28 @@ Flag meanings: - `--compiler`: exact executable to run. Use `gcc-13`, `clang-18`, `/usr/bin/gfortran-12`, `/opt/.../ifx`, etc. x2py treats this as one argv item, not a shell command string. -- `--compile-commands`: C project database. x2py finds the entry for the input +- `--compile-commands`: project database. x2py finds the entry for the input file, strips compile-only flags such as `-c` and `-o`, adds `-E`, and uses that entry's compiler unless `--compiler` overrides it. +- `--preprocessor-adapter` / `--preprocess-template`: custom adapter path for + compiler families that do not match the GCC-compatible C or GNU Fortran + command shape. The command must write expanded source to stdout. - `-I` / `--include-dir`: include path passed as `-IDIR` in compiler mode. In C internal mode it is also used for quoted include resolution. - `-D` / `--define`: macro definition. `-D NAME` means `NAME=1`; `-D NAME=VALUE` preserves `VALUE`. -- `-U` / `--undef`: macro undefinition. In Fortran internal mode this selects - inactive branches for that macro. +- `-U` / `--undef`: macro undefinition passed to compiler preprocessing. - `--std`: language standard passed as `-std=STANDARD`, for example `c11`, `c23`, `f2008`, or `f2018`. - `--compiler-arg`: one raw compiler argument. For values beginning with `-`, use the equals form, for example `--compiler-arg=-target`. -`--define` and `--undef` belong to compiler-assisted preprocessing for C, not -to raw parser-side macro evaluation. Raw C mode records directives and parses -ordinary visible declarations only; it does not select `#if` branches or expand -macros. A declaration prefixed by an unexpanded object-like macro is deferred -as macro-dependent rather than treated as an invalid type sequence. +`--define` and `--undef` belong to compiler-assisted preprocessing for C and +Fortran, not to raw parser-side macro evaluation. Raw C mode records directives +and parses ordinary visible declarations only; it does not select `#if` +branches or expand macros. A declaration prefixed by an unexpanded object-like +macro is deferred as macro-dependent rather than treated as an invalid type +sequence. ## Current Partial Behavior diff --git a/docs/c_parser/c_parser_reference.md b/docs/c_parser/c_parser_reference.md index 3b154e6a4..e243088a7 100644 --- a/docs/c_parser/c_parser_reference.md +++ b/docs/c_parser/c_parser_reference.md @@ -470,9 +470,10 @@ such as `{"g1:b0"}` or `{"g1:b1"}`. A `CProject` stores such alternatives in unique `functions` entry. Compiler-preprocessed input contains the selected configuration and therefore does not need this ambiguity representation. -`macro_defines` is reserved for future compiler-assisted preprocessing -configuration. It must not mean that raw mode evaluates C preprocessor -conditionals or expands macros internally. +`macro_defines` is accepted for API compatibility only. It must not mean that +raw mode evaluates C preprocessor conditionals or expands macros internally. +Compiler mode should receive the already-expanded translation unit from +`x2py.preprocessing`. The parser itself should stay parse-only. If the C frontend later gains wrappability assessment, that should live in the semantic layer after C parser diff --git a/docs/diagnostic_codes.md b/docs/diagnostic_codes.md index 90dee7ffb..74b2f7290 100644 --- a/docs/diagnostic_codes.md +++ b/docs/diagnostic_codes.md @@ -50,6 +50,22 @@ traceback unless `--debug` is used. | `CPARSE_INVALID_SPECIFIER_SEQUENCE` | C | Invalid C primitive-specifier sequence. | | `CPARSE_INVALID_SYNTAX` | C | Syntax cannot be consumed in a modeled C grammar region. | +## Preprocessing Diagnostics + +Compiler-backed preprocessing failures are rendered by the CLI without a +Python traceback unless `--debug` is used. They occur before the parser consumes +the expanded source. + +| Code | Meaning | +| --- | --- | +| `PREPROCESSOR_NOT_FOUND` | The configured compiler/preprocessor executable could not be started. | +| `PREPROCESSOR_FAILED` | The compiler/preprocessor returned a non-zero status, timed out, or could not be executed. Compiler stderr is preserved. | +| `INVALID_COMPILER_ARGUMENTS` | The preprocessing configuration is invalid, such as a malformed macro name or unusable compile database entry. | +| `UNSUPPORTED_COMPILER_CAPABILITY` | The selected adapter was asked for metadata it cannot provide. | +| `PROVENANCE_UNAVAILABLE` | Expanded source was produced, but the adapter cannot provide accurate source mappings. | +| `INCLUDE_NOT_FOUND` | A native Fortran `include "..."` target could not be resolved or read. | +| `INCLUDE_CYCLE` | Recursive native Fortran INCLUDE expansion found a cycle. | + ## C Report Diagnostics The C parser can preserve partial metadata and attach `CDiagnostic` records. diff --git a/docs/fortran/fortran_parser.md b/docs/fortran/fortran_parser.md index 732bfc89e..b064ef6b3 100644 --- a/docs/fortran/fortran_parser.md +++ b/docs/fortran/fortran_parser.md @@ -100,7 +100,7 @@ sections so maintainers can navigate the file by concern instead of by history: - shared declaration parsing for module variables, program/block-data variables, procedure arguments/results, and derived-type fields - `_helper_*` methods for scoped parsing, expression resolution, - preprocessor branch selection, same-level duplicate checks, and shared + raw preprocessor branch structure, same-level duplicate checks, and shared specification-part collection - Thin module-level convenience wrappers that delegate to a shared parser instance @@ -370,12 +370,22 @@ Expected JSON layout: - `programs` - `block_data` -When `x2py --parse --json` applies Fortran preprocessing settings, the -per-file payload also contains `preprocessing_recipe`. Internal `-D`/`-U` -branch selection records those macro settings. `--preprocess compiler` records -the exact compiler executable and argv, include paths, macro flags, standard, -extra compiler arguments, and working directory used to produce the parsed -stdout stream. +When `x2py --parse --json` applies compiler preprocessing, the per-file payload +also contains `preprocessing_recipe`. Internal parser mode accepts plain or +already-preprocessed source and does not evaluate `-D`/`-U` CPP branches. +`--preprocess compiler` records the exact compiler executable or adapter, +argv, include paths, macro flags, standard, extra compiler arguments, working +directory, include graph, source mappings, diagnostics, and optional macro +metadata used to produce the parsed stdout stream. + +Fortran CPP directives are handled by the configured compiler. Native Fortran +`include "file.inc"` statements are then expanded recursively by the +preprocessing layer before the single parser pass. Native INCLUDE is textual +insertion into the current scope; it is not a `use` import from a separately +compiled module. Include lookup is relative to the including file first, then +the configured include directories, duplicate textual inclusion is preserved, +and missing files or cycles produce `INCLUDE_NOT_FOUND` or `INCLUDE_CYCLE` +diagnostics. `use` import shape: diff --git a/docs/fortran/parser_implementation_reference.md b/docs/fortran/parser_implementation_reference.md index 347cd983c..14889e98d 100644 --- a/docs/fortran/parser_implementation_reference.md +++ b/docs/fortran/parser_implementation_reference.md @@ -653,20 +653,22 @@ When updating parser behavior, keep this fail-fast contract aligned with tests: - "Unsupported but recognized" constructs may be carried far enough for semantic readiness or `.pyi` completion to decide wrappability. Unknown datatype syntax should crash early. -- **Preprocessor-conditional duplicate procedures (guarded allowance):** - - The parser does **not** run a full C preprocessor stage before parsing. - - While slicing source units, simple directive structure is tracked for - `#ifdef`, `#ifndef`, `#elif`, `#else`, and `#endif` to model - mutually-exclusive branches. - - `visit_file(..., macro_defines=...)` can provide macro decisions; inactive conditional branches are skipped before unit parsing so the active code path is selected. The module-level `parse_fortran_file(...)` convenience function delegates to this visitor. - - accepted forms: `set[str]` or `dict[str, int|bool|str]` - - dictionary values are truthy/falsey (`0`, `False`, `"0"`, `"false"` treated as undefined/disabled) - - Basic `#if` expressions are supported for branch selection (`defined(X)`, `!`, `&&`, `||`, parentheses, `0`/`1`). +- **Preprocessor-conditional duplicate procedures (raw parser tolerance):** + - The parser does **not** run a C preprocessor or evaluate CPP expressions. + Compiler preprocessing is responsible for `#if`, `#ifdef`, macro + expansion, and CPP includes before production wrapper parsing. + - While slicing raw source units, simple directive structure is tracked for + `#ifdef`, `#ifndef`, `#elif`, `#else`, and `#endif` only to recognize + mutually exclusive branches in unresolved raw input. + - `visit_file(..., macro_defines=...)` remains accepted for compatibility, + but macro decisions are ignored. Active branch selection must happen in the + compiler-backed preprocessing layer. - Duplicate procedure-name checks in a module/global scope are evaluated - against same-level sliced units and this branch context: + against same-level sliced units and this structural branch context: - if two same-name procedure headers are reachable in an overlapping branch context, raise `FortranParseError` (duplicate procedure name). - if they are only present in mutually-exclusive branches of the same conditional group, allow both signatures. - - This is a structural exclusivity model (branch groups), not semantic evaluation of macro expressions. In other words, branch mutual exclusivity is honored without requiring expression truth evaluation. + - This is not semantic macro evaluation. Multiple build configurations + should be preprocessed and parsed separately. `FortranParseError` is a subclass of `ValueError` and carries structured location metadata: - `filename` — source file path (if provided) diff --git a/fortran_parser/parser.py b/fortran_parser/parser.py index 69ec219ac..c4be1c83b 100644 --- a/fortran_parser/parser.py +++ b/fortran_parser/parser.py @@ -225,8 +225,8 @@ class FortranParser: """Stateful parser entrypoint and orchestration object. State carried on the instance: - - `macro_defines`: optional macro-selection configuration used while - selecting preprocessor branches before source-unit slicing. + - `macro_defines`: accepted for backward-compatible call signatures, but + CPP branch selection is handled by the compiler preprocessing layer. Parsing pipeline used by `visit_file`: 1. Preprocess source into normalized lines (`_preprocessed_lines`). @@ -272,7 +272,8 @@ class FortranParser: # ------------------------------------------------------------------ def __init__(self, macro_defines: set[str] | dict[str, int | bool | str] | None = None): - self.macro_defines = macro_defines + del macro_defines + self.macro_defines = None def visit_file( self, @@ -289,11 +290,10 @@ def visit_file( else: code = str(source_or_path) - effective_macro_defines = self.macro_defines if macro_defines is None else macro_defines + del macro_defines lines, root_scope, top_units = self._helper_prepare_source_units( code, filename, - macro_defines=effective_macro_defines, ) modules: list[FortranModule] = [] submodules: list[FortranSubmodule] = [] @@ -947,8 +947,8 @@ def _helper_prepare_source_units( `_SourceUnit` objects, one module unit and one procedure unit, both carrying original source line numbers. """ + del macro_defines lines = self._preprocessed_lines(code, filename) - lines = self._helper_select_active_preprocessor_lines(lines, macro_defines) root_scope = _ParserScope(kind="file", name=None) units = self._helper_slice_child_units(lines, parent_scope=root_scope, filename=filename) self._helper_validate_file_scope_unparsed_lines(lines, filename) @@ -1042,46 +1042,6 @@ def collect(scope: _ParserScope, child_units: list[_SourceUnit]) -> None: collect(root_scope, self._helper_slice_child_units(lines, parent_scope=root_scope, filename=filename)) return types - def _helper_select_active_preprocessor_lines( - self, - lines: _PreprocessedLines, - macro_defines: set[str] | dict[str, int | bool | str] | None, - ) -> _PreprocessedLines: - """Drop inactive preprocessor branches when macro selection is enabled. - - This runs before source-unit slicing, so duplicate declarations hidden - behind inactive ``#ifdef`` branches do not produce false same-scope - conflicts. - - Example: - With ``macro_defines={"USE_FAST"}``, only the active branch of - ``#ifdef USE_FAST`` is returned to `_helper_slice_child_units`; - with ``macro_defines=None``, all branches are preserved and their - condition sets are used for overlap-aware duplicate checks. - """ - if macro_defines is None: - return lines - selected: _PreprocessedLines = [] - macro_names = self._normalize_macro_defines(macro_defines) - pp_condition_stack: list[tuple[int, int]] = [] - pp_active_stack: list[bool] = [] - pp_group_counter = 0 - for line, _lineno, _source_line in lines: - handled_pp, pp_group_counter = self._handle_procedure_preprocessor_line( - line.strip(), - macro_selection_enabled=True, - macro_names=macro_names, - pp_condition_stack=pp_condition_stack, - pp_active_stack=pp_active_stack, - pp_group_counter=pp_group_counter, - ) - if handled_pp: - continue - if pp_active_stack and not all(pp_active_stack): - continue - selected.append((line, _lineno, _source_line)) - return selected - def _handle_procedure_preprocessor_line( self, line: str, @@ -1113,28 +1073,25 @@ def _handle_procedure_preprocessor_line( if directive_low.startswith("ifdef "): pp_group_counter += 1 pp_condition_stack.append((pp_group_counter, 0)) - expr = directive.split(None, 1)[1].strip() if len(directive.split(None, 1)) > 1 else "" - pp_active_stack.append((bool(expr) and expr.lower() in macro_names) if macro_selection_enabled else True) + pp_active_stack.append(True) return True, pp_group_counter if directive_low.startswith("ifndef "): pp_group_counter += 1 pp_condition_stack.append((pp_group_counter, 0)) - expr = directive.split(None, 1)[1].strip() if len(directive.split(None, 1)) > 1 else "" - pp_active_stack.append(((not expr) or expr.lower() not in macro_names) if macro_selection_enabled else True) + pp_active_stack.append(True) return True, pp_group_counter if directive_low.startswith("if "): pp_group_counter += 1 pp_condition_stack.append((pp_group_counter, 0)) - expr = directive.split(None, 1)[1].strip() if len(directive.split(None, 1)) > 1 else "" - pp_active_stack.append(self._eval_cpp_expr(expr, macro_names) if macro_selection_enabled else True) + pp_active_stack.append(True) return True, pp_group_counter if directive_low.startswith("else"): if pp_condition_stack: group_id, branch_id = pp_condition_stack.pop() pp_condition_stack.append((group_id, branch_id + 1)) if pp_active_stack: - prev = pp_active_stack.pop() - pp_active_stack.append((not prev) if macro_selection_enabled else True) + pp_active_stack.pop() + pp_active_stack.append(True) return True, pp_group_counter if directive_low.startswith("elif "): if pp_condition_stack: @@ -1142,8 +1099,7 @@ def _handle_procedure_preprocessor_line( pp_condition_stack.append((group_id, branch_id + 1)) if pp_active_stack: pp_active_stack.pop() - expr = directive.split(None, 1)[1].strip() if len(directive.split(None, 1)) > 1 else "" - pp_active_stack.append(self._eval_cpp_expr(expr, macro_names) if macro_selection_enabled else True) + pp_active_stack.append(True) return True, pp_group_counter if directive_low.startswith("endif"): if pp_condition_stack: @@ -4170,42 +4126,6 @@ def _preprocessor_conditions_overlap(c1: frozenset[str], c2: frozenset[str]) -> values[group] = branch return True - @staticmethod - def _normalize_macro_defines(macro_defines: set[str] | dict[str, int | bool | str] | None) -> set[str]: - if not macro_defines: - return set() - if isinstance(macro_defines, dict): - out = set() - for k, v in macro_defines.items(): - if str(v).strip() not in {"", "0", "false", "False"}: - out.add(str(k).lower()) - return out - return {str(x).lower() for x in macro_defines} - - @staticmethod - def _eval_cpp_expr(expr: str, macro_names: set[str]) -> bool: - """Evaluate a small C-preprocessor boolean expression.""" - txt = expr.strip() - if not txt: # pragma: no cover - bare #if is not valid Fortran preprocessing input. - return False - txt = re.sub(r"defined\s*\(\s*([A-Za-z_]\w*)\s*\)", lambda m: str(m.group(1).lower() in macro_names), txt) - txt = re.sub(r"\bdefined\s+([A-Za-z_]\w*)\b", lambda m: str(m.group(1).lower() in macro_names), txt) - def _ident_to_bool(m: re.Match[str]) -> str: - token = m.group(1) - if token in {"True", "False", "and", "or", "not"}: - return token - return "True" if token.lower() in macro_names else "False" - txt = re.sub(r"\b([A-Za-z_]\w*)\b", _ident_to_bool, txt) - txt = txt.replace("&&", " and ").replace("||", " or ") - txt = re.sub(r"(?])!(?!=)", " not ", txt) - txt = re.sub(r"\b0\b", "False", txt) - txt = re.sub(r"\b1\b", "True", txt) - try: - node = ast.parse(txt, mode="eval") - return bool(eval(compile(node, "", "eval"), {"__builtins__": {}}, {})) - except Exception: - return False - @staticmethod def _looks_like_existing_source_path(source: str | Path) -> bool: """Return True when ``source`` names a readable source file path.""" diff --git a/semantics/c2ir.py b/semantics/c2ir.py index e2bdc7326..6afba906e 100644 --- a/semantics/c2ir.py +++ b/semantics/c2ir.py @@ -306,6 +306,7 @@ def visit_file( }, ), ) + self._apply_include_exposure(module, c_file) return module finally: self.typedefs, self.structs, self.unions, self.enums = previous @@ -887,6 +888,47 @@ def _file_metadata(self, c_file: CFile) -> dict[str, Any]: metadata["readiness_blockers"] = blockers return metadata + @staticmethod + def _private_recipe_paths(c_file: CFile) -> set[str]: + recipe = c_file.preprocessing_recipe or {} + private_paths: set[str] = set() + for item in recipe.get("included_files") or []: + if not isinstance(item, dict): + continue + path = item.get("path") + if isinstance(path, str) and item.get("exposure") == "private": + private_paths.add(path) + return private_paths + + @staticmethod + def _source_filename(location: dict[str, Any] | None) -> str | None: + if not isinstance(location, dict): + return None + filename = location.get("filename") + return filename if isinstance(filename, str) else None + + def _apply_include_exposure(self, module: SemanticModule, c_file: CFile) -> None: + private_paths = self._private_recipe_paths(c_file) + if not private_paths: + return + + def is_private_origin(origin: SemanticOrigin) -> bool: + filename = self._source_filename(origin.source_location) + return filename in private_paths + + for function in module.functions: + if is_private_origin(function.origin): + function.visibility = "private" + for variable in module.variables: + if is_private_origin(variable.origin): + variable.visibility = "private" + for cls in module.classes: + if is_private_origin(cls.origin): + cls.visibility = "private" + cls.fields = [] + if "Opaque" not in cls.base_classes: + cls.base_classes.append("Opaque") + def _project_metadata(self, project: CProject) -> dict[str, Any]: metadata: dict[str, Any] = { "source_language": "c", diff --git a/tests/parser/test_error_handling.py b/tests/parser/test_error_handling.py index f36e50c99..5dede0138 100644 --- a/tests/parser/test_error_handling.py +++ b/tests/parser/test_error_handling.py @@ -253,7 +253,7 @@ def test_duplicate_procedure_name_in_mutually_exclusive_macro_branches_allowed() assert all(sig.name.lower() == "work" for sig in procedures) -def test_macro_defines_select_active_branch_only(): +def test_macro_defines_do_not_select_active_branch_in_parser(): code = """ module m #ifdef USE_MPI @@ -272,11 +272,11 @@ def test_macro_defines_select_active_branch_only(): """ parsed = parse_fortran_file(code, filename="macro_alt_work.f90", macro_defines={"USE_MPI"}) procedures = parsed.modules[0].procedures - assert len(procedures) == 1 - assert procedures[0].kind == "subroutine" + assert len(procedures) == 2 + assert {proc.kind for proc in procedures} == {"subroutine", "function"} -def test_if_defined_macro_expression_selects_branch(): +def test_if_defined_macro_expression_is_not_evaluated_by_parser(): code = """ module m #if defined(USE_MPI) && !defined(USE_SERIAL) @@ -299,8 +299,8 @@ def test_if_defined_macro_expression_selects_branch(): macro_defines={"USE_MPI": 1, "USE_SERIAL": 0}, ) procedures = parsed.modules[0].procedures - assert len(procedures) == 1 - assert procedures[0].kind == "subroutine" + assert len(procedures) == 2 + assert {proc.kind for proc in procedures} == {"subroutine", "function"} def test_duplicate_procedure_name_error_carries_location(): diff --git a/tests/parser/test_preprocessing_cli.py b/tests/parser/test_preprocessing_cli.py index 82214691d..d00711861 100644 --- a/tests/parser/test_preprocessing_cli.py +++ b/tests/parser/test_preprocessing_cli.py @@ -16,6 +16,8 @@ build_compile_commands_invocation, build_direct_preprocess_invocation, build_preprocess_invocation, + build_template_preprocess_invocation, + expand_native_fortran_includes, run_compiler_preprocessor, run_compiler_preprocessor_with_recipe, validate_macro_name, @@ -45,6 +47,21 @@ def _fake_compiler(tmp_path: Path, output: str) -> tuple[Path, Path, dict[str, s return script, args_file, env +def _failing_compiler(tmp_path: Path, stderr: str) -> Path: + script = tmp_path / "failing-cc" + script.write_text( + f"""#!{sys.executable} +import sys + +sys.stderr.write({stderr!r}) +sys.exit(1) +""", + encoding="utf-8", + ) + script.chmod(0o755) + return script + + def test_direct_c_preprocess_invocation_uses_exact_compiler_and_flags(tmp_path: Path): source = tmp_path / "api.h" config = PreprocessingConfig( @@ -222,13 +239,135 @@ def test_compile_commands_invocation_reports_missing_file_and_supports_command_s assert invocation.argv == ["clang", "-E", str(source)] -def test_build_preprocess_invocation_rejects_fortran_compile_database(tmp_path: Path): - with pytest.raises(PreprocessingError, match="only supported for --language c"): - build_preprocess_invocation( - tmp_path / "api.f90", - language="fortran", - config=PreprocessingConfig(mode="compiler", compile_commands="compile_commands.json"), - ) +def test_build_preprocess_invocation_supports_fortran_compile_database(tmp_path: Path): + source = tmp_path / "solver.F90" + source.write_text("subroutine solve()\nend subroutine solve\n", encoding="utf-8") + compiler = tmp_path / "toolchains" / "gfortran-13" + compiler.parent.mkdir() + database = tmp_path / "compile_commands.json" + database.write_text( + json.dumps( + [ + { + "directory": str(tmp_path), + "file": str(source), + "arguments": [ + str(compiler), + "-Iproject/include", + "-cpp", + "-c", + str(source), + "-o", + "solver.o", + ], + } + ] + ), + encoding="utf-8", + ) + + invocation = build_preprocess_invocation( + source, + language="fortran", + config=PreprocessingConfig(mode="compiler", compile_commands=str(database)), + ) + + assert invocation.argv == [ + str(compiler), + "-E", + "-cpp", + "-Iproject/include", + "-cpp", + str(source), + ] + + +def test_command_template_preprocess_invocation_expands_placeholders(tmp_path: Path): + source = tmp_path / "api.h" + config = PreprocessingConfig( + mode="compiler", + adapter="command-template", + command_template="vendor-cc --preprocess {include_dirs} {defines} {undefs} {standard} {compiler_args} {source}", + include_dirs=["include"], + defines=["API_EXPORT="], + undefs=["DEBUG"], + std="c11", + compiler_args=["--target=x86_64-linux"], + ) + + invocation = build_template_preprocess_invocation(source, language="c", config=config) + + assert invocation.argv == [ + "vendor-cc", + "--preprocess", + "-Iinclude", + "-DAPI_EXPORT=", + "-UDEBUG", + "-std=c11", + "--target=x86_64-linux", + str(source), + ] + + +def test_native_fortran_include_expansion_is_recursive_and_preserves_duplicates(tmp_path: Path): + root = tmp_path / "src" / "root.F90" + include = root.parent / "decls.inc" + nested = root.parent / "nested.inc" + root.parent.mkdir() + root.write_text("module m\ninclude \"decls.inc\"\ninclude \"decls.inc\"\nend module m\n", encoding="utf-8") + include.write_text("include \"nested.inc\"\ninteger :: from_decls\n", encoding="utf-8") + nested.write_text("real :: from_nested\n", encoding="utf-8") + + expanded, included_files, mappings, diagnostics = expand_native_fortran_includes( + root.read_text(encoding="utf-8"), + root_path=root, + include_dirs=[], + ) + + assert diagnostics == [] + assert expanded.count("integer :: from_decls") == 2 + assert expanded.count("real :: from_nested") == 2 + assert [Path(item.path).name for item in included_files].count("decls.inc") == 2 + assert any(Path(mapping.original_path).name == "nested.inc" for mapping in mappings) + + +def test_native_fortran_include_lookup_order_missing_and_cycle_diagnostics(tmp_path: Path): + root = tmp_path / "src" / "root.F90" + include_dir = tmp_path / "include" + root.parent.mkdir() + include_dir.mkdir() + root.write_text("include \"shared.inc\"\n", encoding="utf-8") + (root.parent / "shared.inc").write_text("integer :: relative_wins\n", encoding="utf-8") + (include_dir / "shared.inc").write_text("integer :: include_dir_loses\n", encoding="utf-8") + + expanded, _included_files, _mappings, diagnostics = expand_native_fortran_includes( + root.read_text(encoding="utf-8"), + root_path=root, + include_dirs=[str(include_dir)], + ) + + assert diagnostics == [] + assert "relative_wins" in expanded + assert "include_dir_loses" not in expanded + + missing_source = "include \"absent.inc\"\n" + _expanded, _included_files, _mappings, diagnostics = expand_native_fortran_includes( + missing_source, + root_path=root, + include_dirs=[str(include_dir)], + ) + assert [diagnostic.category for diagnostic in diagnostics] == ["INCLUDE_NOT_FOUND"] + + cycle_a = root.parent / "a.inc" + cycle_b = root.parent / "b.inc" + cycle_a.write_text("include \"b.inc\"\n", encoding="utf-8") + cycle_b.write_text("include \"a.inc\"\n", encoding="utf-8") + _expanded, _included_files, _mappings, diagnostics = expand_native_fortran_includes( + "include \"a.inc\"\n", + root_path=root, + include_dirs=[], + ) + assert "INCLUDE_CYCLE" in [diagnostic.category for diagnostic in diagnostics] def test_run_compiler_preprocessor_success_and_failures(monkeypatch, tmp_path: Path): @@ -381,9 +520,26 @@ def test_cli_compiler_specific_flags_require_compiler_mode(tmp_path: Path): assert "--compiler requires --preprocess compiler" in res.stderr -def test_cli_rejects_compile_database_for_fortran_compiler_mode(tmp_path: Path): +def test_cli_accepts_compile_database_for_fortran_compiler_mode(tmp_path: Path): source = tmp_path / "solver.F90" source.write_text("subroutine solve()\nend subroutine solve\n", encoding="utf-8") + compiler, _args_file, env = _fake_compiler( + tmp_path, + "subroutine from_database()\nend subroutine from_database\n", + ) + database = tmp_path / "compile_commands.json" + database.write_text( + json.dumps( + [ + { + "directory": str(tmp_path), + "file": str(source), + "arguments": [str(compiler), "-cpp", "-c", str(source), "-o", "solver.o"], + } + ] + ), + encoding="utf-8", + ) res = subprocess.run( [ @@ -392,19 +548,21 @@ def test_cli_rejects_compile_database_for_fortran_compiler_mode(tmp_path: Path): "x2py", str(source), "--parse", + "--json", "--preprocess", "compiler", - "--compiler", - "gfortran", "--compile-commands", - "compile_commands.json", + str(database), ], capture_output=True, text=True, + check=True, + env=env, ) - assert res.returncode == 2 - assert "--compile-commands is only supported with --language c" in res.stderr + payload = json.loads(res.stdout)[str(source)] + assert [signature["name"] for signature in payload["signatures"]] == ["from_database"] + assert payload["preprocessing_recipe"]["compile_commands"] == str(database) def test_cli_c_compiler_mode_runs_exact_compiler_and_parses_preprocessed_stdout(tmp_path: Path): @@ -476,6 +634,55 @@ def test_cli_c_compiler_mode_runs_exact_compiler_and_parses_preprocessed_stdout( assert payload["original_source_paths"] == ["include/api.h"] +def test_cli_preprocessing_failure_has_category_without_traceback_unless_debug(tmp_path: Path): + header = tmp_path / "api.h" + header.write_text("int run(void);\n", encoding="utf-8") + compiler = _failing_compiler(tmp_path, "bad option\n") + + res = subprocess.run( + [ + sys.executable, + "-m", + "x2py", + str(header), + "--language", + "c", + "--parse", + "--preprocess", + "compiler", + "--compiler", + str(compiler), + ], + capture_output=True, + text=True, + ) + debug_res = subprocess.run( + [ + sys.executable, + "-m", + "x2py", + str(header), + "--language", + "c", + "--parse", + "--preprocess", + "compiler", + "--compiler", + str(compiler), + "--debug", + ], + capture_output=True, + text=True, + ) + + assert res.returncode == 1 + assert "error[PREPROCESSOR_FAILED]" in res.stderr + assert "bad option" in res.stderr + assert "Traceback" not in res.stderr + assert debug_res.returncode == 1 + assert "Traceback" in debug_res.stderr + + def test_cli_c_compile_commands_mode_uses_exact_database_compiler(tmp_path: Path): source = tmp_path / "api.c" source.write_text("API(int) hidden(void);\n", encoding="utf-8") @@ -538,7 +745,41 @@ def test_cli_c_compile_commands_mode_uses_exact_database_compiler(tmp_path: Path assert recipe["compile_commands_entry"]["arguments"][0] == str(compiler) -def test_cli_fortran_internal_mode_uses_define_and_undef_for_branch_selection(tmp_path: Path): +def test_cli_c_compiler_mode_macro_metadata_flows_to_semantic_constants(tmp_path: Path): + header = tmp_path / "api.h" + header.write_text("#define API_VERSION 3\nint api(void);\n", encoding="utf-8") + compiler, _args_file, env = _fake_compiler( + tmp_path, + "#define API_VERSION 3\nint api(void);\n", + ) + + res = subprocess.run( + [ + sys.executable, + "-m", + "x2py", + str(header), + "--language", + "c", + "--semantics", + "--json", + "--preprocess", + "compiler", + "--compiler", + str(compiler), + ], + capture_output=True, + text=True, + check=True, + env=env, + ) + + module = json.loads(res.stdout)[str(header)]["semantic_modules"][0] + constants = {variable["name"]: variable for variable in module["variables"]} + assert constants["API_VERSION"]["default_value"] == "3" + + +def test_cli_fortran_internal_mode_rejects_define_flags_that_need_compiler_selection(tmp_path: Path): source = tmp_path / "branch.F90" source.write_text( """ @@ -553,47 +794,31 @@ def test_cli_fortran_internal_mode_uses_define_and_undef_for_branch_selection(tm encoding="utf-8", ) - selected = subprocess.run( + res = subprocess.run( [sys.executable, "-m", "x2py", str(source), "--parse", "-D", "USE_MPI"], capture_output=True, text=True, - check=True, - ) - fallback = subprocess.run( - [sys.executable, "-m", "x2py", str(source), "--parse", "-U", "USE_MPI"], - capture_output=True, - text=True, - check=True, ) - assert "subroutine selected" in selected.stdout - assert "subroutine fallback" not in selected.stdout - assert "subroutine fallback" in fallback.stdout - assert "subroutine selected" not in fallback.stdout + assert res.returncode == 2 + assert "internal Fortran parsing does not evaluate CPP branches" in res.stderr -def test_cli_fortran_internal_json_records_macro_selection_recipe(tmp_path: Path): +def test_cli_fortran_internal_json_does_not_record_macro_selection_recipe(tmp_path: Path): source = tmp_path / "branch.F90" source.write_text( - "#ifdef USE_MPI\nsubroutine selected()\nend subroutine selected\n#endif\n", + "subroutine selected()\nend subroutine selected\n", encoding="utf-8", ) res = subprocess.run( - [sys.executable, "-m", "x2py", str(source), "--parse", "--json", "-D", "USE_MPI", "-U", "DEBUG"], + [sys.executable, "-m", "x2py", str(source), "--parse", "--json"], capture_output=True, text=True, check=True, ) - recipe = json.loads(res.stdout)[str(source)]["preprocessing_recipe"] - assert recipe["mode"] == "internal" - assert recipe["language"] == "fortran" - assert recipe["source_path"] == str(source) - assert recipe["compiler"] is None - assert recipe["argv"] == [] - assert recipe["defines"] == ["USE_MPI"] - assert recipe["undefs"] == ["DEBUG"] + assert "preprocessing_recipe" not in json.loads(res.stdout)[str(source)] def test_cli_fortran_internal_mode_rejects_include_dirs_that_need_compiler(tmp_path: Path): diff --git a/tests/parser/test_preprocessor_and_execution_boundaries.py b/tests/parser/test_preprocessor_and_execution_boundaries.py index e05a63d7e..4972b9d4b 100644 --- a/tests/parser/test_preprocessor_and_execution_boundaries.py +++ b/tests/parser/test_preprocessor_and_execution_boundaries.py @@ -74,7 +74,7 @@ def test_signature_shape_helpers_evaluate_publicly_parsed_signature(): assert evaluated.arguments[0].shape == ["0:3", "1:3"] assert sig.arguments[0].shape == ["0:nx-1", "1:ny"] -def test_ifndef_and_defined_without_parentheses_macro_selection(): +def test_cpp_directives_are_preserved_without_parser_branch_selection(): code = """ #if defined USE_FAST subroutine fast_path(x) @@ -99,7 +99,12 @@ def test_ifndef_and_defined_without_parentheses_macro_selection(): parsed = parse_fortran_file(code, macro_defines={"USE_FAST": True}) - assert [proc.name for proc in parsed.procedures] == ["fast_path", "default_path"] + assert [proc.name for proc in parsed.procedures] == [ + "fast_path", + "slow_path", + "default_path", + "selected_slow_path", + ] def test_include_and_ignored_spec_lines_do_not_change_public_signature(): code = """ @@ -224,7 +229,7 @@ def test_openmp_declarative_directives_raise_but_executable_directives_are_body_ assert parse_fortran_file(executable, filename="omp_body.f90").procedures[0].name == "omp_body" assert parse_fortran_file(fixed_form_executable, filename="fixed_omp.f").procedures[0].name == "fixed_omp" -def test_cpp_selection_false_and_malformed_expressions_choose_else_branch(): +def test_cpp_false_and_malformed_expressions_are_not_evaluated_by_parser(): code = """ #if 0 subroutine false_if_branch() @@ -245,7 +250,12 @@ def test_cpp_selection_false_and_malformed_expressions_choose_else_branch(): parsed = parse_fortran_file(code, macro_defines=set()) - assert [proc.name for proc in parsed.procedures] == ["false_if_else", "malformed_if_else"] + assert [proc.name for proc in parsed.procedures] == [ + "false_if_branch", + "false_if_else", + "malformed_if_branch", + "malformed_if_else", + ] def test_statement_function_and_numeric_label_before_execution_part(): code = """ @@ -261,7 +271,7 @@ def test_statement_function_and_numeric_label_before_execution_part(): assert sig.arguments[0].base_type == "real" -def test_preprocessor_boolean_identifiers_and_stray_directives_from_public_parse(): +def test_preprocessor_boolean_identifiers_are_not_evaluated_by_public_parse(): code = """ #if USE_FAST && !USE_SLOW subroutine selected_fast() @@ -283,6 +293,7 @@ def test_preprocessor_boolean_identifiers_and_stray_directives_from_public_parse assert [proc.name for proc in parsed.procedures] == [ "selected_fast", + "selected_slow", "after_stray_directives", ] diff --git a/tests/parser/test_procedure_and_type_parsing.py b/tests/parser/test_procedure_and_type_parsing.py index 07eb6221b..a6f122d82 100644 --- a/tests/parser/test_procedure_and_type_parsing.py +++ b/tests/parser/test_procedure_and_type_parsing.py @@ -236,7 +236,7 @@ def test_fixed_form_and_interface_detection(): assert parsed.interfaces[0].procedures[0].in_interface is True -def test_preprocessor_macro_selection_uses_active_branch_from_inline_fortran(): +def test_preprocessor_macro_defines_do_not_select_active_branch_from_inline_fortran(): source = """ #ifdef USE_A subroutine selected_a(x) @@ -256,10 +256,9 @@ def test_preprocessor_macro_selection_uses_active_branch_from_inline_fortran(): selected = parse_fortran_file(source, macro_defines={"USE_B": True}) fallback = parse_fortran_file(source, macro_defines={"USE_A": False, "USE_B": False}) - assert [proc.name for proc in selected.procedures] == ["selected_b"] - assert selected.procedures[0].arguments[0].base_type == "real" - assert [proc.name for proc in fallback.procedures] == ["fallback"] - assert fallback.procedures[0].arguments[0].base_type == "logical" + assert [proc.name for proc in selected.procedures] == ["selected_a", "selected_b", "fallback"] + assert [proc.arguments[0].base_type for proc in selected.procedures] == ["integer", "real", "logical"] + assert [proc.name for proc in fallback.procedures] == ["selected_a", "selected_b", "fallback"] def test_legacy_character_and_star_kind_declarations_from_inline_fortran(): diff --git a/tests/parser/test_scope_handling.py b/tests/parser/test_scope_handling.py index 867afd341..55fa41932 100644 --- a/tests/parser/test_scope_handling.py +++ b/tests/parser/test_scope_handling.py @@ -119,7 +119,7 @@ def test_same_name_in_overlapping_ifdef_branches_errors_when_both_active(): end module m """ with pytest.raises(FortranParseError, match="Duplicate procedure name"): - parse_fortran_file(code, filename="scope_ifdef_overlap.f90", macro_defines={"USE_A", "USE_B"}) + parse_fortran_file(code, filename="scope_ifdef_overlap.f90") def test_module_symbol_tables_keep_derived_type_fields_scoped_to_type(): diff --git a/tests/semantics/test_c2ir.py b/tests/semantics/test_c2ir.py index 2918d9cd0..063bc23a2 100644 --- a/tests/semantics/test_c2ir.py +++ b/tests/semantics/test_c2ir.py @@ -141,6 +141,37 @@ def test_c2ir_converts_structs_and_opaque_struct_pointers(): assert report["wrappable"] is True +def test_c2ir_private_include_types_remain_available_as_opaque_handles(): + parsed = parse_c_file( + """ +# 1 "private.h" 1 +struct private_context { int internal; }; +# 1 "api.h" 2 +struct private_context *make_context(void); +void use_context(struct private_context *ctx); +""", + filename="api.h", + preprocessing="compiler", + ) + parsed.preprocessing_recipe = { + "included_files": [ + {"path": "api.h", "dependency_kind": "root", "exposure": "public"}, + {"path": "private.h", "dependency_kind": "project", "exposure": "private"}, + ] + } + + module = c_file_to_semantic_modules(parsed)[0] + private_context = next(cls for cls in module.classes if cls.name == "private_context") + make_context = _function(module, "make_context") + use_context = _function(module, "use_context") + + assert private_context.visibility == "private" + assert private_context.base_classes == ["Opaque"] + assert private_context.fields == [] + assert make_context.return_type.name == "private_context" + assert use_context.arguments[0].semantic_type.name == "private_context" + + def test_c2ir_converts_enum_constants_and_simple_macro_constants(): parsed = parse_c_file( """ diff --git a/x2py/c_type_probe.py b/x2py/c_type_probe.py index 13bb67eb5..8c40c7e04 100644 --- a/x2py/c_type_probe.py +++ b/x2py/c_type_probe.py @@ -18,7 +18,7 @@ import tempfile from collections.abc import Sequence -from .preprocessing import PreprocessingConfig, validate_macro_name +from .preprocessing import PreprocessingConfig, PreprocessingError, validate_macro_name class CStandardTypeProbeError(ValueError): @@ -291,7 +291,7 @@ def main(argv: list[str] | None = None) -> int: ), runner=args.runner or None, ) - except ValueError as exc: + except (PreprocessingError, ValueError) as exc: parser.error(str(exc)) print(json.dumps(report.to_dict(), indent=2)) return 0 diff --git a/x2py/cli.py b/x2py/cli.py index ec2c59abc..1fa9ba444 100644 --- a/x2py/cli.py +++ b/x2py/cli.py @@ -7,7 +7,7 @@ from dataclasses import asdict, fields, is_dataclass from pathlib import Path -from c_parser.cli import expand_c_paths, format_c_report, parse_c_report +from c_parser.cli import attach_preprocessing_recipe, expand_c_paths, format_c_report, parse_c_report from c_parser.models import CParseError from c_parser.parser import CParser from fortran_parser.models import FortranParseError @@ -155,10 +155,9 @@ def _fortran_source_for_path( config=preprocessing, ) return source, None, recipe.to_dict() - macro_defines = preprocessing.fortran_macro_defines() return ( path.read_text(encoding="utf-8"), - macro_defines or None, + None, preprocessing.fortran_internal_recipe(path), ) @@ -203,7 +202,7 @@ def _parse_c_path( include_dirs=preprocessing.include_dirs, preprocessing=_c_parser_preprocessing_mode(preprocessing), ) - parsed.preprocessing_recipe = preprocessing_recipe + attach_preprocessing_recipe(parsed, preprocessing_recipe) return parsed @@ -413,31 +412,43 @@ def _build_preprocessing_config(args: argparse.Namespace, parser: argparse.Argum mode=args.preprocess, compiler=args.compiler, compile_commands=args.compile_commands, + adapter=args.preprocessor_adapter, + command_template=args.preprocess_template, include_dirs=list(args.include_dirs or []), defines=defines, undefs=undefs, std=args.std, compiler_args=list(args.compiler_args or []), + include_exposure=args.include_exposure, + public_includes=list(args.public_includes or []), + private_includes=list(args.private_includes or []), ) compiler_only_flags = [ ("--compiler", args.compiler), ("--compile-commands", args.compile_commands), + ("--preprocessor-adapter", None if args.preprocessor_adapter == "auto" else args.preprocessor_adapter), + ("--preprocess-template", args.preprocess_template), ("--std", args.std), ("--compiler-arg", args.compiler_args), + ("--include-exposure", None if args.include_exposure == "reachable-project" else args.include_exposure), + ("--public-include", args.public_includes), + ("--private-include", args.private_includes), ] for option, value in compiler_only_flags: if value and not config.uses_compiler: parser.error(f"{option} requires --preprocess compiler") - if config.uses_compiler and not config.compiler and not config.compile_commands: + if config.uses_compiler and config.command_template and config.adapter != "command-template": + parser.error("--preprocess-template requires --preprocessor-adapter command-template") + if config.uses_compiler and not config.compiler and not config.compile_commands and not config.command_template: parser.error("--preprocess compiler requires --compiler with an exact executable, for example gcc-13 or /usr/bin/clang-18") - if config.compile_commands and args.language != "c": - parser.error("--compile-commands is only supported with --language c") if config.compile_commands and not config.uses_compiler: parser.error("--compile-commands requires --preprocess compiler") if args.language == "c" and not config.uses_compiler and (defines or undefs): parser.error("-D/--define and -U/--undef affect C only with --preprocess compiler; raw C mode records source macros without selecting branches") + if args.language == "fortran" and not config.uses_compiler and (defines or undefs): + parser.error("-D/--define and -U/--undef affect Fortran only with --preprocess compiler; internal Fortran parsing does not evaluate CPP branches") if args.language == "fortran" and not config.uses_compiler and config.include_dirs: parser.error("-I/--include-dir affects Fortran only with --preprocess compiler") return config @@ -497,10 +508,10 @@ def main() -> int: " python -m x2py path/to/api.c --language c --parse --preprocess compiler --compiler /usr/bin/gcc-13 --compiler-arg=--sysroot=/opt/sdk\n" " Parse C with compile_commands.json for project flags:\n" " python -m x2py path/to/api.c --language c --parse --preprocess compiler --compile-commands build/compile_commands.json\n" - " Parse Fortran with internal macro branch selection:\n" - " python -m x2py path/to/file.F90 --parse -D USE_MPI -U DEBUG\n" " Parse Fortran with an exact compiler executable:\n" " python -m x2py path/to/file.F90 --parse --preprocess compiler --compiler /usr/bin/gfortran-12 -I include -D USE_MPI\n" + " Parse with a custom preprocessing command template:\n" + " python -m x2py path/to/api.h --language c --parse --preprocess compiler --preprocessor-adapter command-template --preprocess-template 'cc -E {include_dirs} {defines} {source}'\n" " Write parser JSON:\n" " python -m x2py path/to/file.f90 --parse --json --out report.json\n" " Write one JSON file next to each source:\n" @@ -540,11 +551,17 @@ def main() -> int: choices=("internal", "compiler"), default="internal", help=( - "Preprocessing mode. 'internal' keeps the current lightweight parser preprocessing " - "(Fortran macro selection, C raw directive metadata). 'compiler' runs the exact " + "Preprocessing mode. 'internal' parses plain or already-expanded source without CPP branch selection. " + "'compiler' runs the exact " "compiler/preprocessor configured by --compiler or --compile-commands." ), ) + parser.add_argument( + "--preprocessor-adapter", + choices=("auto", "gcc-compatible-c", "gnu-fortran", "command-template"), + default="auto", + help="Compiler adapter family. Use command-template for unsupported compiler families.", + ) parser.add_argument( "--compiler", help=( @@ -555,7 +572,12 @@ def main() -> int: parser.add_argument( "--compile-commands", metavar="PATH", - help="C compile_commands.json database used with --language c --preprocess compiler.", + help="compile_commands.json database used with --preprocess compiler.", + ) + parser.add_argument( + "--preprocess-template", + metavar="TEMPLATE", + help="Custom preprocessing command template. Supported placeholders include {source}, {include_dirs}, {defines}, {undefs}, {standard}, and {compiler_args}.", ) parser.add_argument( "-I", @@ -593,6 +615,26 @@ def main() -> int: metavar="ARG", help="Raw compiler preprocessing argument. Use --compiler-arg=-target for values starting with '-'.", ) + parser.add_argument( + "--include-exposure", + choices=("reachable-project", "roots-only"), + default="reachable-project", + help="Public wrapper exposure policy for reachable included files.", + ) + parser.add_argument( + "--public-include", + dest="public_includes", + action="append", + metavar="PATH_OR_PATTERN", + help="Force a matched included file to be public in wrapper output.", + ) + parser.add_argument( + "--private-include", + dest="private_includes", + action="append", + metavar="PATH_OR_PATTERN", + help="Force a matched included file to be private in wrapper output.", + ) parser.add_argument( "--show-vars", action="store_true", @@ -674,6 +716,21 @@ def main() -> int: raise print(exc.format_diagnostic(color=_diagnostic_color_enabled(disabled=args.no_color), debug=False), file=sys.stderr) return 1 + except PreprocessingError as exc: + if args.debug or _env_flag("X2PY_DEBUG"): + raise + if exc.diagnostics: + for diagnostic in exc.diagnostics: + location = diagnostic.path or "" + if diagnostic.line is not None: + location = f"{location}:{diagnostic.line}" + print( + f"{location}: error[{diagnostic.category}]: {diagnostic.message}", + file=sys.stderr, + ) + else: + print(f"x2py: error[{exc.category}]: {exc}", file=sys.stderr) + return 1 except (SyntaxError, ValueError) as exc: if args.debug or _env_flag("X2PY_DEBUG"): raise diff --git a/x2py/fortran_type_probe.py b/x2py/fortran_type_probe.py index 20ca686a7..9653ae2ca 100644 --- a/x2py/fortran_type_probe.py +++ b/x2py/fortran_type_probe.py @@ -21,7 +21,7 @@ import subprocess import tempfile -from .preprocessing import PreprocessingConfig, validate_macro_name +from .preprocessing import PreprocessingConfig, PreprocessingError, validate_macro_name class FortranTypeProbeError(ValueError): @@ -400,7 +400,7 @@ def main(argv: list[str] | None = None) -> int: args.expressions, runner=args.runner or None, ) - except ValueError as exc: + except (PreprocessingError, ValueError) as exc: parser.error(str(exc)) print(json.dumps(report.to_dict(), indent=2)) return 0 diff --git a/x2py/preprocessing.py b/x2py/preprocessing.py index 956edb8a2..c208f350a 100644 --- a/x2py/preprocessing.py +++ b/x2py/preprocessing.py @@ -1,7 +1,8 @@ -"""Compiler preprocessing module for x2py. +"""Compiler-backed preprocessing support for x2py wrapper pipelines. -This module provides compiler-based preprocessing for both Fortran and C code, -resolving includes and expanding macros using the actual compiler preprocessor. +The parser frontends intentionally parse one source stream. This module owns the +compiler/preprocessor invocation, side-channel metadata, source provenance, and +the native Fortran INCLUDE expansion that GNU Fortran CPP leaves unresolved. """ from __future__ import annotations @@ -9,222 +10,1241 @@ import json import os import re +import shlex +import shutil import subprocess -import tempfile from dataclasses import dataclass, field from pathlib import Path -from typing import Optional +from typing import Literal, Protocol, Sequence + + +PreprocessingCategory = Literal[ + "PREPROCESSOR_NOT_FOUND", + "PREPROCESSOR_FAILED", + "INVALID_COMPILER_ARGUMENTS", + "UNSUPPORTED_COMPILER_CAPABILITY", + "PROVENANCE_UNAVAILABLE", + "INCLUDE_NOT_FOUND", + "INCLUDE_CYCLE", +] + +IncludeMechanism = Literal["c_include", "cpp_include", "fortran_include"] +DependencyKind = Literal["root", "project", "system"] +Exposure = Literal["public", "private"] class PreprocessingError(Exception): """Raised when preprocessing configuration or execution fails.""" - pass + + def __init__( + self, + message: str, + *, + category: PreprocessingCategory = "PREPROCESSOR_FAILED", + diagnostics: Sequence["PreprocessingDiagnostic"] | None = None, + ) -> None: + self.category = category + self.diagnostics = list(diagnostics or []) + super().__init__(message) + + +@dataclass +class Invocation: + """Concrete command line used to obtain preprocessed source.""" + + argv: list[str] + cwd: str | None = None + adapter: str = "direct" + language: str | None = None + compiler: str | None = None + compile_commands: str | None = None + compile_commands_entry: dict[str, object] | None = None + capabilities: dict[str, bool] = field(default_factory=dict) + + +@dataclass +class PreprocessingDiagnostic: + category: PreprocessingCategory + message: str + severity: Literal["error", "warning", "note"] = "error" + path: str | None = None + line: int | None = None + command: list[str] = field(default_factory=list) + + def to_dict(self) -> dict[str, object]: + return { + "category": self.category, + "message": self.message, + "severity": self.severity, + "path": self.path, + "line": self.line, + "command": list(self.command), + } + + +@dataclass +class PreprocessingPlan: + language: str + source_path: str + adapter: str + compiler: str | None = None + cwd: str | None = None + include_dirs: list[str] = field(default_factory=list) + defines: list[str] = field(default_factory=list) + undefs: list[str] = field(default_factory=list) + standard: str | None = None + compiler_args: list[str] = field(default_factory=list) + compile_commands: str | None = None + command_template: str | None = None + + def to_dict(self) -> dict[str, object]: + return { + "language": self.language, + "source_path": self.source_path, + "adapter": self.adapter, + "compiler": self.compiler, + "cwd": self.cwd, + "include_dirs": list(self.include_dirs), + "defines": list(self.defines), + "undefs": list(self.undefs), + "standard": self.standard, + "compiler_args": list(self.compiler_args), + "compile_commands": self.compile_commands, + "command_template": self.command_template, + } + + +@dataclass +class IncludedFile: + path: str + included_by: str | None = None + include_line: int | None = None + mechanism: IncludeMechanism = "cpp_include" + dependency_kind: DependencyKind = "project" + exposure: Exposure = "public" + + def to_dict(self) -> dict[str, object]: + return { + "path": self.path, + "included_by": self.included_by, + "include_line": self.include_line, + "mechanism": self.mechanism, + "dependency_kind": self.dependency_kind, + "exposure": self.exposure, + } + + +@dataclass +class SourceMapping: + generated_line: int + original_path: str + original_line: int + include_stack: list[str] = field(default_factory=list) + + def to_dict(self) -> dict[str, object]: + return { + "generated_line": self.generated_line, + "original_path": self.original_path, + "original_line": self.original_line, + "include_stack": list(self.include_stack), + } + + +@dataclass +class MacroDefinition: + name: str + value: str | None = None + function_like: bool = False + parameters: list[str] | None = None + path: str | None = None + line: int | None = None + builtin: bool = False + + def to_dict(self) -> dict[str, object]: + return { + "name": self.name, + "value": self.value, + "function_like": self.function_like, + "parameters": list(self.parameters) if self.parameters is not None else None, + "path": self.path, + "line": self.line, + "builtin": self.builtin, + } + + +@dataclass +class PreprocessResult: + source: str + recipe: dict[str, object] + included_files: list[IncludedFile] = field(default_factory=list) + source_mappings: list[SourceMapping] = field(default_factory=list) + macros: list[MacroDefinition] = field(default_factory=list) + diagnostics: list[PreprocessingDiagnostic] = field(default_factory=list) + + def to_dict(self) -> dict[str, object]: + return { + "source": self.source, + "recipe": dict(self.recipe), + "included_files": [item.to_dict() for item in self.included_files], + "source_mappings": [item.to_dict() for item in self.source_mappings], + "macros": [item.to_dict() for item in self.macros], + "diagnostics": [item.to_dict() for item in self.diagnostics], + } @dataclass class PreprocessingRecipe: - """Metadata about how a source file was preprocessed.""" + """JSON-compatible metadata about one preprocessing operation.""" + language: str - compiler: str + compiler: str | None mode: str = "compiler" + adapter: str = "direct" + argv: list[str] = field(default_factory=list) + cwd: str | None = None include_dirs: list[str] = field(default_factory=list) defines: list[str] = field(default_factory=list) undefs: list[str] = field(default_factory=list) - std: Optional[str] = None + standard: str | None = None compiler_args: list[str] = field(default_factory=list) - source_file: Optional[str] = None - - def to_dict(self) -> dict: - """Convert to a JSON-serializable dictionary.""" + source_path: str | None = None + compile_commands: str | None = None + compile_commands_entry: dict[str, object] | None = None + command_template: str | None = None + included_files: list[dict[str, object]] = field(default_factory=list) + source_mappings: list[dict[str, object]] = field(default_factory=list) + macros: list[dict[str, object]] = field(default_factory=list) + diagnostics: list[dict[str, object]] = field(default_factory=list) + capabilities: dict[str, bool] = field(default_factory=dict) + + @property + def std(self) -> str | None: + """Backward-compatible alias for older callers.""" + return self.standard + + def to_dict(self) -> dict[str, object]: return { "language": self.language, "compiler": self.compiler, "mode": self.mode, - "include_dirs": self.include_dirs, - "defines": self.defines, - "undefs": self.undefs, - "std": self.std, - "compiler_args": self.compiler_args, - "source_file": self.source_file, + "adapter": self.adapter, + "argv": list(self.argv), + "cwd": self.cwd, + "include_dirs": list(self.include_dirs), + "defines": list(self.defines), + "undefs": list(self.undefs), + "standard": self.standard, + "std": self.standard, + "compiler_args": list(self.compiler_args), + "source_path": self.source_path, + "source_file": self.source_path, + "compile_commands": self.compile_commands, + "compile_commands_entry": self.compile_commands_entry, + "command_template": self.command_template, + "included_files": list(self.included_files), + "source_mappings": list(self.source_mappings), + "macros": list(self.macros), + "diagnostics": list(self.diagnostics), + "capabilities": dict(self.capabilities), } @dataclass class PreprocessingConfig: - """Configuration for preprocessing operations.""" - mode: str = "internal" # "internal" or "compiler" - compiler: Optional[str] = None - compile_commands: Optional[str] = None + """Configuration for compiler-backed preprocessing operations.""" + + mode: str = "internal" + compiler: str | None = None + compile_commands: str | None = None + adapter: str = "auto" + command_template: str | None = None include_dirs: list[str] = field(default_factory=list) defines: list[str] = field(default_factory=list) undefs: list[str] = field(default_factory=list) - std: Optional[str] = None + std: str | None = None compiler_args: list[str] = field(default_factory=list) - + include_exposure: Literal["reachable-project", "roots-only"] = "reachable-project" + public_includes: list[str] = field(default_factory=list) + private_includes: list[str] = field(default_factory=list) + collect_macro_metadata: bool = False + @property def uses_compiler(self) -> bool: - """True if compiler-based preprocessing is configured.""" return self.mode == "compiler" - + def fortran_macro_defines(self) -> dict[str, int | str] | None: - """Extract macro defines for Fortran parser (internal mode only).""" + """Return legacy parser macros only for non-production internal tests.""" if self.uses_compiler: return None - if not self.defines and not self.undefs: return None - - result = {} + result: dict[str, int | str] = {} for define in self.defines: - if "=" in define: - name, value = define.split("=", 1) - result[name] = value - else: - result[define] = 1 - return result if result else None - + name, value = define.split("=", 1) if "=" in define else (define, 1) + result[name] = value + for undef in self.undefs: + result[undef] = 0 + return result + def fortran_internal_recipe(self, path: Path) -> dict[str, object] | None: - """Generate a recipe dict for Fortran internal preprocessing.""" - if self.uses_compiler: + if self.uses_compiler or not (self.defines or self.undefs): return None - - recipe = PreprocessingRecipe( + return PreprocessingRecipe( language="fortran", - compiler="internal", + compiler=None, mode="internal", - defines=self.defines, - undefs=self.undefs, - source_file=str(path), - ) - return recipe.to_dict() + adapter="parser-test", + argv=[], + defines=list(self.defines), + undefs=list(self.undefs), + source_path=str(path), + ).to_dict() + + +class CompilerAdapter(Protocol): + name: str + capabilities: dict[str, bool] + + def build_preprocess_invocation( + self, + source_path: Path, + *, + language: str, + config: PreprocessingConfig, + ) -> Invocation: + ... + + def collect_dependencies(self, result: "PreprocessResult") -> list[IncludedFile]: + ... + + def collect_macros(self, result: "PreprocessResult") -> list[MacroDefinition]: + ... + + def parse_linemarkers(self, source: str, filename: str | None = None) -> list[SourceMapping]: + ... + + +_VALID_LANGUAGES = {"c", "fortran"} +_C_SOURCE_SUFFIXES = {".c", ".h", ".i"} +_FORTRAN_SOURCE_SUFFIXES = {".f", ".for", ".ftn", ".f77", ".f90", ".f95", ".f03", ".f08"} +_DEFINE_RE = re.compile(r"^\s*#\s*define\s+([A-Za-z_]\w*)(\(([^)]*)\))?(?:\s+(.*))?$") +_LINEMARKER_RE = re.compile( + r'^\s*#\s+(?P\d+)\s+(?:"(?P(?:[^"\\]|\\.)*)"|(?P\S+))(?P(?:\s+\d+)*)\s*$' +) +_LINE_DIRECTIVE_RE = re.compile( + r'^\s*#\s*line\s+(?P\d+)(?:\s+(?:"(?P(?:[^"\\]|\\.)*)"|(?P\S+)))?\s*$' +) +_FORTRAN_INCLUDE_RE = re.compile(r"^\s*include\s*(?P['\"])(?P[^'\"]+)(?P=quote)\s*$", re.IGNORECASE) def validate_macro_name(macro_str: str, context: str) -> None: - """Validate that a macro definition has a valid name.""" + """Validate that a command-line macro definition has a usable name.""" + if not macro_str: - raise PreprocessingError(f"{context}: empty macro definition") - - # Extract name part (before = if present) + raise PreprocessingError( + f"{context} requires a macro name", + category="INVALID_COMPILER_ARGUMENTS", + ) name = macro_str.split("=", 1)[0] - - # Check valid C/Fortran identifier - if not re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", name): + if not name: + raise PreprocessingError( + f"{context} requires a macro name before '='", + category="INVALID_COMPILER_ARGUMENTS", + ) + if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", name): + raise PreprocessingError( + f"{context}: invalid macro name '{name}'; must be a valid identifier", + category="INVALID_COMPILER_ARGUMENTS", + ) + + +def _require_language(language: str) -> None: + if language not in _VALID_LANGUAGES: + raise PreprocessingError( + f"compiler preprocessing is not supported for language {language!r}", + category="INVALID_COMPILER_ARGUMENTS", + ) + + +def _compiler_required(config: PreprocessingConfig, language: str) -> str: + if not config.compiler: raise PreprocessingError( - f"{context}: invalid macro name '{name}'; must be a valid identifier" + f"{language} compiler preprocessing requires --compiler with an exact executable", + category="INVALID_COMPILER_ARGUMENTS", ) + return config.compiler -def _get_compiler_for_language(language: str, compiler: Optional[str]) -> str: - """Determine the compiler to use based on language.""" - if compiler: - return compiler - +def _preprocessor_options(config: PreprocessingConfig, *, language: str, include_language_flag: bool) -> list[str]: + args: list[str] = ["-E"] + if include_language_flag and language == "c": + args.extend(["-x", "c"]) if language == "fortran": - return "gfortran" - elif language == "c": - return "gcc" + args.append("-cpp") + for include_dir in config.include_dirs: + args.append(f"-I{include_dir}") + for define in config.defines: + args.append(f"-D{define}") + for undef in config.undefs: + args.append(f"-U{undef}") + if config.std: + args.append(f"-std={config.std}") + args.extend(config.compiler_args) + return args + + +class GCCCompatibleCAdapter: + name = "gcc-compatible-c" + capabilities = {"dependency_output": True, "macro_dump": True, "linemarkers": True} + + def build_preprocess_invocation( + self, + source_path: Path, + *, + language: str, + config: PreprocessingConfig, + ) -> Invocation: + return build_direct_preprocess_invocation(source_path, language=language, config=config) + + def collect_dependencies(self, result: PreprocessResult) -> list[IncludedFile]: + return list(result.included_files) + + def collect_macros(self, result: PreprocessResult) -> list[MacroDefinition]: + return list(result.macros) + + def parse_linemarkers(self, source: str, filename: str | None = None) -> list[SourceMapping]: + return parse_linemarker_mappings(source, filename=filename) + + +class GNUFortranAdapter(GCCCompatibleCAdapter): + name = "gnu-fortran" + + +class CommandTemplateAdapter(GCCCompatibleCAdapter): + name = "command-template" + capabilities = {"dependency_output": False, "macro_dump": False, "linemarkers": False} + + def build_preprocess_invocation( + self, + source_path: Path, + *, + language: str, + config: PreprocessingConfig, + ) -> Invocation: + return build_template_preprocess_invocation(source_path, language=language, config=config) + + +def build_direct_preprocess_invocation( + source_path: Path | str, + *, + language: str, + config: PreprocessingConfig, +) -> Invocation: + """Build an exact direct compiler invocation for preprocessing.""" + + _require_language(language) + compiler = _compiler_required(config, language) + source = Path(source_path) + argv = [ + compiler, + *_preprocessor_options(config, language=language, include_language_flag=language == "c"), + str(source), + ] + adapter = "gnu-fortran" if language == "fortran" else "gcc-compatible-c" + return Invocation( + argv=argv, + cwd=None, + adapter=adapter, + language=language, + compiler=compiler, + capabilities={"dependency_output": True, "macro_dump": True, "linemarkers": True}, + ) + + +def _load_compile_commands(path: str | os.PathLike[str] | None) -> list[dict[str, object]]: + if not path: + raise PreprocessingError( + "compile_commands database path is missing", + category="INVALID_COMPILER_ARGUMENTS", + ) + database_path = Path(path) + try: + raw = database_path.read_text(encoding="utf-8") + except OSError as exc: + raise PreprocessingError( + f"cannot read compile commands file {database_path}: {exc}", + category="INVALID_COMPILER_ARGUMENTS", + ) from exc + try: + payload = json.loads(raw) + except json.JSONDecodeError as exc: + raise PreprocessingError( + f"invalid compile commands JSON: {exc}", + category="INVALID_COMPILER_ARGUMENTS", + ) from exc + if not isinstance(payload, list): + raise PreprocessingError( + "compile_commands.json must contain a list", + category="INVALID_COMPILER_ARGUMENTS", + ) + return payload + + +def _entry_file_path(entry: dict[str, object]) -> Path: + if "file" not in entry: + raise PreprocessingError( + "compile_commands entry is missing 'file'", + category="INVALID_COMPILER_ARGUMENTS", + ) + directory = Path(str(entry.get("directory") or ".")) + file_path = Path(str(entry["file"])) + if not file_path.is_absolute(): + file_path = directory / file_path + return file_path + + +def _same_source(left: Path, right: Path) -> bool: + try: + return left.resolve() == right.resolve() + except OSError: + return left.absolute() == right.absolute() + + +def _compile_command_argv(entry: dict[str, object]) -> list[str]: + if "arguments" in entry: + arguments = entry["arguments"] + if not isinstance(arguments, list): + raise PreprocessingError( + "compile_commands entry 'arguments' must contain a list", + category="INVALID_COMPILER_ARGUMENTS", + ) + argv = [str(arg) for arg in arguments] + elif "command" in entry: + command = entry["command"] + if not isinstance(command, str): + raise PreprocessingError( + "compile_commands entry 'command' must contain a string", + category="INVALID_COMPILER_ARGUMENTS", + ) + argv = shlex.split(command) else: - raise PreprocessingError(f"Unknown language: {language}") + raise PreprocessingError( + "compile_commands entry must contain 'arguments' or 'command'", + category="INVALID_COMPILER_ARGUMENTS", + ) + if not argv: + raise PreprocessingError( + "compile_commands entry has an empty command", + category="INVALID_COMPILER_ARGUMENTS", + ) + return argv + + +def _is_source_arg(arg: str, source: Path, cwd: Path) -> bool: + path = Path(arg) + if not path.suffix: + return False + candidate = path if path.is_absolute() else cwd / path + return _same_source(candidate, source) + + +def _filter_compile_only_args(args: list[str], source: Path, cwd: Path) -> list[str]: + filtered: list[str] = [] + index = 0 + while index < len(args): + arg = args[index] + if arg in {"-c", "/c"}: + index += 1 + continue + if arg == "-o": + index += 2 + continue + if arg.startswith("-o") and arg != "-o": + index += 1 + continue + if arg.startswith("/Fo"): + index += 1 + continue + if arg in {"-MF", "-MT", "-MQ"}: + index += 2 + continue + if arg.startswith(("-MF", "-MT", "-MQ")): + index += 1 + continue + if _is_source_arg(arg, source, cwd): + index += 1 + continue + filtered.append(arg) + index += 1 + return filtered + + +def _compile_commands_entry(source_path: Path, database: list[dict[str, object]]) -> dict[str, object]: + matches: list[dict[str, object]] = [] + for entry in database: + if not isinstance(entry, dict): + raise PreprocessingError( + "compile_commands entries must be objects", + category="INVALID_COMPILER_ARGUMENTS", + ) + entry_path = _entry_file_path(entry) + if _same_source(entry_path, source_path): + matches.append(entry) + if not matches: + raise PreprocessingError( + f"no compile_commands entry found for {source_path}", + category="INVALID_COMPILER_ARGUMENTS", + ) + if len(matches) > 1: + raise PreprocessingError( + f"multiple compile_commands entries found for {source_path}", + category="INVALID_COMPILER_ARGUMENTS", + ) + return matches[0] -def _build_preprocessor_flags( +def build_compile_commands_invocation( + source_path: Path | str, + *, config: PreprocessingConfig, + language: str = "c", +) -> Invocation: + """Build a preprocessing invocation from a compile_commands.json entry.""" + + _require_language(language) + source = Path(source_path) + database = _load_compile_commands(config.compile_commands) + entry = _compile_commands_entry(source, database) + cwd = Path(str(entry.get("directory") or ".")) + compile_argv = _compile_command_argv(entry) + compiler = config.compiler or compile_argv[0] + compile_args = _filter_compile_only_args(compile_argv[1:], source, cwd) + argv = [ + compiler, + *_preprocessor_options(config, language=language, include_language_flag=False), + *compile_args, + str(source), + ] + adapter = "gnu-fortran" if language == "fortran" else "gcc-compatible-c" + return Invocation( + argv=argv, + cwd=str(cwd), + adapter=adapter, + language=language, + compiler=compiler, + compile_commands=str(config.compile_commands) if config.compile_commands else None, + compile_commands_entry=dict(entry), + capabilities={"dependency_output": True, "macro_dump": True, "linemarkers": True}, + ) + + +def _template_token_value(token: str, source: Path, language: str, config: PreprocessingConfig) -> list[str]: + if token == "{source}": + return [str(source)] + if token == "{compiler}": + return [config.compiler or ""] + if token == "{language}": + return [language] + if token == "{include_dirs}": + return [f"-I{item}" for item in config.include_dirs] + if token == "{defines}": + return [f"-D{item}" for item in config.defines] + if token == "{undefs}": + return [f"-U{item}" for item in config.undefs] + if token == "{standard}": + return [f"-std={config.std}"] if config.std else [] + if token == "{compiler_args}": + return list(config.compiler_args) + return [token.format( + source=str(source), + compiler=config.compiler or "", + language=language, + standard=config.std or "", + )] + + +def build_template_preprocess_invocation( + source_path: Path | str, + *, language: str, -) -> list[str]: - """Build compiler preprocessor flags from configuration.""" - flags = [] - - # Add preprocessing flag first - flags.append("-E") # Preprocess only, no compilation - - # Add include directories - for include_dir in config.include_dirs: - flags.append(f"-I{include_dir}") - - # Add defines - for define in config.defines: - if "=" in define: - flags.append(f"-D{define}") + config: PreprocessingConfig, +) -> Invocation: + _require_language(language) + if not config.command_template: + raise PreprocessingError( + "custom command-template adapter requires --preprocess-template", + category="INVALID_COMPILER_ARGUMENTS", + ) + source = Path(source_path) + argv: list[str] = [] + for token in shlex.split(config.command_template): + argv.extend(item for item in _template_token_value(token, source, language, config) if item) + if not argv: + raise PreprocessingError( + "custom command-template adapter expanded to an empty command", + category="INVALID_COMPILER_ARGUMENTS", + ) + return Invocation( + argv=argv, + adapter="command-template", + language=language, + compiler=config.compiler or argv[0], + capabilities={"dependency_output": False, "macro_dump": False, "linemarkers": False}, + ) + + +def build_preprocess_invocation( + source_path: Path | str, + *, + language: str, + config: PreprocessingConfig, +) -> Invocation: + """Build the selected compiler adapter invocation.""" + + _require_language(language) + if config.adapter == "command-template" or config.command_template: + return build_template_preprocess_invocation(source_path, language=language, config=config) + if config.compile_commands: + return build_compile_commands_invocation(source_path, language=language, config=config) + return build_direct_preprocess_invocation(source_path, language=language, config=config) + + +def _unescape_linemarker_filename(text: str) -> str: + out: list[str] = [] + escaped = False + for char in text: + if escaped: + out.append({"n": "\n", "r": "\r", "t": "\t", "\\": "\\", '"': '"'}.get(char, char)) + escaped = False + elif char == "\\": + escaped = True else: - flags.append(f"-D{define}=1") - - # Add undefs - for undef in config.undefs: - flags.append(f"-U{undef}") - - # Add standard flag if provided - if config.std: - flags.append(f"-std={config.std}") - - # Add raw compiler args - flags.extend(config.compiler_args) - - return flags + out.append(char) + if escaped: + out.append("\\") + return "".join(out) -def run_compiler_preprocessor_with_recipe( +def _parse_linemarker(line: str) -> tuple[int, str | None, list[int]] | None: + match = _LINE_DIRECTIVE_RE.match(line.strip()) + if match is not None: + filename = match.group("quoted") or match.group("bare") + return int(match.group("line")), _unescape_linemarker_filename(filename) if filename else None, [] + match = _LINEMARKER_RE.match(line.strip()) + if match is None: + return None + filename = match.group("quoted") or match.group("bare") + flags = [int(flag) for flag in (match.group("flags") or "").split()] + return int(match.group("line")), _unescape_linemarker_filename(filename) if filename else None, flags + + +def _dependency_kind(path: str, flags: Sequence[int] = ()) -> DependencyKind: + if 3 in flags: + return "system" + if path.startswith("<") and path.endswith(">"): + return "system" + return "project" + + +def _exposure_for(path: str, kind: DependencyKind, config: PreprocessingConfig) -> Exposure: + if any(Path(path).match(pattern) or pattern in path for pattern in config.private_includes): + return "private" + if any(Path(path).match(pattern) or pattern in path for pattern in config.public_includes): + return "public" + if kind == "system": + return "private" + if config.include_exposure == "roots-only" and kind != "root": + return "private" + return "public" + + +def parse_linemarker_mappings(source: str, filename: str | None = None) -> list[SourceMapping]: + mappings: list[SourceMapping] = [] + current_path = filename or "" + current_line = 1 + include_stack: list[str] = [current_path] if current_path else [] + for generated_line, line in enumerate(source.splitlines(), start=1): + marker = _parse_linemarker(line) + if marker is not None: + marker_line, marker_path, flags = marker + if marker_path is not None: + if 1 in flags: + if not include_stack or include_stack[-1] != marker_path: + include_stack.append(marker_path) + elif 2 in flags: + if marker_path in include_stack: + include_stack = include_stack[: include_stack.index(marker_path) + 1] + else: + include_stack = [marker_path] + elif include_stack: + include_stack[-1] = marker_path + else: + include_stack = [marker_path] + current_path = marker_path + current_line = marker_line + continue + mappings.append( + SourceMapping( + generated_line=generated_line, + original_path=current_path, + original_line=current_line, + include_stack=list(include_stack), + ) + ) + current_line += 1 + return mappings + + +def _included_files_from_linemarkers( + source: str, + *, + root_path: Path, + language: str, + config: PreprocessingConfig, +) -> list[IncludedFile]: + files: list[IncludedFile] = [ + IncludedFile( + path=str(root_path), + included_by=None, + include_line=None, + mechanism="cpp_include" if language == "fortran" else "c_include", + dependency_kind="root", + exposure="public", + ) + ] + seen = {str(root_path)} + current_path = str(root_path) + current_line = 1 + stack: list[str] = [str(root_path)] + for line in source.splitlines(): + marker = _parse_linemarker(line) + if marker is None: + current_line += 1 + continue + marker_line, marker_path, flags = marker + if marker_path is not None: + if 1 in flags and marker_path not in seen: + kind = _dependency_kind(marker_path, flags) + files.append( + IncludedFile( + path=marker_path, + included_by=stack[-1] if stack else current_path, + include_line=current_line, + mechanism="cpp_include" if language == "fortran" else "c_include", + dependency_kind=kind, + exposure=_exposure_for(marker_path, kind, config), + ) + ) + seen.add(marker_path) + if 1 in flags: + stack.append(marker_path) + elif 2 in flags: + if marker_path in stack: + stack = stack[: stack.index(marker_path) + 1] + else: + stack = [marker_path] + elif stack: + stack[-1] = marker_path + current_path = marker_path + current_line = marker_line + return files + + +def _parse_macro_definitions(source: str, mappings: Sequence[SourceMapping]) -> list[MacroDefinition]: + macros: list[MacroDefinition] = [] + mapping_by_generated = {mapping.generated_line: mapping for mapping in mappings} + for generated_line, line in enumerate(source.splitlines(), start=1): + match = _DEFINE_RE.match(line) + if match is None: + continue + name, params_text, params, value = match.groups() + mapping = mapping_by_generated.get(generated_line) + macros.append( + MacroDefinition( + name=name, + value=value.strip() if value else None, + function_like=params_text is not None, + parameters=[item.strip() for item in params.split(",")] if params is not None and params.strip() else ([] if params_text else None), + path=mapping.original_path if mapping else None, + line=mapping.original_line if mapping else None, + builtin=(mapping.original_path.startswith("<") if mapping else False), + ) + ) + return macros + + +def _mapping_for_generated_line(mappings: Sequence[SourceMapping], generated_line: int, fallback: Path) -> SourceMapping: + for mapping in mappings: + if mapping.generated_line == generated_line: + return mapping + return SourceMapping(generated_line=generated_line, original_path=str(fallback), original_line=generated_line, include_stack=[str(fallback)]) + + +def _resolve_fortran_include(target: str, including_file: str, include_dirs: Sequence[str]) -> Path | None: + candidates = [Path(including_file).parent / target] + candidates.extend(Path(include_dir) / target for include_dir in include_dirs) + for candidate in candidates: + try: + if candidate.is_file(): + return candidate + except OSError: + continue + return None + + +def _line_marker(line: int, path: str, flag: int | None = None) -> str: + escaped = path.replace("\\", "\\\\").replace('"', '\\"') + suffix = f" {flag}" if flag is not None else "" + return f'# {line} "{escaped}"{suffix}' + + +def expand_native_fortran_includes( + source: str, + *, + root_path: Path, + include_dirs: Sequence[str], + config: PreprocessingConfig | None = None, +) -> tuple[str, list[IncludedFile], list[SourceMapping], list[PreprocessingDiagnostic]]: + """Resolve native Fortran INCLUDE statements by textual insertion.""" + + config = config or PreprocessingConfig() + diagnostics: list[PreprocessingDiagnostic] = [] + included_files: list[IncludedFile] = [] + generated_mappings: list[SourceMapping] = [] + line_counter = 0 + + def emit_line(line: str, mapping: SourceMapping, out: list[str]) -> None: + nonlocal line_counter + out.append(line) + line_counter += 1 + generated_mappings.append( + SourceMapping( + generated_line=line_counter, + original_path=mapping.original_path, + original_line=mapping.original_line, + include_stack=list(mapping.include_stack), + ) + ) + + def expand_text(text: str, current_file: Path, stack: list[Path]) -> list[str]: + out: list[str] = [] + mappings = parse_linemarker_mappings(text, filename=str(current_file)) + mapping_by_line = {mapping.generated_line: mapping for mapping in mappings} + for generated_line, line in enumerate(text.splitlines(), start=1): + marker = _parse_linemarker(line) + if marker is not None: + mapping = _mapping_for_generated_line(mappings, generated_line, current_file) + emit_line(line, mapping, out) + continue + mapping = mapping_by_line.get(generated_line) or SourceMapping( + generated_line=generated_line, + original_path=str(current_file), + original_line=generated_line, + include_stack=[str(path) for path in stack], + ) + match = _FORTRAN_INCLUDE_RE.match(line) + if match is None: + emit_line(line, mapping, out) + continue + + target = match.group("path") + resolved = _resolve_fortran_include(target, mapping.original_path, include_dirs) + if resolved is None: + diagnostics.append( + PreprocessingDiagnostic( + category="INCLUDE_NOT_FOUND", + message=f'Fortran INCLUDE file "{target}" was not found', + path=mapping.original_path, + line=mapping.original_line, + ) + ) + continue + try: + resolved_abs = resolved.resolve() + except OSError: + resolved_abs = resolved.absolute() + if resolved_abs in stack: + cycle = " -> ".join(str(path) for path in [*stack, resolved_abs]) + diagnostics.append( + PreprocessingDiagnostic( + category="INCLUDE_CYCLE", + message=f"Fortran INCLUDE cycle detected: {cycle}", + path=mapping.original_path, + line=mapping.original_line, + ) + ) + continue + + kind: DependencyKind = "project" + included_files.append( + IncludedFile( + path=str(resolved_abs), + included_by=mapping.original_path, + include_line=mapping.original_line, + mechanism="fortran_include", + dependency_kind=kind, + exposure=_exposure_for(str(resolved_abs), kind, config), + ) + ) + emit_line(_line_marker(1, str(resolved_abs), 1), mapping, out) + try: + include_text = resolved.read_text(encoding="utf-8") + except OSError as exc: + diagnostics.append( + PreprocessingDiagnostic( + category="INCLUDE_NOT_FOUND", + message=f'Fortran INCLUDE file "{target}" could not be read: {exc}', + path=mapping.original_path, + line=mapping.original_line, + ) + ) + continue + out.extend(expand_text(include_text, resolved_abs, [*stack, resolved_abs])) + emit_line(_line_marker(mapping.original_line + 1, mapping.original_path, 2), mapping, out) + return out + + root_abs = root_path.resolve() if root_path.exists() else root_path.absolute() + expanded_lines = expand_text(source, root_abs, [root_abs]) + return "\n".join(expanded_lines) + ("\n" if source.endswith("\n") else ""), included_files, generated_mappings, diagnostics + + +def _recipe_from_invocation( source_path: Path, language: str, config: PreprocessingConfig, -) -> tuple[str, PreprocessingRecipe]: - """Run compiler preprocessor on a source file, returning preprocessed code and recipe. - - Args: - source_path: Path to the source file - language: "fortran" or "c" - config: Preprocessing configuration - - Returns: - Tuple of (preprocessed_source_code, preprocessing_recipe) - - Raises: - PreprocessingError: If preprocessing fails - """ + invocation: Invocation, + result: PreprocessResult | None = None, +) -> PreprocessingRecipe: + return PreprocessingRecipe( + language=language, + compiler=invocation.compiler, + mode="compiler", + adapter=invocation.adapter, + argv=list(invocation.argv), + cwd=invocation.cwd, + include_dirs=list(config.include_dirs), + defines=list(config.defines), + undefs=list(config.undefs), + standard=config.std, + compiler_args=list(config.compiler_args), + source_path=str(source_path), + compile_commands=invocation.compile_commands, + compile_commands_entry=invocation.compile_commands_entry, + command_template=config.command_template, + included_files=[item.to_dict() for item in result.included_files] if result else [], + source_mappings=[item.to_dict() for item in result.source_mappings] if result else [], + macros=[item.to_dict() for item in result.macros] if result else [], + diagnostics=[item.to_dict() for item in result.diagnostics] if result else [], + capabilities=dict(invocation.capabilities), + ) + + +def preprocess_source( + source_path: Path | str, + *, + language: str, + config: PreprocessingConfig, +) -> PreprocessResult: + """Run compiler preprocessing and return expanded source plus provenance.""" + if not config.uses_compiler: - raise PreprocessingError("Compiler preprocessing not configured") - - compiler = _get_compiler_for_language(language, config.compiler) - - # Build preprocessor command - flags = _build_preprocessor_flags(config, language) - command = [compiler] + flags + [str(source_path)] - + raise PreprocessingError( + "Compiler preprocessing not configured", + category="INVALID_COMPILER_ARGUMENTS", + ) + source = Path(source_path) + invocation = build_preprocess_invocation(source, language=language, config=config) + executable = invocation.argv[0] if invocation.argv else "" + if executable and os.sep not in executable and shutil.which(executable) is None: + raise PreprocessingError( + f"preprocessor not found: {executable}", + category="PREPROCESSOR_NOT_FOUND", + diagnostics=[ + PreprocessingDiagnostic( + category="PREPROCESSOR_NOT_FOUND", + message=f"preprocessor not found: {executable}", + command=list(invocation.argv), + ) + ], + ) try: - result = subprocess.run( - command, + completed = subprocess.run( + invocation.argv, + cwd=invocation.cwd, capture_output=True, text=True, timeout=60, check=False, ) - except FileNotFoundError: + except FileNotFoundError as exc: + raise PreprocessingError( + f"preprocessor not found: {invocation.argv[0]}", + category="PREPROCESSOR_NOT_FOUND", + diagnostics=[ + PreprocessingDiagnostic( + category="PREPROCESSOR_NOT_FOUND", + message=f"preprocessor not found: {invocation.argv[0]}", + command=list(invocation.argv), + ) + ], + ) from exc + except subprocess.TimeoutExpired as exc: + raise PreprocessingError( + "compiler preprocessing failed: timed out after 60 seconds", + category="PREPROCESSOR_FAILED", + diagnostics=[ + PreprocessingDiagnostic( + category="PREPROCESSOR_FAILED", + message="compiler preprocessing timed out after 60 seconds", + command=list(invocation.argv), + ) + ], + ) from exc + except OSError as exc: + raise PreprocessingError( + f"failed to run compiler preprocessor: {exc}", + category="PREPROCESSOR_FAILED", + diagnostics=[ + PreprocessingDiagnostic( + category="PREPROCESSOR_FAILED", + message=f"failed to run compiler preprocessor: {exc}", + command=list(invocation.argv), + ) + ], + ) from exc + + if completed.returncode != 0: + stderr = completed.stderr.strip() + message = f"compiler preprocessing failed with exit code {completed.returncode}" + if stderr: + message = f"{message}\n{stderr}" raise PreprocessingError( - f"Compiler not found: {compiler}\n" - f"Ensure {compiler} is installed and in your PATH" + message, + category="PREPROCESSOR_FAILED", + diagnostics=[ + PreprocessingDiagnostic( + category="PREPROCESSOR_FAILED", + message=stderr or message, + command=list(invocation.argv), + ) + ], ) - except subprocess.TimeoutExpired: - raise PreprocessingError(f"Compiler preprocessing timed out after 60 seconds") - except Exception as e: - raise PreprocessingError(f"Failed to run preprocessor: {e}") - - if result.returncode != 0: + + expanded_source = completed.stdout + mappings = parse_linemarker_mappings(expanded_source, filename=str(source)) + included_files = _included_files_from_linemarkers( + expanded_source, + root_path=source, + language=language, + config=config, + ) + diagnostics: list[PreprocessingDiagnostic] = [] + if invocation.capabilities.get("linemarkers") is False and not mappings: + diagnostics.append( + PreprocessingDiagnostic( + category="PROVENANCE_UNAVAILABLE", + message="selected compiler adapter did not provide source linemarkers", + severity="warning", + command=list(invocation.argv), + ) + ) + macros = _parse_macro_definitions(expanded_source, mappings) + + if language == "fortran": + expanded_source, native_includes, native_mappings, native_diagnostics = expand_native_fortran_includes( + expanded_source, + root_path=source, + include_dirs=config.include_dirs, + config=config, + ) + included_files.extend(native_includes) + mappings = native_mappings or parse_linemarker_mappings(expanded_source, filename=str(source)) + diagnostics.extend(native_diagnostics) + + result = PreprocessResult( + source=expanded_source, + recipe={}, + included_files=included_files, + source_mappings=mappings, + macros=macros, + diagnostics=diagnostics, + ) + result.recipe = _recipe_from_invocation(source, language, config, invocation, result).to_dict() + if any(diagnostic.severity == "error" for diagnostic in diagnostics): + first = next(diagnostic for diagnostic in diagnostics if diagnostic.severity == "error") raise PreprocessingError( - f"Compiler preprocessing failed with exit code {result.returncode}\n" - f"Command: {' '.join(command)}\n" - f"Error output:\n{result.stderr}" + first.message, + category=first.category, + diagnostics=diagnostics, ) - - # Create recipe for this preprocessing operation + return result + + +def run_compiler_preprocessor_with_recipe( + source_path: Path | str, + language: str, + config: PreprocessingConfig, +) -> tuple[str, PreprocessingRecipe]: + """Run compiler preprocessing and return expanded source plus recipe.""" + + result = preprocess_source(source_path, language=language, config=config) recipe = PreprocessingRecipe( - language=language, - compiler=compiler, - mode="compiler", - include_dirs=config.include_dirs, - defines=config.defines, - undefs=config.undefs, - std=config.std, - compiler_args=config.compiler_args, - source_file=str(source_path), + language=str(result.recipe.get("language")), + compiler=result.recipe.get("compiler") if isinstance(result.recipe.get("compiler"), str) else None, + mode=str(result.recipe.get("mode") or "compiler"), + adapter=str(result.recipe.get("adapter") or "direct"), + argv=list(result.recipe.get("argv") or []), + cwd=result.recipe.get("cwd") if isinstance(result.recipe.get("cwd"), str) else None, + include_dirs=list(result.recipe.get("include_dirs") or []), + defines=list(result.recipe.get("defines") or []), + undefs=list(result.recipe.get("undefs") or []), + standard=result.recipe.get("standard") if isinstance(result.recipe.get("standard"), str) else None, + compiler_args=list(result.recipe.get("compiler_args") or []), + source_path=result.recipe.get("source_path") if isinstance(result.recipe.get("source_path"), str) else None, + compile_commands=result.recipe.get("compile_commands") if isinstance(result.recipe.get("compile_commands"), str) else None, + compile_commands_entry=result.recipe.get("compile_commands_entry") if isinstance(result.recipe.get("compile_commands_entry"), dict) else None, + command_template=result.recipe.get("command_template") if isinstance(result.recipe.get("command_template"), str) else None, + included_files=list(result.recipe.get("included_files") or []), + source_mappings=list(result.recipe.get("source_mappings") or []), + macros=list(result.recipe.get("macros") or []), + diagnostics=list(result.recipe.get("diagnostics") or []), + capabilities=dict(result.recipe.get("capabilities") or {}), ) - - return result.stdout, recipe + return result.source, recipe + + +def run_compiler_preprocessor( + source_path: Path | str, + language: str, + config: PreprocessingConfig, +) -> str: + source, _recipe = run_compiler_preprocessor_with_recipe(source_path, language, config) + return source + + +__all__ = ( + "CommandTemplateAdapter", + "CompilerAdapter", + "GCCCompatibleCAdapter", + "GNUFortranAdapter", + "IncludedFile", + "Invocation", + "MacroDefinition", + "PreprocessResult", + "PreprocessingConfig", + "PreprocessingDiagnostic", + "PreprocessingError", + "PreprocessingPlan", + "PreprocessingRecipe", + "SourceMapping", + "build_compile_commands_invocation", + "build_direct_preprocess_invocation", + "build_preprocess_invocation", + "build_template_preprocess_invocation", + "expand_native_fortran_includes", + "parse_linemarker_mappings", + "preprocess_source", + "run_compiler_preprocessor", + "run_compiler_preprocessor_with_recipe", + "validate_macro_name", +) diff --git a/x2py/preprocessing_test_example.md b/x2py/preprocessing_test_example.md index da8926892..103666b97 100644 --- a/x2py/preprocessing_test_example.md +++ b/x2py/preprocessing_test_example.md @@ -1,154 +1,68 @@ -# Compiler Preprocessing Implementation for x2py +# Compiler-Backed Preprocessing Notes -## Summary -Implemented `x2py/preprocessing.py` module that provides compiler-based preprocessing for both Fortran and C code. This solves the issue of handling includes in both parsers by leveraging the actual compiler's preprocessor. +`x2py/preprocessing.py` owns compiler-backed preprocessing for the wrapper +pipeline. The parsers consume one expanded source stream; they do not evaluate +CPP branches or emulate macro expansion. -## Key Components +## Pipeline -### 1. PreprocessingRecipe (Dataclass) -- Metadata about how a source file was preprocessed -- Tracks: language, compiler, mode, include dirs, defines, undefs, std, compiler args, source file -- Can be serialized to JSON for reporting - -### 2. PreprocessingConfig (Dataclass) -- Configuration for preprocessing operations -- Supports both "internal" (lightweight) and "compiler" (full preprocessing) modes -- Methods: - - `uses_compiler`: Check if compiler preprocessing is enabled - - `fortran_macro_defines()`: Extract macros for Fortran parser - - `fortran_internal_recipe()`: Generate recipe metadata for internal mode - -### 3. Core Functions - -#### `validate_macro_name(macro_str, context)` -- Validates macro definitions have valid identifiers -- Prevents invalid macro syntax - -#### `_get_compiler_for_language(language, compiler)` -- Determines which compiler to use (gfortran for Fortran, gcc for C) -- Respects user-specified compiler path - -#### `_build_preprocessor_flags(config, language)` -- Builds compiler command-line flags: - - `-E` flag for preprocessing only - - Include directories (`-I`) - - Macro definitions (`-D`) - - Macro undefs (`-U`) - - Language standard (`-std`) - - Custom compiler arguments - -#### `run_compiler_preprocessor_with_recipe(source_path, language, config)` -- Main entry point for preprocessing -- Runs the compiler with `-E` flag to: - - Resolve all `#include` (C) and `include` (Fortran) statements - - Expand all macros with provided defines/undefs - - Handle include paths from `-I` flags -- Returns both: - - Preprocessed source code (ready for parser) - - Preprocessing recipe (metadata about the operation) -- Error handling: - - Checks if compiler exists in PATH - - Validates compiler exit code - - Provides helpful error messages - - Timeout protection (60 seconds) - -## How It Works - -### Workflow for Fortran -1. User calls parser with `--preprocess compiler --compiler gfortran-12 -I include -D USE_MPI` -2. `PreprocessingConfig` is created with these settings -3. `run_compiler_preprocessor_with_recipe()` is called -4. `gfortran-12 -E -I include -D USE_MPI=1 source.f90` is executed -5. Compiler: - - Resolves all `include "file.inc"` statements - - Expands `USE_MPI` macro in the code - - Outputs fully expanded source -6. Preprocessed source is fed to Fortran parser -7. Parser now has complete type information from includes -8. Recipe metadata is attached to the parsed output - -### Workflow for C -1. Similar to Fortran but uses `gcc` or `clang` -2. `gcc -E -I include -D API_EXPORT= source.h` is executed -3. Compiler: - - Resolves all `#include "header.h"` and `#include ` - - Expands macros like `API_EXPORT` - - Outputs fully expanded C code -4. Preprocessed source is fed to C parser -5. Parser has access to all type definitions from headers - -## Integration with Existing x2py Code - -The module is already imported in `x2py/cli.py` (lines 20-25): -```python -from x2py.preprocessing import ( - PreprocessingConfig, - PreprocessingError, - run_compiler_preprocessor_with_recipe, - validate_macro_name, -) +```text +source path + -> build preprocessing invocation + -> run compiler adapter + -> collect expanded stdout, linemarkers, macros, include files, diagnostics + -> expand remaining native Fortran INCLUDE statements + -> parse expanded source once ``` -It's actively used in: -- `_fortran_source_for_path()`: Preprocesses Fortran sources -- `_c_source_loader()`: Preprocesses C sources -- `_build_preprocessing_config()`: Validates and builds config -- CLI argument parsing: Handles `--preprocess`, `--compiler`, `-I`, `-D`, `-U`, `--std` +The compiler is authoritative for `#include`, `#define`, `#if`/`#ifdef`, +predefined macros, `-D`, `-U`, include paths, target flags, and sysroot +behavior. GNU Fortran does not preprocess files referenced by native Fortran +`include "file.inc"`, so x2py expands those textually after compiler CPP +output. -## Example Usage +## Main Models -### Command Line (Fortran) -```bash -python -m x2py mycode.f90 --parse --preprocess compiler --compiler gfortran-12 -I ./include -D USE_MPI -``` +- `PreprocessingConfig`: user/compiler configuration, adapter selection, + include exposure controls, and passthrough compiler arguments. +- `Invocation`: exact argv/cwd sent to the compiler adapter. +- `PreprocessResult`: expanded source plus recipe, included files, source + mappings, macro metadata, and preprocessing diagnostics. +- `IncludedFile`: include graph edge with mechanism, system/project + classification, and public/private exposure. +- `SourceMapping`: generated line to original file/line and include stack. +- `MacroDefinition`: active macro metadata when the adapter output exposes it. -### Command Line (C) -```bash -python -m x2py api.h --language c --parse --preprocess compiler --compiler gcc-13 -I ./include -D API_EXPORT= -``` - -### Programmatic (Python) -```python -from pathlib import Path -from x2py.preprocessing import PreprocessingConfig, run_compiler_preprocessor_with_recipe +## Adapters -config = PreprocessingConfig( - mode="compiler", - compiler="gfortran-12", - include_dirs=["./include"], - defines=["USE_MPI"], -) +Built-in adapters cover GCC-compatible C/Clang (`-E -x c`), GNU Fortran +(`-E -cpp`), compile database ingestion, and custom command templates for other +compiler families. A custom template must write expanded source to stdout: -source, recipe = run_compiler_preprocessor_with_recipe( - Path("mycode.f90"), - language="fortran", - config=config, -) - -print(f"Preprocessed with: {recipe.compiler}") -print(source) # Fully expanded source with all includes resolved +```bash +python -m x2py include/api.h --language c --parse \ + --preprocess compiler \ + --preprocessor-adapter command-template \ + --preprocess-template 'vendor-cc --preprocess {include_dirs} {defines} {source}' ``` -## Benefits +## Diagnostics -1. **Complete Macro Expansion**: Uses the compiler's actual preprocessor, ensuring correct macro behavior -2. **Include Resolution**: All `#include` and `include` statements are fully resolved -3. **Standard Compliance**: Respects language standards (C99, C11, F95, F2008, etc.) -4. **Flexible**: Supports custom compiler paths, flags, and compile_commands.json -5. **Metadata Tracking**: Records how preprocessing was done for reproducibility -6. **Error Handling**: Clear error messages when compiler is not found or preprocessing fails -7. **Works with Both Languages**: Same API for Fortran and C preprocessing +Preprocessing errors use explicit categories and are printed by the CLI without +a traceback unless `--debug` is used: -## What Happens to Include Files +- `PREPROCESSOR_NOT_FOUND` +- `PREPROCESSOR_FAILED` +- `INVALID_COMPILER_ARGUMENTS` +- `UNSUPPORTED_COMPILER_CAPABILITY` +- `PROVENANCE_UNAVAILABLE` +- `INCLUDE_NOT_FOUND` +- `INCLUDE_CYCLE` -When the compiler preprocesses code with includes: -- `include "file.inc"` or `#include "file.h"` statements are replaced with the actual contents of those files -- All macros in included files are expanded -- Line markers (`#line` directives) are inserted to track original locations -- The wrapper can now parse the complete, expanded code +## Include Exposure -## Status -✅ Implementation complete on branch `feature/compiler-preprocessing` -✅ Ready for integration into Fortran and C parsers -✅ Integration already exists in x2py/cli.py -✅ All CLI flags and error handling implemented +Root files and reachable project includes are public by default; system headers +are private. Use `--include-exposure roots-only`, `--public-include`, and +`--private-include` to control wrapper export. Private declarations remain +available to resolve public signatures, and private C handle types can be +emitted as opaque classes. From 4e628468cde322cc60486995c7e4d73fcf5d10c4 Mon Sep 17 00:00:00 2001 From: said Date: Sun, 31 May 2026 07:40:53 +0100 Subject: [PATCH 08/13] handle compiler extensions --- README.md | 8 + c_parser/lexer.py | 71 ++- c_parser/parser.py | 499 +++++++++++++++++- docs/c_parser/c_parser_architecture.md | 27 +- docs/c_parser/c_parser_cli_workflow.md | 21 +- docs/c_parser/c_parser_reference.md | 27 +- docs/diagnostic_codes.md | 1 + tests/parser/c/test_c_compiler_extensions.py | 212 ++++++++ .../c/test_c_declarations_and_declarators.py | 32 +- tests/parser/c/test_c_functions.py | 13 +- 10 files changed, 844 insertions(+), 67 deletions(-) create mode 100644 tests/parser/c/test_c_compiler_extensions.py diff --git a/README.md b/README.md index 8e855b8d7..e602d58ed 100644 --- a/README.md +++ b/README.md @@ -204,6 +204,14 @@ available internally for type resolution. Public signatures that refer to private C handle types can use private opaque classes rather than exposing data members. +The C parser tolerates common compiler-expanded declaration syntax from system +headers, including GNU attributes, `__declspec(...)`, alternate qualifier +spellings, declaration-level `asm(...)`, calling-convention keywords, +`typeof(...)`, `_BitInt(...)`, and selected extended scalar names. Harmless +syntax is accepted without exposing private header declarations. Ignored +extensions that can affect ABI, layout, symbol identity, or type identity +produce `C_UNMODELED_COMPILER_EXTENSION` warnings. + Preprocessing failures print explicit categories such as `PREPROCESSOR_NOT_FOUND`, `PREPROCESSOR_FAILED`, `INVALID_COMPILER_ARGUMENTS`, `UNSUPPORTED_COMPILER_CAPABILITY`, diff --git a/c_parser/lexer.py b/c_parser/lexer.py index 0523418ee..1869ad671 100644 --- a/c_parser/lexer.py +++ b/c_parser/lexer.py @@ -93,6 +93,7 @@ class CTopLevelSegment: _LINE_DIRECTIVE_RE = re.compile( r'^\s*#\s*line\s+(?P\d+)(?:\s+(?:"(?P(?:[^"\\]|\\.)*)"|(?P\S+)))?' ) +_AGGREGATE_HEADER_ATTRIBUTE_RE = re.compile(r"\b(?:__attribute__?|__declspec(?:__)?)\b") def _source_line(lines: list[str], line_number: int) -> str | None: @@ -317,8 +318,56 @@ def top_level_partition(text: str, delimiter: str = "=") -> tuple[str, str | Non return text.strip(), None -def _is_aggregate_definition_header(header: str) -> bool: +def _balanced_invocation_end(text: str, open_index: int) -> int | None: + """Return the offset after one balanced parenthesized invocation.""" + depth = 0 + quote = "" + escaped = False + for index in range(open_index, len(text)): + char = text[index] + if quote: + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif char == quote: + quote = "" + continue + if char in {'"', "'"}: + quote = char + elif char == "(": + depth += 1 + elif char == ")": + depth -= 1 + if depth == 0: + return index + 1 + return None + + +def _strip_aggregate_header_attributes(header: str) -> str: + """Blank attributes that can appear between an aggregate keyword and tag.""" + characters = list(header) + for match in _AGGREGATE_HEADER_ATTRIBUTE_RE.finditer(header): + end = match.end() + open_index = end + while open_index < len(header) and header[open_index].isspace(): + open_index += 1 + if open_index < len(header) and header[open_index] == "(": + end = _balanced_invocation_end(header, open_index) or end + for index in range(match.start(), end): + if characters[index] != "\n": + characters[index] = " " + return "".join(characters) + + +def _is_aggregate_definition_header( + header: str, + *, + tolerate_compiler_extensions: bool = False, +) -> bool: """Identify a tag definition before deciding that a brace starts a body.""" + if tolerate_compiler_extensions: + header = _strip_aggregate_header_attributes(header) compact = " ".join(header.split()) if "(" in compact or "=" in compact: return False @@ -326,9 +375,19 @@ def _is_aggregate_definition_header(header: str) -> bool: return any(word in {"struct", "union", "enum"} for word in words) -def _is_braced_declaration_header(header: str) -> bool: +def _is_braced_declaration_header( + header: str, + *, + tolerate_compiler_extensions: bool = False, +) -> bool: """Return whether a brace belongs to a declaration preserved through `;`.""" - return _is_aggregate_definition_header(header) or top_level_partition(header, "=")[1] is not None + return ( + _is_aggregate_definition_header( + header, + tolerate_compiler_extensions=tolerate_compiler_extensions, + ) + or top_level_partition(header, "=")[1] is not None + ) def split_top_level_c_source( @@ -337,6 +396,7 @@ def split_top_level_c_source( *, skip_preprocessor: bool = True, use_linemarkers: bool = False, + tolerate_compiler_extensions: bool = False, ) -> list[CTopLevelSegment]: """Split C source into top-level declarations and definition headers.""" stripped = strip_c_comments(source) @@ -420,7 +480,10 @@ def split_top_level_c_source( block_start_line = start_line block_start_column = start_column block_source_line = start_mapping.source_line - braced_declaration = _is_braced_declaration_header(header) + braced_declaration = _is_braced_declaration_header( + header, + tolerate_compiler_extensions=tolerate_compiler_extensions, + ) brace_depth = 1 if not braced_declaration: start_index = None diff --git a/c_parser/parser.py b/c_parser/parser.py index 7c0ded12c..198e6d19e 100644 --- a/c_parser/parser.py +++ b/c_parser/parser.py @@ -39,7 +39,7 @@ """ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, replace import re from collections.abc import Mapping, Sequence from pathlib import Path, PurePosixPath @@ -108,6 +108,46 @@ _STORAGE_CLASSES = {"typedef", "extern", "static", "register", "_Thread_local"} _TYPE_QUALIFIERS = {"const", "restrict", "volatile", "_Atomic"} _FUNCTION_SPECIFIERS = {"inline", "_Noreturn"} +_STORAGE_CLASS_ALIASES = {"_tls": "_Thread_local"} +_COMPILER_KEYWORD_NORMALIZATIONS = { + "__thread": "_tls", + "__const": "const", + "__const__": "const", + "__restrict": "restrict", + "__restrict__": "restrict", + "__volatile": "volatile", + "__volatile__": "volatile", + "__inline": "inline", + "__inline__": "inline", + "__forceinline": "inline", + "__signed": "signed", + "__signed__": "signed", +} +_EXTENDED_SCALAR_NORMALIZATIONS = { + "__int8": "_xi8", + "__int16": "_xi16", + "__int32": "_xi32", + "__int64": "_xi64", + "__int128": "_xi128", + "__int128_t": "_xi128t", + "__uint128_t": "_xu128t", + "__fp16": "_xhf16", + "_Float16": "_xf16", + "_Float32": "_xf32", + "_Float32x": "_xf32x", + "_Float64": "_xf64", + "_Float64x": "_xf64x", + "_Float128": "_xf128", + "_Decimal32": "_xd32", + "_Decimal64": "_xd64", + "_Decimal128": "_xd128", +} +_COMPILER_KEYWORD_NORMALIZATIONS.update(_EXTENDED_SCALAR_NORMALIZATIONS) +_EXTENDED_SCALAR_SPELLINGS = { + normalized: spelling + for spelling, normalized in _EXTENDED_SCALAR_NORMALIZATIONS.items() +} +_EXTENDED_SCALAR_WORDS = set(_EXTENDED_SCALAR_SPELLINGS) _TAG_KINDS = {"struct", "union", "enum"} _UNSUPPORTED_DECLARATION_MARKERS = ( "__attribute__", @@ -119,6 +159,48 @@ _RAW_CONDITIONAL_DIRECTIVE_RE = re.compile( r"^\s*#\s*(?Pif|ifdef|ifndef|elif|else|endif)\b" ) +_GNU_ATTRIBUTE_KEYWORDS = {"__attribute", "__attribute__"} +_DECLSPEC_KEYWORDS = {"__declspec", "__declspec__"} +_ASM_KEYWORDS = {"asm", "__asm", "__asm__"} +_TYPEOF_KEYWORDS = {"typeof", "__typeof", "__typeof__"} +_ALIGNMENT_KEYWORDS = {"_Alignas", "alignas"} +_IGNORABLE_EXTENSION_KEYWORDS = {"__extension__"} +_CALLING_CONVENTION_KEYWORDS = { + "__cdecl", + "__fastcall", + "__regcall", + "__stdcall", + "__thiscall", + "__vectorcall", +} +_IGNORABLE_DECLARATION_KEYWORDS = { + "__w64", +} +_ABI_DECLARATION_KEYWORDS = { + "__ptr32", + "__ptr64", + "__unaligned", +} +_ABI_SIGNIFICANT_ATTRIBUTE_NAMES = { + "alias", + "align", + "aligned", + "cdecl", + "fastcall", + "ifunc", + "mode", + "ms_abi", + "packed", + "regcall", + "stdcall", + "sysv_abi", + "thiscall", + "thread", + "transparent_union", + "vector_size", + "vectorcall", +} +_ATTRIBUTE_NAME_RE = re.compile(r"[A-Za-z_]\w*") _PRIMITIVE_WORDS = { "void", "char", @@ -216,6 +298,16 @@ class _ParsedDeclarator: source_text: str = "" +@dataclass(frozen=True) +class _UnmodeledCompilerExtension: + """Ignored compiler syntax whose semantics can matter to generated wrappers.""" + + kind: str + name: str + offset: int + message: str + + class _UnsupportedDeclaratorSyntax(ValueError): """Raised internally when a declarator has unconsumed syntax.""" @@ -374,6 +466,7 @@ def visit_file( source, filename, use_linemarkers=True, + normalize_compiler_extensions=True, ) parsed.functions = functions parsed.structs = structs @@ -1233,13 +1326,17 @@ def _parse_specifiers(self, spec_text: str) -> tuple[CType, list[str], list[str] function_specifiers: list[str] = [] type_words: list[str] = [] - for word in words: - if word in _STORAGE_CLASSES: - storage.append(word) - elif word in _TYPE_QUALIFIERS: - qualifiers.append(word) - elif word in _FUNCTION_SPECIFIERS: - function_specifiers.append(word) + for raw_word in words: + word = self._canonical_primitive_word(raw_word) + storage_class = self._canonical_storage_class(word) + qualifier = self._canonical_type_qualifier(word) + function_specifier = self._canonical_function_specifier(word) + if storage_class is not None: + storage.append(storage_class) + elif qualifier is not None: + qualifiers.append(qualifier) + elif function_specifier is not None: + function_specifiers.append(function_specifier) else: type_words.append(word) @@ -1277,13 +1374,29 @@ def _parse_specifiers(self, spec_text: str) -> tuple[CType, list[str], list[str] tag_kwargs["is_incomplete"] = True type_: CType = tag_type(**tag_kwargs) elif type_words: - spelling = " ".join(type_words) + displayed_type_words = [ + _EXTENDED_SCALAR_SPELLINGS.get(word, word) + for word in type_words + ] + spelling = " ".join(displayed_type_words) primitive = _PRIMITIVE_TYPE_SIGNATURES.get(tuple(sorted(type_words))) if primitive is not None: type_ = primitive( qualifiers=self._qualifiers(qualifiers), source_text=" ".join([*qualifiers, *type_words]), ) + elif ( + sum(word in _EXTENDED_SCALAR_WORDS for word in type_words) == 1 + and all( + word in _EXTENDED_SCALAR_WORDS | {"signed", "unsigned", "_Complex"} + for word in type_words + ) + ): + type_ = CUnknownType( + spelling=spelling, + qualifiers=self._qualifiers(qualifiers), + source_text=" ".join([*qualifiers, *displayed_type_words]), + ) elif len(type_words) == 1 and type_words[0] not in _PRIMITIVE_WORDS: type_ = CTypedef( name=type_words[0], @@ -1320,6 +1433,331 @@ def _read_identifier(self, text: str, index: int) -> tuple[str, int] | None: end += 1 return text[index:end], end + @staticmethod + def _canonical_storage_class(word: str) -> str | None: + """Return the standard spelling for a recognized storage class.""" + if word in _STORAGE_CLASSES: + return word + return _STORAGE_CLASS_ALIASES.get(word) + + @staticmethod + def _canonical_type_qualifier(word: str) -> str | None: + """Return the standard spelling for a recognized type qualifier.""" + return word if word in _TYPE_QUALIFIERS else None + + @staticmethod + def _canonical_function_specifier(word: str) -> str | None: + """Return the standard spelling for a recognized function specifier.""" + return word if word in _FUNCTION_SPECIFIERS else None + + @staticmethod + def _canonical_primitive_word(word: str) -> str: + """Normalize alternate compiler spellings for primitive type words.""" + return word + + @staticmethod + def _blank_span(characters: list[str], start: int, end: int) -> None: + """Blank one syntax span while retaining line and column accounting.""" + for index in range(start, end): + if characters[index] != "\n": + characters[index] = " " + + @staticmethod + def _replace_span(characters: list[str], start: int, end: int, replacement: str) -> None: + """Replace a syntax span with a short token and pad the remaining width.""" + writable = [index for index in range(start, end) if characters[index] != "\n"] + if len(replacement) > len(writable): + replacement = "_T" + for index, char in zip(writable, replacement): + characters[index] = char + for index in writable[len(replacement) :]: + characters[index] = " " + + @staticmethod + def _significant_attribute_names(payload: str) -> list[str]: + """Return ABI- or linkage-relevant attribute names from one payload.""" + names: list[str] = [] + for raw_name in _ATTRIBUTE_NAME_RE.findall(payload): + name = raw_name.strip("_") + if name in _ABI_SIGNIFICANT_ATTRIBUTE_NAMES and name not in names: + names.append(name) + return names + + def _attribute_extension_facts( + self, + payload: str, + *, + offset: int, + ) -> list[_UnmodeledCompilerExtension]: + """Describe ignored attributes whose semantics remain wrapper-relevant.""" + return [ + _UnmodeledCompilerExtension( + kind="compiler_attribute", + name=name, + offset=offset, + message=( + f"Compiler attribute {name!r} was accepted for parsing but " + "its ABI or linkage semantics are not modeled." + ), + ) + for name in self._significant_attribute_names(payload) + ] + + @staticmethod + def _find_double_bracket_end(text: str, start: int) -> int | None: + """Return the end offset after one C23/C++-style `[[...]]` attribute.""" + index = start + 2 + quote = "" + escaped = False + while index < len(text) - 1: + char = text[index] + if quote: + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif char == quote: + quote = "" + index += 1 + continue + if char in {'"', "'"}: + quote = char + index += 1 + continue + if text[index : index + 2] == "]]": + return index + 2 + index += 1 + return None + + def _compiler_extension_invocation_end(self, text: str, start: int) -> int: + """Return the end of a keyword's optional balanced parenthesized payload.""" + open_index = self._skip_whitespace(text, start) + if open_index >= len(text) or text[open_index] != "(": + return start + close_index = self._find_matching_delimiter(text, open_index, "(", ")") + return close_index + 1 if close_index is not None else start + + def _normalize_compiler_extensions( + self, + text: str, + ) -> tuple[str, list[_UnmodeledCompilerExtension]]: + """Remove tolerated compiler syntax while retaining source coordinates. + + The parser extracts wrapper-facing C types, not compiler attribute + semantics. Harmless syntax is blanked before grammar parsing. Extensions + that can affect ABI, layout, or symbol identity also produce warnings. + """ + characters = list(text) + extensions: list[_UnmodeledCompilerExtension] = [] + index = 0 + state = "normal" + quote = "" + + while index < len(text): + char = text[index] + nxt = text[index + 1] if index + 1 < len(text) else "" + if state == "line_comment": + if char == "\n": + state = "normal" + index += 1 + continue + if state == "block_comment": + if char == "*" and nxt == "/": + state = "normal" + index += 2 + else: + index += 1 + continue + if state in {"string", "char"}: + if char == "\\" and nxt: + index += 2 + continue + if char == quote: + state = "normal" + quote = "" + index += 1 + continue + if char == "/" and nxt == "/": + state = "line_comment" + index += 2 + continue + if char == "/" and nxt == "*": + state = "block_comment" + index += 2 + continue + if char in {'"', "'"}: + state = "string" if char == '"' else "char" + quote = char + index += 1 + continue + + if text[index : index + 2] == "[[": + span_end = self._find_double_bracket_end(text, index) + if span_end is not None: + extensions.extend( + self._attribute_extension_facts( + text[index + 2 : span_end - 2], + offset=index, + ) + ) + self._blank_span(characters, index, span_end) + index = span_end + continue + + identifier = self._read_identifier(text, index) + if identifier is None: + index += 1 + continue + word, word_end = identifier + + if word in _COMPILER_KEYWORD_NORMALIZATIONS: + self._replace_span( + characters, + index, + word_end, + _COMPILER_KEYWORD_NORMALIZATIONS[word], + ) + index = word_end + continue + + if word in _GNU_ATTRIBUTE_KEYWORDS | _DECLSPEC_KEYWORDS: + span_end = self._compiler_extension_invocation_end(text, word_end) + if span_end > word_end: + extensions.extend( + self._attribute_extension_facts( + text[word_end:span_end], + offset=index, + ) + ) + else: + span_end = word_end + self._blank_span(characters, index, span_end) + index = span_end + continue + + if word in _ALIGNMENT_KEYWORDS: + span_end = self._compiler_extension_invocation_end(text, word_end) + if span_end <= word_end: + span_end = word_end + extensions.append( + _UnmodeledCompilerExtension( + kind="alignment_specifier", + name=word, + offset=index, + message=( + f"Alignment specifier {word!r} was accepted for parsing " + "but its layout semantics are not modeled." + ), + ) + ) + self._blank_span(characters, index, span_end) + index = span_end + continue + + if word in _ASM_KEYWORDS: + payload_start = self._skip_whitespace(text, word_end) + while True: + qualifier = self._read_identifier(text, payload_start) + if qualifier is None or qualifier[0] not in {"goto", "volatile", "__volatile", "__volatile__"}: + break + payload_start = self._skip_whitespace(text, qualifier[1]) + span_end = self._compiler_extension_invocation_end(text, payload_start) + if span_end <= payload_start: + span_end = word_end + extensions.append( + _UnmodeledCompilerExtension( + kind="asm_label", + name=word, + offset=index, + message=( + "Assembler label syntax was accepted for parsing but " + "the alternate native symbol identity is not modeled." + ), + ) + ) + self._blank_span(characters, index, span_end) + index = span_end + continue + + if word in _TYPEOF_KEYWORDS or word == "_BitInt": + span_end = self._compiler_extension_invocation_end(text, word_end) + if span_end > word_end: + placeholder = "_typeof" if word in _TYPEOF_KEYWORDS else "_bitint" + extensions.append( + _UnmodeledCompilerExtension( + kind="compiler_type", + name=word, + offset=index, + message=( + f"Compiler type expression {word!r} was accepted as " + "an opaque type placeholder." + ), + ) + ) + self._replace_span(characters, index, span_end, placeholder) + index = span_end + continue + + if word in _CALLING_CONVENTION_KEYWORDS: + extensions.append( + _UnmodeledCompilerExtension( + kind="calling_convention", + name=word, + offset=index, + message=( + f"Calling convention {word!r} was accepted for parsing " + "but its ABI semantics are not modeled." + ), + ) + ) + self._blank_span(characters, index, word_end) + index = word_end + continue + + if word in _ABI_DECLARATION_KEYWORDS: + extensions.append( + _UnmodeledCompilerExtension( + kind="compiler_qualifier", + name=word, + offset=index, + message=( + f"Compiler qualifier {word!r} was accepted for parsing " + "but its ABI semantics are not modeled." + ), + ) + ) + self._blank_span(characters, index, word_end) + index = word_end + continue + + if word in _IGNORABLE_EXTENSION_KEYWORDS | _IGNORABLE_DECLARATION_KEYWORDS: + self._blank_span(characters, index, word_end) + index = word_end + continue + + index = word_end + + return "".join(characters), extensions + + def _normalized_extension_segment( + self, + segment: CTopLevelSegment, + ) -> tuple[CTopLevelSegment, list[CDiagnostic]]: + """Normalize one segment and report semantically significant omissions.""" + normalized, extensions = self._normalize_compiler_extensions(segment.text) + diagnostics = [ + CDiagnostic( + code="C_UNMODELED_COMPILER_EXTENSION", + message=extension.message, + severity="warning", + location=self._source_location_at(segment, extension.offset), + unit_kind=extension.kind, + unit_name=extension.name, + ) + for extension in extensions + ] + return replace(segment, text=normalized), diagnostics + def _find_matching_delimiter( self, text: str, @@ -1389,12 +1827,16 @@ def _split_declaration_specifiers(self, text: str) -> tuple[str, str]: spec_end = index continue - if word in _STORAGE_CLASSES or word in _TYPE_QUALIFIERS or word in _FUNCTION_SPECIFIERS: + if ( + self._canonical_storage_class(word) is not None + or self._canonical_type_qualifier(word) is not None + or self._canonical_function_specifier(word) is not None + ): index = end spec_end = end continue - if word in _PRIMITIVE_WORDS: + if self._canonical_primitive_word(word) in _PRIMITIVE_WORDS or word in _EXTENDED_SCALAR_WORDS: consumed_type = True index = end spec_end = end @@ -1441,9 +1883,10 @@ def _parse_pointer_ops( if identifier is None: break word, end = identifier - if word not in _TYPE_QUALIFIERS: + qualifier = self._canonical_type_qualifier(word) + if qualifier is None: break - qualifiers.append(word) + qualifiers.append(qualifier) index = end pointers.append(_PointerOp(qualifiers=qualifiers)) index = self._skip_whitespace(text, index) @@ -1458,8 +1901,8 @@ def _parse_array_op(self, content: str) -> _ArrayOp: for word in words: if word == "static": is_static = True - elif word in _TYPE_QUALIFIERS: - qualifiers.append(word) + elif self._canonical_type_qualifier(word) is not None: + qualifiers.append(self._canonical_type_qualifier(word)) else: remaining.append(word) normalized = " ".join(remaining) @@ -1763,10 +2206,14 @@ def _raise_for_unsupported_old_style_definitions( filename: str | None, *, use_linemarkers: bool = False, + normalize_compiler_extensions: bool = False, ) -> None: """Raise before top-level splitting hides unsupported K&R declarations.""" source_lines = source.splitlines() - stripped_lines = strip_c_comments(source).splitlines() + normalized_source = source + if normalize_compiler_extensions: + normalized_source, _extensions = self._normalize_compiler_extensions(source) + stripped_lines = strip_c_comments(normalized_source).splitlines() line_mappings = line_mappings_for_source( source, filename=filename, @@ -1919,10 +2366,16 @@ def _tag_definition_header(self, text: str) -> tuple[str, list[str], str | None] for index, word in enumerate(words): if word not in _TAG_KINDS: continue - prefix = words[:index] + prefix: list[str] = [] + for prefix_word in words[:index]: + normalized = ( + self._canonical_storage_class(prefix_word) + or self._canonical_type_qualifier(prefix_word) + ) + if normalized is None: + return None + prefix.append(normalized) suffix = words[index + 1 :] - if any(item not in _STORAGE_CLASSES | _TYPE_QUALIFIERS for item in prefix): - return None if len(suffix) > 1: return None return word, prefix, suffix[0] if suffix else None @@ -2489,6 +2942,7 @@ def _parse_translation_unit( object_like_macros: set[str] | None = None, use_linemarkers: bool = False, condition_sets_by_line: Mapping[int, frozenset[str]] | None = None, + normalize_compiler_extensions: bool = False, ) -> tuple[ list[CFunction], list[CStruct], @@ -2509,6 +2963,7 @@ def _parse_translation_unit( source, filename, use_linemarkers=use_linemarkers, + normalize_compiler_extensions=normalize_compiler_extensions, ) functions: list[CFunction] = [] @@ -2526,7 +2981,13 @@ def _parse_translation_unit( source, filename=filename, use_linemarkers=use_linemarkers, + tolerate_compiler_extensions=normalize_compiler_extensions, ): + if normalize_compiler_extensions: + segment, extension_diagnostics = self._normalized_extension_segment(segment) + diagnostics.extend(extension_diagnostics) + if not segment.text.strip(): + continue self._raise_for_invalid_top_level_syntax(segment) macro_dependency = self._segment_macro_dependency( segment, diff --git a/docs/c_parser/c_parser_architecture.md b/docs/c_parser/c_parser_architecture.md index 4a39acfa3..27643a76a 100644 --- a/docs/c_parser/c_parser_architecture.md +++ b/docs/c_parser/c_parser_architecture.md @@ -64,11 +64,16 @@ Implemented now: parameter declarations preserve their written `declared_type` and expose pointer-adjusted effective `type` values. Declarations prefixed by unexpanded object-like macros are deferred as macro dependencies rather than - misreported as invalid type sequences. Selected unsupported declaration - forms, including attributes, alignment specifiers and static assertions, are - reported as diagnostics with explicit `unit_kind` values. A declarator must - be fully consumed before a concrete object is returned; unknown suffixes - become diagnostics. Grammar-invalid input raises `CParseError` with + misreported as invalid type sequences. In compiler/preprocessed mode, common + GCC/Clang and MS declaration syntax is normalized before grammar parsing: + attributes, `__declspec(...)`, + `[[...]]`, `__extension__`, alternate qualifier/inline spellings, + declaration-level `asm(...)`, calling-convention keywords, `typeof(...)`, + `_BitInt(...)`, and selected extended scalar names. Ignored extension + semantics that can affect ABI, layout, symbol identity, or type identity + produce `C_UNMODELED_COMPILER_EXTENSION` warnings. Static assertions remain + diagnostic-only. A declarator must be fully consumed before a concrete object + is returned; unknown suffixes become diagnostics. Grammar-invalid input raises `CParseError` with `CPARSE_INVALID_SYNTAX`; identifier spellings are not used to guess another language. Primitive specifier order is normalized, and invalid combinations such as `unsigned float` raise `CParseError` with code @@ -115,7 +120,9 @@ Deferred: - full typedef/tag resolution policy beyond basic project-level link-up and callback policy metadata, for example conflict diagnostics, active semantic wrappability decisions -- compiler attributes and alignment specifiers +- semantic modeling for compiler attributes, alignment, calling conventions, + assembler aliases, opaque compiler type expressions, and extended scalar ABI + facts beyond the accepted declaration syntax - broader compiler-family validation for preprocessing; parsed declarations already retain preprocessed origin and mapped source identity - broader C callback/ownership policy beyond exact starter `.pyi` stubs @@ -264,7 +271,8 @@ Current and planned responsibilities: record folding for backslash-newline, string/character literal awareness, lightweight tokens with source locations, preprocessed `#line`/linemarker remapping for top-level segments, top-level splitting with block end - locations, and delimiter splitting aware of nesting and literals. + locations, aggregate-header attribute tolerance during brace + classification, and delimiter splitting aware of nesting and literals. - Planned: richer token helpers as extension and initializer-expression parsing require them. - `c_parser/preprocessor.py` @@ -289,7 +297,10 @@ Current and planned responsibilities: `CPARSE_INVALID_SPECIFIER_SEQUENCE`. Array and function parameters preserve `declared_type` while effective `type` uses C parameter adjustment. Raw declarations beginning with an object-like macro - name are retained as macro-dependent diagnostics. + name are retained as macro-dependent diagnostics. Compiler/preprocessed mode + normalizes common GCC/Clang and MS declaration extensions before grammar + parsing; ignored ABI-relevant semantics produce + `C_UNMODELED_COMPILER_EXTENSION` warnings. - Planned: symbol resolution and additional declaration-specifier and extension coverage. - `c_parser/project.py` diff --git a/docs/c_parser/c_parser_cli_workflow.md b/docs/c_parser/c_parser_cli_workflow.md index 4ee7ed677..761b04dc9 100644 --- a/docs/c_parser/c_parser_cli_workflow.md +++ b/docs/c_parser/c_parser_cli_workflow.md @@ -77,10 +77,15 @@ function index. The object class distinguishes declarations (`CFunction`, `CVariable`, `CTypedef`, `CStruct`, `CUnion`, or `CEnum`), and incomplete tag declarations set `is_incomplete=True`. -Known unsupported declaration forms such as declaration attributes, alignment -specifiers, and static assertions are -reported in diagnostics with explicit `unit_kind` values; unconsumed declarator -suffixes are diagnosed instead of silently omitted. Invalid flexible array +In compiler/preprocessed mode, common GCC/Clang and MS declaration syntax is +normalized before grammar parsing: attributes, `__declspec(...)`, `[[...]]`, +`__extension__`, alternate qualifier/inline spellings, declaration-level +`asm(...)`, calling-convention +keywords, `typeof(...)`, `_BitInt(...)`, and selected extended scalar names. +Ignored extension semantics that can affect ABI, layout, symbol identity, or +type identity produce `C_UNMODELED_COMPILER_EXTENSION` warnings. Static +assertions and remaining unsupported declarator suffixes are diagnosed instead +of silently omitted. Invalid flexible array member placement and flexible union members produce `C_INVALID_FLEXIBLE_ARRAY_MEMBER` error diagnostics at the field location. The parser reports `parser_status: "partial"`. C parse diagnostics, currently including @@ -647,6 +652,9 @@ The active CLI/parser tests cover the current partial subset: by focused C tests. - array and function parameter declared/effective type adjustment is covered by focused C tests. +- common GNU/MS declaration extension normalization, ABI-relevant omission + warnings, linemarker preservation, and a GCC-preprocessed standard-header + smoke test are covered by focused C tests. - `--show-vars` and `--print-limit` are rejected in C mode until C-specific display controls exist. - `--semantics`, `--wrap-readiness`, and `--pyi` with `--language c` use @@ -703,7 +711,10 @@ Completed order: without hard-coded host type aliases. 23. Added C semantic IR, readiness, and starter exact-contract `.pyi` output through `x2py --language c`. +24. Added declaration-normalization tolerance for common GCC/Clang and MS + compiler extensions, explicit warnings for ABI-relevant omissions, and + GCC-preprocessed standard-header regression coverage. -Next implementation work should continue with fixture-driven compiler +Next implementation work should continue with broader fixture-driven compiler extension policy and broader project conflict policy while keeping the explicit `--language c` gate in place. diff --git a/docs/c_parser/c_parser_reference.md b/docs/c_parser/c_parser_reference.md index e243088a7..01e60448c 100644 --- a/docs/c_parser/c_parser_reference.md +++ b/docs/c_parser/c_parser_reference.md @@ -160,6 +160,10 @@ The supported subset focuses on stable wrapper-relevant APIs: - simple object-like numeric and string macros - include dependency tracking - cross-file typedef and tag resolution within parsed project files +- compiler/preprocessed-mode tolerance for common GCC/Clang and MS declaration syntax: + GNU attributes, `__declspec(...)`, `[[...]]`, `__extension__`, alternate + qualifier/inline spellings, declaration-level `asm(...)`, calling-convention + keywords, `typeof(...)`, `_BitInt(...)`, and selected extended scalar names ## Unsupported And Deferred Subset @@ -171,7 +175,6 @@ The C parser explicitly reports or defers: - token pasting and stringification - macro-generated declarations - complex conditional compilation evaluation -- all compiler extensions - arbitrary GCC extensions - arbitrary MSVC extensions - C++ parsing @@ -182,7 +185,8 @@ The C parser explicitly reports or defers: - inline assembly - `_Generic` semantic evaluation - atomic operation semantics and validation beyond parsed type facts -- arbitrary attributes before fixture-driven support exists +- full semantic modeling of compiler attributes, calling conventions, assembler + aliases, `typeof(...)`, `_BitInt(...)`, and extended scalar ABI facts ## Preprocessing Policy @@ -399,10 +403,14 @@ Member records carry their own field location. A legal final incomplete array member in a struct is marked as `CArray(is_flexible=True)`; non-final, sole-member, and union incomplete-array member forms are retained with `C_INVALID_FLEXIBLE_ARRAY_MEMBER` error diagnostics. -Selected unsupported forms, such as static assertions, attributes, and -alignment specifiers, are reported in `diagnostics` with explicit `unit_kind` -values. Grammar-invalid input raises `CParseError`; identifier spellings are not -used to guess that input belongs to another language. +In compiler/preprocessed mode, common compiler declaration syntax is normalized +before grammar parsing. +Harmless attributes are accepted without dropping their declarations. Ignored +extensions that can affect layout, calling convention, symbol identity, or type +identity produce `C_UNMODELED_COMPILER_EXTENSION` warnings with explicit +`unit_kind` values. Static assertions remain diagnostic-only. Grammar-invalid +input raises `CParseError`; identifier spellings are not used to guess that +input belongs to another language. Unconsumed declarator suffixes are also diagnosed instead of producing partial objects. Functions include `prototype_style`, currently `"prototype"` for @@ -676,8 +684,9 @@ Active declaration tests currently cover: references - `_Atomic int` and `_Atomic(type)` qualifier placement on scalar and pointer declaration forms -- diagnostics for selected unsupported attributes, alignment, K&R definitions, - and trailing declarator extensions +- tolerance for common GNU/MS declaration extensions, explicit warnings for + unmodeled ABI-relevant extension semantics, and diagnostics for K&R + definitions and remaining trailing declarator extensions - fatal diagnostics for grammar-invalid syntax and invalid primitive-specifier combinations while unresolved single typedef-name uses remain deferred @@ -690,7 +699,7 @@ declarations. | --- | --- | --- | --- | | Typedef/tag resolution | `typedef unsigned long size_t; size_t count(void);` and `struct state { int id; }; void step(struct state *s);` | Basic project parsing links typedef chains and struct/union/enum tag references while preserving unresolved objects when context is absent. | Deepen conflict policy for broader projects; included/generated files are parsed only when supplied as project inputs. | | Preprocessed declarations | `#define API(ret) ret` followed by `API(int) run(void);` | Raw mode records macro metadata and does not claim the expanded declaration; compiler or `.i` mode parses expanded declarations, maps locations through `#line` markers, and records `origin="preprocessed"`; x2py-generated streams also record their recipe. | Broaden fixture-driven extension and compiler-family coverage. | -| Additional extension families | `int run(void) __attribute__((visibility("default")));` | Known attribute/alignment forms are diagnosed; broader compiler extensions are not modeled. | Add fixture-driven support or a focused diagnostic for each required extension family. | +| Additional extension families | `int run(void) __attribute__((visibility("default")));` | Common GNU/MS declaration syntax is accepted; ignored ABI-, layout-, symbol-, or type-relevant semantics produce `C_UNMODELED_COMPILER_EXTENSION`. Broader compiler extensions are not modeled. | Add fixture-driven tolerance or a focused diagnostic for each required extension family. | ### Represented With Focused Tests diff --git a/docs/diagnostic_codes.md b/docs/diagnostic_codes.md index 74b2f7290..90ad2f7f7 100644 --- a/docs/diagnostic_codes.md +++ b/docs/diagnostic_codes.md @@ -77,6 +77,7 @@ These records do not necessarily stop parsing; inspect each diagnostic's | `C_UNRESOLVED_INCLUDE` | A local include could not be resolved. | | `C_UNSUPPORTED_FUNCTION_LIKE_MACRO` | A function-like macro was recorded but not expanded. | | `C_MACRO_DEPENDENT_DECLARATION` | Declaration parsing requires macro expansion. | +| `C_UNMODELED_COMPILER_EXTENSION` | Compiler syntax was accepted for declaration extraction, but ABI-, layout-, type-, or symbol-relevant extension semantics remain unmodeled. | | `C_UNSUPPORTED_DECLARATION` | Recognized declaration form is outside the modeled subset. | | `C_UNSUPPORTED_DECLARATOR` | Declarator form is outside the modeled subset. | | `C_UNSUPPORTED_FIELD_DECLARATION` | Aggregate field form is outside the modeled subset. | diff --git a/tests/parser/c/test_c_compiler_extensions.py b/tests/parser/c/test_c_compiler_extensions.py new file mode 100644 index 000000000..981a4fc69 --- /dev/null +++ b/tests/parser/c/test_c_compiler_extensions.py @@ -0,0 +1,212 @@ +from __future__ import annotations + +from pathlib import Path +import shutil + +import pytest + + +def test_raw_mode_keeps_compiler_extension_declarations_conservative(): + from c_parser import parse_c_file + + parsed = parse_c_file( + 'int exported(void) __attribute__((visibility("default")));\n', + filename="raw_extensions.h", + preprocessing="raw", + ) + + assert parsed.functions == [] + assert [ + (diagnostic.code, diagnostic.unit_kind) + for diagnostic in parsed.diagnostics + ] == [ + ("C_UNSUPPORTED_DECLARATION", "attribute_declaration"), + ] + + +def test_gnu_header_spelling_aliases_and_harmless_attributes_are_tolerated(): + from c_parser import CComposedType, CConst, CRestrict, parse_c_file + + parsed = parse_c_file( + """ +__extension__ typedef __signed__ long int signed_long; +extern __inline__ int inlined(__const char *__restrict__ input) + __attribute__((__nothrow__, __nonnull__(1))); +__thread int per_thread; +struct annotated { + __const__ char *name __attribute__((deprecated)); +}; +""", + filename="gnu_aliases.h", + preprocessing="compiler", + ) + + assert [typedef.name for typedef in parsed.typedefs] == ["signed_long"] + assert [function.name for function in parsed.functions] == ["inlined"] + assert [variable.name for variable in parsed.variables] == ["per_thread"] + assert parsed.variables[0].storage == ["_Thread_local"] + parameter_type = parsed.functions[0].parameters[0].type + assert isinstance(parameter_type, CComposedType) + assert parameter_type.components[0].qualifiers == [CRestrict()] + assert parameter_type.components[1].qualifiers == [CConst()] + assert [struct.name for struct in parsed.structs] == ["annotated"] + assert [member.name for member in parsed.structs[0].members] == ["name"] + assert parsed.diagnostics == [] + + +def test_layout_and_abi_attributes_are_parsed_with_explicit_warnings(): + from c_parser import parse_c_file + + parsed = parse_c_file( + """ +struct __attribute__((packed)) packet { + int value __attribute__((aligned(8))); +}; +typedef int vector4 __attribute__((__vector_size__(16))); +extern int abi_call(void) __attribute__((ms_abi)); +""", + filename="abi_attributes.h", + preprocessing="compiler", + ) + + assert [struct.name for struct in parsed.structs] == ["packet"] + assert [member.name for member in parsed.structs[0].members] == ["value"] + assert [typedef.name for typedef in parsed.typedefs] == ["vector4"] + assert [function.name for function in parsed.functions] == ["abi_call"] + assert [ + (diagnostic.code, diagnostic.unit_kind, diagnostic.unit_name) + for diagnostic in parsed.diagnostics + ] == [ + ("C_UNMODELED_COMPILER_EXTENSION", "compiler_attribute", "packed"), + ("C_UNMODELED_COMPILER_EXTENSION", "compiler_attribute", "aligned"), + ("C_UNMODELED_COMPILER_EXTENSION", "compiler_attribute", "vector_size"), + ("C_UNMODELED_COMPILER_EXTENSION", "compiler_attribute", "ms_abi"), + ] + + +def test_declspec_calling_conventions_asm_labels_and_top_level_asm_are_tolerated(): + from c_parser import parse_c_file + + parsed = parse_c_file( + """ +__declspec(align(16)) struct block { int value; }; +extern __declspec(dllimport) int __stdcall imported(int value); +__declspec(thread) extern int tls_value; +extern int renamed(void) __asm__("renamed_v2"); +__asm__(".ident \\"compiler metadata\\""); +""", + filename="vendor_extensions.h", + preprocessing="compiler", + ) + + assert [struct.name for struct in parsed.structs] == ["block"] + assert [function.name for function in parsed.functions] == ["imported", "renamed"] + assert [variable.name for variable in parsed.variables] == ["tls_value"] + assert [ + (diagnostic.unit_kind, diagnostic.unit_name) + for diagnostic in parsed.diagnostics + ] == [ + ("compiler_attribute", "align"), + ("calling_convention", "__stdcall"), + ("compiler_attribute", "thread"), + ("asm_label", "__asm__"), + ("asm_label", "__asm__"), + ] + + +def test_typeof_bitint_and_extended_scalars_remain_parseable_as_opaque_types(): + from c_parser import CTypedef, CUnknownType, parse_c_file + + parsed = parse_c_file( + """ +extern __typeof__(errno) errno_alias; +_BitInt(17) bit_counter; +unsigned __int128 wide_counter; +_Float128 wide_float; +""", + filename="compiler_types.h", + preprocessing="compiler", + ) + + variables = {variable.name: variable for variable in parsed.variables} + assert isinstance(variables["errno_alias"].type, CTypedef) + assert variables["errno_alias"].type.name == "_typeof" + assert isinstance(variables["bit_counter"].type, CTypedef) + assert variables["bit_counter"].type.name == "_bitint" + assert isinstance(variables["wide_counter"].type, CUnknownType) + assert variables["wide_counter"].type.spelling == "unsigned __int128" + assert isinstance(variables["wide_float"].type, CUnknownType) + assert variables["wide_float"].type.spelling == "_Float128" + assert [ + (diagnostic.unit_kind, diagnostic.unit_name) + for diagnostic in parsed.diagnostics + ] == [ + ("compiler_type", "__typeof__"), + ("compiler_type", "_BitInt"), + ] + + +def test_preprocessed_extension_diagnostics_and_declarations_use_linemarkers(): + from c_parser import parse_c_file + + parsed = parse_c_file( + """ +# 1 "private_types.h" 1 +struct __attribute__((packed)) wire_value { int value; }; +# 20 "api.h" 2 +extern int consume_wire(struct wire_value *value) + __attribute__((visibility("default"))); +""", + filename="generated.i", + preprocessing="preprocessed", + ) + + assert parsed.structs[0].source_location.filename == "private_types.h" + assert parsed.structs[0].source_location.line == 1 + assert parsed.functions[0].source_location.filename == "api.h" + assert parsed.functions[0].source_location.line == 20 + assert len(parsed.diagnostics) == 1 + assert parsed.diagnostics[0].unit_name == "packed" + assert parsed.diagnostics[0].location.filename == "private_types.h" + assert parsed.diagnostics[0].location.line == 1 + + +def test_gcc_preprocessed_standard_headers_remain_parseable(tmp_path: Path): + from c_parser import parse_c_file + from x2py.preprocessing import PreprocessingConfig, preprocess_source + + compiler = shutil.which("cc") + if compiler is None: + pytest.skip("cc is not available") + + header = tmp_path / "system_api.h" + header.write_text( + """ +#include +#include +#include +#include +int consume_file(FILE *stream); +uint32_t hash_bytes(const void *data, size_t size); +""", + encoding="utf-8", + ) + preprocessed = preprocess_source( + header, + language="c", + config=PreprocessingConfig(mode="compiler", compiler=compiler), + ) + parsed = parse_c_file( + preprocessed.source, + filename=str(header), + preprocessing="compiler", + ) + + function_names = {function.name for function in parsed.functions} + assert {"consume_file", "hash_bytes"} <= function_names + assert len(parsed.typedefs) >= 50 + assert any( + included.dependency_kind == "system" and included.exposure == "private" + for included in preprocessed.included_files + ) + assert not any(diagnostic.severity == "error" for diagnostic in parsed.diagnostics) diff --git a/tests/parser/c/test_c_declarations_and_declarators.py b/tests/parser/c/test_c_declarations_and_declarators.py index 711da5878..c97f759d2 100644 --- a/tests/parser/c/test_c_declarations_and_declarators.py +++ b/tests/parser/c/test_c_declarations_and_declarators.py @@ -548,7 +548,7 @@ def test_recursive_compositions_cover_tables_callback_arrays_and_function_result ] -def test_unimplemented_declaration_extensions_are_diagnosed_not_partially_modeled(): +def test_declaration_attributes_are_tolerated_and_layout_omissions_are_diagnosed(): from c_parser import parse_c_file parsed = parse_c_file( @@ -556,15 +556,17 @@ def test_unimplemented_declaration_extensions_are_diagnosed_not_partially_modele int visible __attribute__((visibility("default"))); int outdated [[deprecated]]; _Alignas(16) int aligned_value; -""", + """, filename="extensions.h", + preprocessing="compiler", ) - assert parsed.variables == [] - assert [diagnostic.unit_kind for diagnostic in parsed.diagnostics] == [ - "attribute_declaration", - "attribute_declaration", - "alignment_declaration", + assert [variable.name for variable in parsed.variables] == ["visible", "outdated", "aligned_value"] + assert [ + (diagnostic.code, diagnostic.unit_kind, diagnostic.unit_name) + for diagnostic in parsed.diagnostics + ] == [ + ("C_UNMODELED_COMPILER_EXTENSION", "alignment_specifier", "_Alignas"), ] @@ -623,19 +625,23 @@ def test_braced_and_designated_initializer_declarations_preserve_source_text(): assert parsed.diagnostics == [] -def test_unconsumed_declarator_suffixes_are_diagnosed_not_silently_discarded(): +def test_asm_declarator_suffixes_are_tolerated_with_symbol_identity_diagnostics(): from c_parser import parse_c_file parsed = parse_c_file( 'extern int retained, pinned asm("r0");\nint run(int value asm("r0"));\n', filename="declarator_extensions.h", + preprocessing="compiler", ) - assert [variable.name for variable in parsed.variables] == ["retained"] - assert parsed.functions == [] - assert [diagnostic.code for diagnostic in parsed.diagnostics] == [ - "C_UNSUPPORTED_DECLARATOR", - "C_UNSUPPORTED_DECLARATOR", + assert [variable.name for variable in parsed.variables] == ["retained", "pinned"] + assert [function.name for function in parsed.functions] == ["run"] + assert [ + (diagnostic.code, diagnostic.unit_kind) + for diagnostic in parsed.diagnostics + ] == [ + ("C_UNMODELED_COMPILER_EXTENSION", "asm_label"), + ("C_UNMODELED_COMPILER_EXTENSION", "asm_label"), ] diff --git a/tests/parser/c/test_c_functions.py b/tests/parser/c/test_c_functions.py index df37d328a..6ab7f77b5 100644 --- a/tests/parser/c/test_c_functions.py +++ b/tests/parser/c/test_c_functions.py @@ -305,23 +305,18 @@ def test_inline_function_body_in_header_is_recorded_as_definition(): assert function.end.line == 1 -def test_function_declaration_attributes_are_diagnosed_until_extension_support_lands(): +def test_function_declaration_attributes_are_tolerated_when_type_shape_is_unchanged(): from c_parser import parse_c_file parsed = parse_c_file( 'int exported(void) __attribute__((visibility("default")));\n' "int deprecated(void) [[deprecated]];\n", filename="function_attributes.h", + preprocessing="compiler", ) - assert parsed.functions == [] - assert [ - (diagnostic.code, diagnostic.unit_kind, diagnostic.location.line) - for diagnostic in parsed.diagnostics - ] == [ - ("C_UNSUPPORTED_DECLARATION", "attribute_declaration", 1), - ("C_UNSUPPORTED_DECLARATION", "attribute_declaration", 2), - ] + assert [function.name for function in parsed.functions] == ["exported", "deprecated"] + assert parsed.diagnostics == [] def test_conflicting_function_prototypes_report_diagnostic(): From f6ef5cb9e65f24a276be24edc3f91525cfdbc1d4 Mon Sep 17 00:00:00 2001 From: said Date: Sun, 31 May 2026 08:55:16 +0100 Subject: [PATCH 09/13] handle compiler extensions --- fortran_parser/parser.py | 19 +- tests/parser/c/test_c_cli_skeleton.py | 33 +++ tests/parser/test_preprocessing_cli.py | 284 +++++++++++++++++++++++++ x2py/cli.py | 17 +- 4 files changed, 332 insertions(+), 21 deletions(-) diff --git a/fortran_parser/parser.py b/fortran_parser/parser.py index c4be1c83b..0f5556cef 100644 --- a/fortran_parser/parser.py +++ b/fortran_parser/parser.py @@ -224,9 +224,8 @@ def replace_symbol(match: re.Match[str]) -> str: class FortranParser: """Stateful parser entrypoint and orchestration object. - State carried on the instance: - - `macro_defines`: accepted for backward-compatible call signatures, but - CPP branch selection is handled by the compiler preprocessing layer. + Raw parser entrypoints preserve all CPP branch alternatives. Branch + selection belongs to the compiler preprocessing layer. Parsing pipeline used by `visit_file`: 1. Preprocess source into normalized lines (`_preprocessed_lines`). @@ -271,10 +270,6 @@ class FortranParser: # Public visitor entrypoints # ------------------------------------------------------------------ - def __init__(self, macro_defines: set[str] | dict[str, int | bool | str] | None = None): - del macro_defines - self.macro_defines = None - def visit_file( self, source_or_path: str | Path, @@ -282,7 +277,11 @@ def visit_file( macro_defines: set[str] | dict[str, int | bool | str] | None = None, encoding: str = "utf-8", ) -> FortranFile: - """Parse one source string/path into a `FortranFile` aggregate model.""" + """Parse one source string/path into a `FortranFile` aggregate model. + + `macro_defines` is a legacy no-op. Raw parsing does not evaluate CPP + branches; use compiler preprocessing when branch selection is needed. + """ if filename is None and self._looks_like_existing_source_path(source_or_path): path = Path(source_or_path) filename = str(path) @@ -290,7 +289,6 @@ def visit_file( else: code = str(source_or_path) - del macro_defines lines, root_scope, top_units = self._helper_prepare_source_units( code, filename, @@ -932,8 +930,6 @@ def _helper_prepare_source_units( self, code: _SourceOrLines, filename: str | None, - *, - macro_defines: set[str] | dict[str, int | bool | str] | None = None, ) -> tuple[_PreprocessedLines, _ParserScope, list[_SourceUnit]]: """Preprocess, validate, and slice file-level source units. @@ -947,7 +943,6 @@ def _helper_prepare_source_units( `_SourceUnit` objects, one module unit and one procedure unit, both carrying original source line numbers. """ - del macro_defines lines = self._preprocessed_lines(code, filename) root_scope = _ParserScope(kind="file", name=None) units = self._helper_slice_child_units(lines, parent_scope=root_scope, filename=filename) diff --git a/tests/parser/c/test_c_cli_skeleton.py b/tests/parser/c/test_c_cli_skeleton.py index a98005f1d..a512fef14 100644 --- a/tests/parser/c/test_c_cli_skeleton.py +++ b/tests/parser/c/test_c_cli_skeleton.py @@ -80,6 +80,39 @@ def test_cli_c_parse_json_reports_raw_preprocessor_metadata(tmp_path: Path): assert file_payload["diagnostics"][0]["code"] == "C_UNSUPPORTED_FUNCTION_LIKE_MACRO" +def test_attach_preprocessing_recipe_filters_invalid_and_duplicate_macros(): + empty = c_parser_cli.CFile() + c_parser_cli.attach_preprocessing_recipe(empty, None) + assert empty.preprocessing_recipe is None + + parsed = c_parser_cli.CFile( + macros=[ + c_parser_cli.CMacro( + name="EXISTING", + source_location=c_parser_cli.CSourceLocation(filename="api.h", line=2), + ) + ] + ) + recipe = { + "macros": [ + None, + {"name": ""}, + {"name": "EXISTING", "path": "api.h", "line": 2}, + {"name": "NEW", "value": 123, "function_like": 1, "path": 42, "line": "bad"}, + {"name": "WITH_LOC", "value": "1", "path": "api.h", "line": 4}, + ] + } + + c_parser_cli.attach_preprocessing_recipe(parsed, recipe) + + assert parsed.preprocessing_recipe == recipe + assert [macro.name for macro in parsed.macros] == ["EXISTING", "NEW", "WITH_LOC"] + assert parsed.macros[1].value is None + assert parsed.macros[1].function_like is True + assert parsed.macros[1].source_location.filename is None + assert parsed.macros[2].source_location.line == 4 + + def test_cli_c_parse_json_out_writes_file_and_suppresses_stdout(tmp_path: Path): header = tmp_path / "api.h" output = tmp_path / "report.json" diff --git a/tests/parser/test_preprocessing_cli.py b/tests/parser/test_preprocessing_cli.py index d00711861..f53925c07 100644 --- a/tests/parser/test_preprocessing_cli.py +++ b/tests/parser/test_preprocessing_cli.py @@ -125,11 +125,93 @@ def test_preprocessing_config_internal_macros_recipe_and_validation(tmp_path: Pa selected = PreprocessingConfig(defines=["USE_MPI", "VALUE=3"], undefs=["DEBUG"]) assert plain.uses_compiler is False + assert plain.fortran_macro_defines() is None + assert PreprocessingConfig(mode="compiler", defines=["USE_MPI"]).fortran_macro_defines() is None assert plain.fortran_internal_recipe(source) is None assert selected.fortran_macro_defines() == {"USE_MPI": 1, "VALUE": "3", "DEBUG": 0} assert selected.fortran_internal_recipe(source)["source_path"] == str(source) with pytest.raises(PreprocessingError, match="requires a macro name"): validate_macro_name("=value", "--define") + with pytest.raises(PreprocessingError, match="requires a macro name"): + validate_macro_name("", "--define") + with pytest.raises(PreprocessingError, match="invalid macro name"): + validate_macro_name("bad-name", "--define") + + +def test_preprocessing_metadata_models_and_adapter_helpers(tmp_path: Path): + source = tmp_path / "api.c" + source.write_text("int api(void);\n", encoding="utf-8") + diagnostic = preprocessing.PreprocessingDiagnostic( + category="PREPROCESSOR_FAILED", + message="bad flag", + path=str(source), + line=3, + command=["cc", "-E"], + ) + included = preprocessing.IncludedFile( + path=str(tmp_path / "public.h"), + included_by=str(source), + include_line=1, + ) + mapping = preprocessing.SourceMapping( + generated_line=2, + original_path=str(source), + original_line=7, + include_stack=[str(source)], + ) + macro = preprocessing.MacroDefinition( + name="SQR", + value="((x) * (x))", + function_like=True, + parameters=["x"], + path=str(source), + line=4, + ) + plan = preprocessing.PreprocessingPlan( + language="c", + source_path=str(source), + adapter="direct", + compiler="cc", + include_dirs=["include"], + defines=["API=1"], + undefs=["DEBUG"], + standard="c11", + compiler_args=["-Wall"], + ) + result = preprocessing.PreprocessResult( + source="#define SQR(x) ((x) * (x))\n", + recipe={"mode": "compiler"}, + included_files=[included], + source_mappings=[mapping], + macros=[macro], + diagnostics=[diagnostic], + ) + recipe = preprocessing.PreprocessingRecipe(language="c", compiler="cc", standard="c11") + + assert diagnostic.to_dict()["command"] == ["cc", "-E"] + assert plan.to_dict()["include_dirs"] == ["include"] + assert result.to_dict()["macros"][0]["parameters"] == ["x"] + assert recipe.std == "c11" + + adapter = preprocessing.GCCCompatibleCAdapter() + config = PreprocessingConfig(mode="compiler", compiler="cc") + assert adapter.build_preprocess_invocation(source, language="c", config=config).argv[0] == "cc" + assert adapter.collect_dependencies(result) == [included] + assert adapter.collect_macros(result) == [macro] + assert adapter.parse_linemarkers('#line 7 "dir\\\\api\\".h"\nint x;\n')[0].original_line == 7 + + invocation = preprocessing.CommandTemplateAdapter().build_preprocess_invocation( + source, + language="c", + config=PreprocessingConfig( + mode="compiler", + compiler="vendor-cc", + adapter="command-template", + command_template="{compiler} --lang {language} {source}", + ), + ) + assert invocation.argv == ["vendor-cc", "--lang", "c", str(source)] + assert preprocessing.GNUFortranAdapter().name == "gnu-fortran" def test_direct_preprocess_invocation_rejects_missing_compiler_and_unknown_language(tmp_path: Path): @@ -197,6 +279,8 @@ def test_compile_commands_invocation_uses_database_compiler_and_filters_compile_ "multiple compile_commands entries", ), ('[{"directory": ".", "arguments": ["cc"]}]', "missing 'file'"), + ('[{"directory": ".", "file": "api.c", "arguments": "cc -c api.c"}]', "'arguments' must contain a list"), + ('[{"directory": ".", "file": "api.c", "command": ["cc"]}]', "'command' must contain a string"), ('[{"directory": ".", "file": "api.c"}]', "must contain 'arguments' or 'command'"), ('[{"directory": ".", "file": "api.c", "arguments": []}]', "empty command"), ], @@ -239,6 +323,44 @@ def test_compile_commands_invocation_reports_missing_file_and_supports_command_s assert invocation.argv == ["clang", "-E", str(source)] +def test_compile_commands_filters_dependency_and_windows_compile_flags(tmp_path: Path): + source = tmp_path / "src" / "api.c" + source.parent.mkdir() + source.write_text("int api(void);\n", encoding="utf-8") + compiler = tmp_path / "cc" + database = tmp_path / "compile_commands.json" + database.write_text( + json.dumps( + [ + { + "directory": str(tmp_path), + "file": str(source), + "arguments": [ + str(compiler), + "-MF", + "deps.d", + "-MT", + "api.o", + "-MQtarget", + "-MFdeps2.d", + "/c", + "src/api.c", + "-Wall", + ], + } + ] + ), + encoding="utf-8", + ) + + invocation = build_compile_commands_invocation( + source, + config=PreprocessingConfig(mode="compiler", compile_commands=str(database)), + ) + + assert invocation.argv == [str(compiler), "-E", "-Wall", str(source)] + + def test_build_preprocess_invocation_supports_fortran_compile_database(tmp_path: Path): source = tmp_path / "solver.F90" source.write_text("subroutine solve()\nend subroutine solve\n", encoding="utf-8") @@ -309,6 +431,89 @@ def test_command_template_preprocess_invocation_expands_placeholders(tmp_path: P ] +def test_command_template_validation_and_dispatch_edges(tmp_path: Path): + source = tmp_path / "api.h" + source.write_text("int api(void);\n", encoding="utf-8") + + with pytest.raises(PreprocessingError, match="requires --preprocess-template"): + build_template_preprocess_invocation( + source, + language="c", + config=PreprocessingConfig(mode="compiler", adapter="command-template"), + ) + with pytest.raises(PreprocessingError, match="expanded to an empty command"): + build_template_preprocess_invocation( + source, + language="c", + config=PreprocessingConfig( + mode="compiler", + adapter="command-template", + command_template="''", + ), + ) + + invocation = build_preprocess_invocation( + source, + language="c", + config=PreprocessingConfig( + mode="compiler", + compiler="vendor-cc", + adapter="command-template", + command_template="{compiler} {language} --std={standard} {source}", + std="c99", + ), + ) + + assert invocation.argv == ["vendor-cc", "c", "--std=c99", str(source)] + + +def test_linemarker_dependency_exposure_and_macro_edges(tmp_path: Path): + root = tmp_path / "root.c" + source = "\n".join( + [ + '#line 7 "src\\\\api\\".h"', + "int from_line;", + '# 1 "" 1 3', + "#define BUILTIN 1", + '# 2 "" 2', + '# 2 "src/api.h" 2', + '# 1 "public/api.h" 1', + "int public_api;", + '# 1 "private/internal.h" 1', + "int private_api;", + '# 1 "project/hidden.h" 1', + "int hidden_api;", + ] + ) + + mappings = preprocessing.parse_linemarker_mappings(source, filename=str(root)) + macros = preprocessing._parse_macro_definitions(source, mappings) + files = preprocessing._included_files_from_linemarkers( + source, + root_path=root, + language="c", + config=PreprocessingConfig( + include_exposure="roots-only", + public_includes=["public"], + private_includes=["private"], + ), + ) + by_path = {item.path: item for item in files} + + assert mappings[0].original_line == 7 + assert 'api".h' in mappings[0].original_path + assert preprocessing._unescape_linemarker_filename("trailing\\") == "trailing\\" + assert preprocessing._dependency_kind("") == "system" + assert preprocessing._mapping_for_generated_line(mappings, mappings[0].generated_line, root) == mappings[0] + assert macros[0].name == "BUILTIN" + assert macros[0].builtin is True + assert by_path[str(root)].dependency_kind == "root" + assert by_path[""].dependency_kind == "system" + assert by_path["public/api.h"].exposure == "public" + assert by_path["private/internal.h"].exposure == "private" + assert by_path["project/hidden.h"].exposure == "private" + + def test_native_fortran_include_expansion_is_recursive_and_preserves_duplicates(tmp_path: Path): root = tmp_path / "src" / "root.F90" include = root.parent / "decls.inc" @@ -400,6 +605,85 @@ def raise_oserror(*_args, **_kwargs): run_compiler_preprocessor(source, language="c", config=config) +def test_preprocess_source_error_paths_and_fortran_include_diagnostics(monkeypatch, tmp_path: Path): + c_source = tmp_path / "api.c" + c_source.write_text("int api(void);\n", encoding="utf-8") + + with pytest.raises(PreprocessingError, match="not configured"): + preprocessing.preprocess_source(c_source, language="c", config=PreprocessingConfig()) + with pytest.raises(PreprocessingError, match="preprocessor not found"): + preprocessing.preprocess_source( + c_source, + language="c", + config=PreprocessingConfig(mode="compiler", compiler="x2py-definitely-missing-preprocessor"), + ) + + def raise_file_not_found(*_args, **_kwargs): + raise FileNotFoundError("missing") + + monkeypatch.setattr(preprocessing.subprocess, "run", raise_file_not_found) + with pytest.raises(PreprocessingError, match="preprocessor not found"): + preprocessing.preprocess_source( + c_source, + language="c", + config=PreprocessingConfig(mode="compiler", compiler=str(tmp_path / "missing-cc")), + ) + + def raise_timeout(*_args, **_kwargs): + raise subprocess.TimeoutExpired(cmd="cc", timeout=60) + + monkeypatch.setattr(preprocessing.subprocess, "run", raise_timeout) + with pytest.raises(PreprocessingError, match="timed out"): + preprocessing.preprocess_source( + c_source, + language="c", + config=PreprocessingConfig(mode="compiler", compiler=str(tmp_path / "slow-cc")), + ) + + monkeypatch.setattr( + preprocessing.subprocess, + "run", + lambda *_args, **_kwargs: type("Done", (), {"returncode": 2, "stdout": "", "stderr": ""})(), + ) + with pytest.raises(PreprocessingError, match="exit code 2"): + preprocessing.preprocess_source( + c_source, + language="c", + config=PreprocessingConfig(mode="compiler", compiler=str(tmp_path / "bad-cc")), + ) + + monkeypatch.setattr( + preprocessing.subprocess, + "run", + lambda *_args, **_kwargs: type("Done", (), {"returncode": 0, "stdout": "", "stderr": ""})(), + ) + result = preprocessing.preprocess_source( + c_source, + language="c", + config=PreprocessingConfig( + mode="compiler", + adapter="command-template", + command_template=f"{sys.executable} {{source}}", + ), + ) + assert [diagnostic.category for diagnostic in result.diagnostics] == ["PROVENANCE_UNAVAILABLE"] + + fortran_source = tmp_path / "solver.F90" + fortran_source.write_text('include "missing.inc"\n', encoding="utf-8") + monkeypatch.setattr( + preprocessing.subprocess, + "run", + lambda *_args, **_kwargs: type("Done", (), {"returncode": 0, "stdout": 'include "missing.inc"\n', "stderr": ""})(), + ) + with pytest.raises(PreprocessingError) as exc_info: + preprocessing.preprocess_source( + fortran_source, + language="fortran", + config=PreprocessingConfig(mode="compiler", compiler=str(tmp_path / "gfortran")), + ) + assert exc_info.value.category == "INCLUDE_NOT_FOUND" + + def test_cli_help_documents_exact_compiler_and_preprocessing_examples(): res = subprocess.run( [sys.executable, "-m", "x2py", "--help"], diff --git a/x2py/cli.py b/x2py/cli.py index 1fa9ba444..6b9b94f06 100644 --- a/x2py/cli.py +++ b/x2py/cli.py @@ -147,17 +147,16 @@ def language_for_suffix(suffix: str) -> str | None: def _fortran_source_for_path( path: Path, preprocessing: PreprocessingConfig, -) -> tuple[str, dict[str, int | str] | None, dict[str, object] | None]: +) -> tuple[str, dict[str, object] | None]: if preprocessing.uses_compiler: source, recipe = run_compiler_preprocessor_with_recipe( path, language="fortran", config=preprocessing, ) - return source, None, recipe.to_dict() + return source, recipe.to_dict() return ( path.read_text(encoding="utf-8"), - None, preprocessing.fortran_internal_recipe(path), ) @@ -211,8 +210,8 @@ def _parse_report(paths: list[str], preprocessing: PreprocessingConfig | None = out: dict[str, dict] = {} parser = FortranParser() for p in _expand_paths(paths): - code, macro_defines, preprocessing_recipe = _fortran_source_for_path(p, preprocessing) - parsed = parser.visit_file(code, filename=str(p), macro_defines=macro_defines) + code, preprocessing_recipe = _fortran_source_for_path(p, preprocessing) + parsed = parser.visit_file(code, filename=str(p)) payload = { "signatures": [_to_dict_no_parent(s) for s in parsed.procedures], "types": [_to_dict_no_parent(t) for t in parsed.derived_types], @@ -251,8 +250,8 @@ def _semantic_report( parser = FortranParser() for p in _expand_paths(paths): - code, macro_defines, _preprocessing_recipe = _fortran_source_for_path(p, preprocessing) - fobj = parser.visit_file(code, filename=str(p), macro_defines=macro_defines) + code, _preprocessing_recipe = _fortran_source_for_path(p, preprocessing) + fobj = parser.visit_file(code, filename=str(p)) compile_time_values = _fortran_compile_time_values(fobj, preprocessing) modules = [ fortran_module_to_semantic_module(m, compile_time_values=compile_time_values) @@ -300,8 +299,8 @@ def _wrap_readiness_report( modules = [load_pyi_file(p)] source_kind = "pyi" else: - code, macro_defines, _preprocessing_recipe = _fortran_source_for_path(p, preprocessing) - parsed = parser.visit_file(code, filename=str(p), macro_defines=macro_defines) + code, _preprocessing_recipe = _fortran_source_for_path(p, preprocessing) + parsed = parser.visit_file(code, filename=str(p)) compile_time_values = _fortran_compile_time_values(parsed, preprocessing) modules = fortran_file_to_semantic_modules( parsed, From e5962f8cba345a24e41e10548813fe9c70343748 Mon Sep 17 00:00:00 2001 From: said Date: Sun, 31 May 2026 09:23:26 +0100 Subject: [PATCH 10/13] clean --- fortran_parser/parser.py | 9 +-------- tests/parser/test_error_handling.py | 5 ++--- tests/parser/test_parser_public_entrypoints.py | 5 +++++ .../test_preprocessor_and_execution_boundaries.py | 6 +++--- tests/parser/test_procedure_and_type_parsing.py | 10 ++++------ 5 files changed, 15 insertions(+), 20 deletions(-) diff --git a/fortran_parser/parser.py b/fortran_parser/parser.py index 0f5556cef..3e6e9edd3 100644 --- a/fortran_parser/parser.py +++ b/fortran_parser/parser.py @@ -274,14 +274,9 @@ def visit_file( self, source_or_path: str | Path, filename: str | None = None, - macro_defines: set[str] | dict[str, int | bool | str] | None = None, encoding: str = "utf-8", ) -> FortranFile: - """Parse one source string/path into a `FortranFile` aggregate model. - - `macro_defines` is a legacy no-op. Raw parsing does not evaluate CPP - branches; use compiler preprocessing when branch selection is needed. - """ + """Parse one source string/path into a `FortranFile` aggregate model.""" if filename is None and self._looks_like_existing_source_path(source_or_path): path = Path(source_or_path) filename = str(path) @@ -4304,13 +4299,11 @@ def _is_executable_statement_start(line: str) -> bool: def parse_fortran_file( source_or_path: str | Path, filename: str | None = None, - macro_defines: set[str] | dict[str, int | bool | str] | None = None, encoding: str = "utf-8", ) -> FortranFile: return _DEFAULT_PARSER.visit_file( source_or_path, filename=filename, - macro_defines=macro_defines, encoding=encoding, ) diff --git a/tests/parser/test_error_handling.py b/tests/parser/test_error_handling.py index 5dede0138..452449ad8 100644 --- a/tests/parser/test_error_handling.py +++ b/tests/parser/test_error_handling.py @@ -253,7 +253,7 @@ def test_duplicate_procedure_name_in_mutually_exclusive_macro_branches_allowed() assert all(sig.name.lower() == "work" for sig in procedures) -def test_macro_defines_do_not_select_active_branch_in_parser(): +def test_ifdef_macro_branch_is_not_selected_by_parser(): code = """ module m #ifdef USE_MPI @@ -270,7 +270,7 @@ def test_macro_defines_do_not_select_active_branch_in_parser(): #endif end module m """ - parsed = parse_fortran_file(code, filename="macro_alt_work.f90", macro_defines={"USE_MPI"}) + parsed = parse_fortran_file(code, filename="macro_alt_work.f90") procedures = parsed.modules[0].procedures assert len(procedures) == 2 assert {proc.kind for proc in procedures} == {"subroutine", "function"} @@ -296,7 +296,6 @@ def test_if_defined_macro_expression_is_not_evaluated_by_parser(): parsed = parse_fortran_file( code, filename="macro_if_expr.f90", - macro_defines={"USE_MPI": 1, "USE_SERIAL": 0}, ) procedures = parsed.modules[0].procedures assert len(procedures) == 2 diff --git a/tests/parser/test_parser_public_entrypoints.py b/tests/parser/test_parser_public_entrypoints.py index d9892418d..f186a7002 100644 --- a/tests/parser/test_parser_public_entrypoints.py +++ b/tests/parser/test_parser_public_entrypoints.py @@ -81,6 +81,11 @@ def test_file_path_and_unknown_filename_public_parse_paths(tmp_path): assert parsed_from_path.procedures[0].name == "from_path" assert parsed_unknown_suffix.format == "unknown" +@pytest.mark.parametrize("parse", [parse_fortran_file, FortranParser().visit_file]) +def test_public_file_parse_rejects_removed_macro_defines_argument(parse): + with pytest.raises(TypeError, match="macro_defines"): + parse("", macro_defines={"USE_MPI"}) + def test_public_instance_visitor_entrypoints_use_source_strings(): parser = FortranParser() diff --git a/tests/parser/test_preprocessor_and_execution_boundaries.py b/tests/parser/test_preprocessor_and_execution_boundaries.py index 4972b9d4b..b0bb5b6e5 100644 --- a/tests/parser/test_preprocessor_and_execution_boundaries.py +++ b/tests/parser/test_preprocessor_and_execution_boundaries.py @@ -97,7 +97,7 @@ def test_cpp_directives_are_preserved_without_parser_branch_selection(): #endif """ - parsed = parse_fortran_file(code, macro_defines={"USE_FAST": True}) + parsed = parse_fortran_file(code) assert [proc.name for proc in parsed.procedures] == [ "fast_path", @@ -248,7 +248,7 @@ def test_cpp_false_and_malformed_expressions_are_not_evaluated_by_parser(): #endif """ - parsed = parse_fortran_file(code, macro_defines=set()) + parsed = parse_fortran_file(code) assert [proc.name for proc in parsed.procedures] == [ "false_if_branch", @@ -289,7 +289,7 @@ def test_preprocessor_boolean_identifiers_are_not_evaluated_by_public_parse(): end subroutine after_stray_directives """ - parsed = parse_fortran_file(code, filename="cpp_edges.f90", macro_defines={"USE_FAST": True}) + parsed = parse_fortran_file(code, filename="cpp_edges.f90") assert [proc.name for proc in parsed.procedures] == [ "selected_fast", diff --git a/tests/parser/test_procedure_and_type_parsing.py b/tests/parser/test_procedure_and_type_parsing.py index a6f122d82..1caf842aa 100644 --- a/tests/parser/test_procedure_and_type_parsing.py +++ b/tests/parser/test_procedure_and_type_parsing.py @@ -236,7 +236,7 @@ def test_fixed_form_and_interface_detection(): assert parsed.interfaces[0].procedures[0].in_interface is True -def test_preprocessor_macro_defines_do_not_select_active_branch_from_inline_fortran(): +def test_preprocessor_branches_are_preserved_from_inline_fortran(): source = """ #ifdef USE_A subroutine selected_a(x) @@ -253,12 +253,10 @@ def test_preprocessor_macro_defines_do_not_select_active_branch_from_inline_fort #endif """ - selected = parse_fortran_file(source, macro_defines={"USE_B": True}) - fallback = parse_fortran_file(source, macro_defines={"USE_A": False, "USE_B": False}) + parsed = parse_fortran_file(source) - assert [proc.name for proc in selected.procedures] == ["selected_a", "selected_b", "fallback"] - assert [proc.arguments[0].base_type for proc in selected.procedures] == ["integer", "real", "logical"] - assert [proc.name for proc in fallback.procedures] == ["selected_a", "selected_b", "fallback"] + assert [proc.name for proc in parsed.procedures] == ["selected_a", "selected_b", "fallback"] + assert [proc.arguments[0].base_type for proc in parsed.procedures] == ["integer", "real", "logical"] def test_legacy_character_and_star_kind_declarations_from_inline_fortran(): From 65b0a0c50d7d11246bbc9c26a9154d76201b360f Mon Sep 17 00:00:00 2001 From: said Date: Sun, 31 May 2026 11:58:08 +0100 Subject: [PATCH 11/13] add opaque types --- README.md | 52 +++- c_parser/parser.py | 13 +- docs/c_parser/c_parser_architecture.md | 10 +- docs/c_parser/c_parser_reference.md | 9 +- docs/fortran/fortran_parser.md | 4 +- .../parser_implementation_reference.md | 15 +- docs/semantics/c2ir_mapping.md | 10 +- docs/semantics/pyi_format.md | 39 +++ semantics/__init__.py | 8 +- semantics/c2ir.py | 111 +++++++- semantics/fortran2ir.py | 245 ++++++++++++++++-- semantics/models.py | 31 +++ semantics/pyi_parser.py | 102 +++++++- semantics/pyi_printer.py | 123 ++++++++- semantics/readiness.py | 22 +- tests/parser/c/test_c_cli_skeleton.py | 33 +++ tests/parser/c/test_c_public_api_skeleton.py | 18 ++ tests/parser/test_cli.py | 65 +++++ tests/parser/test_preprocessing_cli.py | 3 - tests/pyi/test_pyi_to_ir.py | 80 +++++- tests/semantics/test_c2ir.py | 73 +++++- tests/semantics/test_fortran2ir.py | 66 +++++ tests/semantics/test_pyi_printer.py | 42 +++ x2py/__init__.py | 8 +- x2py/cli.py | 129 ++++++++- x2py/preprocessing.py | 14 - 26 files changed, 1224 insertions(+), 101 deletions(-) diff --git a/README.md b/README.md index e602d58ed..08a2daf82 100644 --- a/README.md +++ b/README.md @@ -78,11 +78,14 @@ generation, and wrap-readiness. Public API entrypoints include: -- `x2py.parse_fortran_file(source_or_path, filename=None, macro_defines=None, encoding="utf-8") -> FortranFile` +- `x2py.parse_fortran_file(source_or_path, filename=None, encoding="utf-8") -> FortranFile` - `x2py.parse_fortran_project(files, encoding="utf-8") -> FortranProject` -- `x2py.parse_c_file(source_or_path, filename=None, macro_defines=None, include_dirs=None, preprocessing="raw", encoding="utf-8") -> CFile` -- `x2py.parse_c_project(files, include_dirs=None, macro_defines=None, preprocessing="raw", encoding="utf-8") -> CProject` +- `x2py.parse_c_file(source_or_path, filename=None, include_dirs=None, preprocessing="raw", encoding="utf-8") -> CFile` +- `x2py.parse_c_project(files, include_dirs=None, preprocessing="raw", encoding="utf-8") -> CProject` - `x2py.fortran_file_to_semantic_modules(parsed_file, standalone_module_name=None) -> list[SemanticModule]` +- `x2py.fortran_project_to_semantic_modules(project) -> list[SemanticModule]` +- `x2py.emit_module_stubs(module_or_modules) -> dict[str, str]` +- `x2py.load_pyi_modules(path_or_paths, encoding="utf-8") -> list[SemanticModule]` - `x2py.assess_semantic_wrap_readiness(semantic_ir, source=None) -> dict` - `x2py.assess_pyi_wrap_readiness(path_or_paths, encoding="utf-8") -> dict` - `x2py.c_type_probe.probe_c_standard_types(config, runner=None) -> CStandardTypeProbeReport` @@ -803,3 +806,46 @@ source/target mapping. A non-renamed `use iso_c_binding, only: c_int` maps `source="delete_input_list"` and `target="delete_input"`. The semantic layer uses that information to emit Python stub imports such as `from list_input import delete_input_list as delete_input`. + +Fortran `use` dependencies are not parsed or wrapped recursively. If a +procedure refers to an imported derived type, semantic IR records its defining +module and represents the reference as an opaque handle unless the defining +module is explicitly part of the wrapping target. Explicitly supplied modules +share one wrapped-type registry, so the imported reference resolves to the +single class emitted by its owner module without being re-exported by the +importing module. Reachable include exposure is already handled separately by +the preprocessing include policy; a future dependency-expansion option would +apply specifically to recursive Fortran `use` traversal. + +When an imported derived type remains external, `.pyi` generation emits an +owner-module dependency stub. For example, wrapping only `physics.f90` may +produce: + +```python +# physics.pyi +from types_mod import particle + +def move(p: Ptr(particle)) -> None: ... +``` + +```python +# types_mod.pyi +class particle(Opaque): + pass +``` + +`python -m x2py physics.f90 --pyi --out` writes both files beside the source. +`load_pyi_modules(...)` loads a file set or directory, preserves opaque classes, +and reconciles imported references against edited owner stubs. Replacing the +opaque placeholder with a concrete edited class changes the semantic reference +from `representation="opaque"` to `representation="wrapped"`. Existing +`Annotated[...]` constraints also round-trip through this editable interface; +richer coercion syntax can be added to the same `.pyi` format later. + +The same opaque-handle file-set model applies to C. A local forward declaration +such as `struct context;` emits `class context(Opaque): pass`. When a public C +header uses a struct from another explicitly supplied header, its generated +stub imports the class from that header's stub. A private included struct used +through a public pointer boundary emits an opaque owner-module dependency stub. +An unresolved C typedef is left unresolved rather than guessed to be opaque, +because its ABI may not be pointer-shaped. diff --git a/c_parser/parser.py b/c_parser/parser.py index 198e6d19e..f8e94fd54 100644 --- a/c_parser/parser.py +++ b/c_parser/parser.py @@ -387,7 +387,6 @@ def visit_file( source_or_path: str | Path, filename: str | None = None, *, - macro_defines: set[str] | dict[str, int | bool | str] | None = None, include_dirs: Sequence[str | Path] | None = None, preprocessing: str = "raw", encoding: str = "utf-8", @@ -396,11 +395,8 @@ def visit_file( The current implementation supports raw preprocessing metadata, compiler-fed preprocessed text, and the partial grammar subset - documented in `docs/c_parser`. `macro_defines` is accepted for API - compatibility with compiler-assisted preprocessing, but raw mode does - not evaluate conditional branches. + documented in `docs/c_parser`. """ - del macro_defines source_path: Path | None = None if _looks_like_existing_source_path(source_or_path): path = Path(source_or_path) @@ -493,7 +489,6 @@ def visit_project( files: Mapping[str, str] | Sequence[str | Path] | str | Path, *, include_dirs: Sequence[str | Path] | None = None, - macro_defines: set[str] | dict[str, int | bool | str] | None = None, preprocessing: str = "raw", encoding: str = "utf-8", ) -> CProject: @@ -509,7 +504,6 @@ def visit_project( source, filename=name, include_dirs=include_dirs, - macro_defines=macro_defines, preprocessing=preprocessing, encoding=encoding, ) @@ -538,7 +532,6 @@ def visit_project( path, filename=key, include_dirs=include_dirs, - macro_defines=macro_defines, preprocessing=preprocessing, encoding=encoding, ) @@ -3288,7 +3281,6 @@ def parse_c_file( source_or_path: str | Path, filename: str | None = None, *, - macro_defines: set[str] | dict[str, int | bool | str] | None = None, include_dirs: Sequence[str | Path] | None = None, preprocessing: str = "raw", encoding: str = "utf-8", @@ -3297,7 +3289,6 @@ def parse_c_file( return _DEFAULT_PARSER.visit_file( source_or_path, filename=filename, - macro_defines=macro_defines, include_dirs=include_dirs, preprocessing=preprocessing, encoding=encoding, @@ -3308,7 +3299,6 @@ def parse_c_project( files: Mapping[str, str] | Sequence[str | Path] | str | Path, *, include_dirs: Sequence[str | Path] | None = None, - macro_defines: set[str] | dict[str, int | bool | str] | None = None, preprocessing: str = "raw", encoding: str = "utf-8", ) -> CProject: @@ -3316,7 +3306,6 @@ def parse_c_project( return _DEFAULT_PARSER.visit_project( files, include_dirs=include_dirs, - macro_defines=macro_defines, preprocessing=preprocessing, encoding=encoding, ) diff --git a/docs/c_parser/c_parser_architecture.md b/docs/c_parser/c_parser_architecture.md index 27643a76a..66352de9f 100644 --- a/docs/c_parser/c_parser_architecture.md +++ b/docs/c_parser/c_parser_architecture.md @@ -330,13 +330,13 @@ Current and planned responsibilities: The public C API mirrors the Fortran style but remains C-specific: ```python -parse_c_file(source_or_path, filename=None, macro_defines=None, include_dirs=None, preprocessing="raw", encoding="utf-8") -> CFile -parse_c_project(files, include_dirs=None, macro_defines=None, preprocessing="raw", encoding="utf-8") -> CProject +parse_c_file(source_or_path, filename=None, include_dirs=None, preprocessing="raw", encoding="utf-8") -> CFile +parse_c_project(files, include_dirs=None, preprocessing="raw", encoding="utf-8") -> CProject ``` -`macro_defines` is accepted for API compatibility only. Raw mode must not -evaluate C preprocessor conditionals or expand macros inside x2py. Compiler -mode receives already-expanded source from the shared preprocessing layer. +Raw mode does not evaluate C preprocessor conditionals or expand macros inside +x2py. Compiler mode receives already-expanded source from the shared +preprocessing layer. Implemented companion class: diff --git a/docs/c_parser/c_parser_reference.md b/docs/c_parser/c_parser_reference.md index 01e60448c..58c7926bc 100644 --- a/docs/c_parser/c_parser_reference.md +++ b/docs/c_parser/c_parser_reference.md @@ -324,7 +324,6 @@ Implemented signatures: parse_c_file( source_or_path, filename=None, - macro_defines=None, include_dirs=None, preprocessing="raw", encoding="utf-8", @@ -333,7 +332,6 @@ parse_c_file( parse_c_project( files, include_dirs=None, - macro_defines=None, preprocessing="raw", encoding="utf-8", ) @@ -478,10 +476,9 @@ such as `{"g1:b0"}` or `{"g1:b1"}`. A `CProject` stores such alternatives in unique `functions` entry. Compiler-preprocessed input contains the selected configuration and therefore does not need this ambiguity representation. -`macro_defines` is accepted for API compatibility only. It must not mean that -raw mode evaluates C preprocessor conditionals or expands macros internally. -Compiler mode should receive the already-expanded translation unit from -`x2py.preprocessing`. +Raw mode does not evaluate C preprocessor conditionals or expand macros +internally. Compiler mode should receive the already-expanded translation unit +from `x2py.preprocessing`. The parser itself should stay parse-only. If the C frontend later gains wrappability assessment, that should live in the semantic layer after C parser diff --git a/docs/fortran/fortran_parser.md b/docs/fortran/fortran_parser.md index b064ef6b3..00aba5cc1 100644 --- a/docs/fortran/fortran_parser.md +++ b/docs/fortran/fortran_parser.md @@ -77,7 +77,7 @@ and practical usage from terminal and Python. Supported public API: -- `parse_fortran_file(source_or_path, filename=None, macro_defines=None, encoding="utf-8") -> FortranFile` +- `parse_fortran_file(source_or_path, filename=None, encoding="utf-8") -> FortranFile` - `parse_fortran_project(files, encoding="utf-8") -> FortranProject` - `assess_semantic_wrap_readiness(semantic_ir, source=None) -> dict` - `assess_pyi_wrap_readiness(path_or_paths, encoding="utf-8") -> dict` @@ -1003,7 +1003,7 @@ Under `implicit none`, these declarations count as valid argument declarations, Use the stable top-level API: -- `parse_fortran_file(source_or_path, filename=None, macro_defines=None, encoding="utf-8") -> FortranFile` +- `parse_fortran_file(source_or_path, filename=None, encoding="utf-8") -> FortranFile` - `parse_fortran_project(files, encoding="utf-8") -> FortranProject` - `assess_semantic_wrap_readiness(semantic_ir, source=None) -> dict` - `assess_pyi_wrap_readiness(path_or_paths, encoding="utf-8") -> dict` diff --git a/docs/fortran/parser_implementation_reference.md b/docs/fortran/parser_implementation_reference.md index 14889e98d..4de571b01 100644 --- a/docs/fortran/parser_implementation_reference.md +++ b/docs/fortran/parser_implementation_reference.md @@ -208,6 +208,16 @@ another source language. - Semantic IR imports use structured `SemanticImport` / `SemanticImportItem` entries when a Fortran `use` has an explicit symbol list; bare imports remain plain module names for compatibility. +- Imported derived types remain external references by default. Semantic type + `external_type_ref` metadata records the defining module, whether that owner + module is explicitly wrapped, and whether the boundary representation is + `opaque` or `wrapped`. The importing module does not emit or re-export a + duplicate class. When the owner is not explicitly wrapped, + `emit_module_stubs(...)` emits an owner-module `class Name(Opaque): pass` + dependency stub. `load_pyi_modules(...)` restores the file set and reconciles + an edited concrete owner class back to a wrapped reference. A future + recursive dependency mode would expand Fortran `use` dependencies only; + preprocessing include exposure is already handled separately. - The `.pyi` printer emits structured imports as `from module import name` or `from module import source as target`; the `.pyi` parser accepts the same syntax and restores the semantic import mapping. @@ -660,9 +670,8 @@ When updating parser behavior, keep this fail-fast contract aligned with tests: - While slicing raw source units, simple directive structure is tracked for `#ifdef`, `#ifndef`, `#elif`, `#else`, and `#endif` only to recognize mutually exclusive branches in unresolved raw input. - - `visit_file(..., macro_defines=...)` remains accepted for compatibility, - but macro decisions are ignored. Active branch selection must happen in the - compiler-backed preprocessing layer. + - Active branch selection must happen in the compiler-backed preprocessing + layer. - Duplicate procedure-name checks in a module/global scope are evaluated against same-level sliced units and this structural branch context: - if two same-name procedure headers are reachable in an overlapping branch context, raise `FortranParseError` (duplicate procedure name). diff --git a/docs/semantics/c2ir_mapping.md b/docs/semantics/c2ir_mapping.md index 41ae34b80..65a886294 100644 --- a/docs/semantics/c2ir_mapping.md +++ b/docs/semantics/c2ir_mapping.md @@ -32,6 +32,11 @@ semantic IR used by Fortran and edited `.pyi` files. variables through the `Constant` constraint. - Struct definitions become `SemanticClass` entries. Incomplete structs become opaque classes and may be used through direct `Ptr(...)` identity contracts. +- Explicit multi-header conversion resolves a struct to the header that defines + it. Other generated stubs import that owner class instead of emitting + duplicate definitions. +- Structs originating from private included headers remain usable through + generated owner-module `class Name(Opaque): pass` dependency stubs. - Declared C arrays, including adjusted array parameters, become semantic array storage contracts with C order for rank greater than one. - Pointers become explicit `SemanticStorageContract` pointer/reference @@ -51,7 +56,7 @@ The converter does not silently invent wrapper policy. It attaches - mutable numeric or `void *` pointer parameters without ownership, scalar-reference, or array policy; - arrays with unknown extents; -- incomplete structs used by value; +- incomplete or external opaque structs used by value; - unions used in semantic signatures; - `long double`, `volatile`, `_Atomic`, bitfields, and unsupported declarator compositions. @@ -61,4 +66,5 @@ The current C semantic path supports `--language c --semantics`, `--language c --pyi` output for this supported subset. Generated stubs remain conservative: ambiguous ownership, callback, ABI-extension, and Pythonic projection policy stays out of the generated `.pyi` until supplied by the -semantic model or an edited interface. +semantic model or an edited interface. In particular, an unresolved typedef is +not assumed to be opaque because its ABI representation is unknown. diff --git a/docs/semantics/pyi_format.md b/docs/semantics/pyi_format.md index 1330d7e0d..0d8f785a9 100644 --- a/docs/semantics/pyi_format.md +++ b/docs/semantics/pyi_format.md @@ -418,6 +418,45 @@ exact-reference adaptation, coercion/contract execution or C wrapper lowering. The C frontend can generate starter exact-contract `.pyi` output for the implemented semantic subset. +## External Opaque Type Stubs + +An external source-language type whose owner module is not part of the explicit +wrapping target is emitted as an owner-module opaque dependency stub. This +applies to imported Fortran derived types and to C opaque structs from external +header surfaces: + +```python +# types_mod.pyi +class particle(Opaque): + pass +``` + +The importing module references that owner rather than re-exporting the type: + +```python +# physics.pyi +from types_mod import particle + +def move(p: Ptr(particle)) -> None: ... +``` + +`emit_module_stubs(...)` produces the complete stub mapping. `load_pyi_modules` +loads one or more files or directories and reconciles those imports back into +semantic `external_type_ref` metadata. If the user replaces the opaque owner +stub with a concrete class body, the imported semantic reference becomes +`representation="wrapped"` without changing the importing stub. + +This file-set round-trip is the editing boundary for future wrapper policy. +Existing type constraints encoded with `Annotated[...]` are preserved now. +Additional coercion and executable contract syntax remains deferred. + +For C, an unresolved typedef is not automatically opaque: its ABI could be an +integer, pointer, struct, or another representation. The C frontend emits an +opaque class when declarations establish that contract, such as a forward +struct declaration or a private included struct used through pointers. An +edited `.pyi` file may also state the policy explicitly with `class +Name(Opaque): pass`. + ## Deferred C Work The shared model represents the current C semantic conversion subset for diff --git a/semantics/__init__.py b/semantics/__init__.py index d0725d2ee..59357c79c 100644 --- a/semantics/__init__.py +++ b/semantics/__init__.py @@ -2,6 +2,7 @@ collect_semantic_compile_time_requirements, fortran_file_to_semantic_modules, fortran_module_to_semantic_module, + fortran_project_to_semantic_modules, resolve_semantic_compile_time_values, ) from .c2ir import ( @@ -15,7 +16,8 @@ c_struct_to_semantic_class, c_type_to_semantic_type, ) -from .pyi_parser import convert_pyi_to_ir, load_pyi_file, parse_pyi_text +from .pyi_parser import convert_pyi_to_ir, load_pyi_file, load_pyi_modules, parse_pyi_text +from .pyi_printer import emit_module_stubs, opaque_dependency_modules from .readiness import assess_pyi_wrap_readiness, assess_semantic_wrap_readiness __all__ = ( @@ -34,7 +36,11 @@ "convert_pyi_to_ir", "fortran_file_to_semantic_modules", "fortran_module_to_semantic_module", + "fortran_project_to_semantic_modules", + "emit_module_stubs", "load_pyi_file", + "load_pyi_modules", + "opaque_dependency_modules", "parse_pyi_text", "resolve_semantic_compile_time_values", ) diff --git a/semantics/c2ir.py b/semantics/c2ir.py index 6afba906e..4821925ca 100644 --- a/semantics/c2ir.py +++ b/semantics/c2ir.py @@ -50,6 +50,7 @@ ) from .models import ( + EXTERNAL_TYPE_REF_METADATA, ProjectionMapping, SemanticArgument, SemanticArrayContract, @@ -60,6 +61,7 @@ SemanticOrigin, SemanticStorageContract, SemanticType, + _iter_module_semantic_types, ) @@ -198,7 +200,7 @@ def visit_project(self, project: CProject) -> list[SemanticModule]: self.structs = dict(project.structs) self.unions = dict(project.unions) self.enums = dict(project.enums) - return [ + modules = [ self.visit_file( c_file, typedefs=self.typedefs, @@ -208,6 +210,8 @@ def visit_project(self, project: CProject) -> list[SemanticModule]: ) for _filename, c_file in sorted(project.files.items()) ] + self._classify_project_external_types(modules, project) + return modules def visit_project_module( self, @@ -307,6 +311,7 @@ def visit_file( ), ) self._apply_include_exposure(module, c_file) + self._externalize_private_classes(module) return module finally: self.typedefs, self.structs, self.unions, self.enums = previous @@ -929,6 +934,104 @@ def is_private_origin(origin: SemanticOrigin) -> bool: if "Opaque" not in cls.base_classes: cls.base_classes.append("Opaque") + def _externalize_private_classes(self, module: SemanticModule) -> None: + external_classes: dict[str, str] = {} + for cls in module.classes: + if cls.visibility != "private" or "Opaque" not in cls.base_classes: + continue + filename = self._source_filename(cls.origin.source_location) + if filename is None: + continue + origin_module = self._module_name_for_filename(filename) + if origin_module != module.name: + external_classes[cls.name] = origin_module + if not external_classes: + return + + for semantic_type in _iter_module_semantic_types(module): + origin_module = external_classes.get(semantic_type.name) + if origin_module is None: + continue + self._set_external_type_ref( + semantic_type, + origin_module=origin_module, + wrapped=False, + ) + self._add_external_opaque_by_value_blocker(semantic_type) + module.classes = [ + cls + for cls in module.classes + if cls.name not in external_classes + ] + + def _classify_project_external_types( + self, + modules: list[SemanticModule], + project: CProject, + ) -> None: + modules_by_filename = { + module.origin.native_name: module + for module in modules + if module.origin.native_name is not None + } + owners: dict[str, tuple[str, bool]] = {} + for struct in project.structs.values(): + if struct.name is None or struct.source_location is None: + continue + owner = modules_by_filename.get(struct.source_location.filename) + if owner is None: + continue + owners[self._identifier(struct.name)] = (owner.name, not struct.is_incomplete) + + for module in modules: + external_names = { + name + for name, (origin_module, _wrapped) in owners.items() + if origin_module != module.name + } + if not external_names: + continue + for semantic_type in _iter_module_semantic_types(module): + owner = owners.get(semantic_type.name) + if owner is None or owner[0] == module.name: + continue + self._set_external_type_ref( + semantic_type, + origin_module=owner[0], + wrapped=owner[1], + ) + module.classes = [ + cls + for cls in module.classes + if cls.name not in external_names + ] + + @staticmethod + def _set_external_type_ref( + semantic_type: SemanticType, + *, + origin_module: str, + wrapped: bool, + ) -> None: + semantic_type.metadata[EXTERNAL_TYPE_REF_METADATA] = { + "name": semantic_type.name, + "local_name": semantic_type.name, + "origin_module": origin_module, + "wrapped": wrapped, + "representation": "wrapped" if wrapped else "opaque", + } + + def _add_external_opaque_by_value_blocker(self, semantic_type: SemanticType) -> None: + if semantic_type.storage is not None and semantic_type.storage.kind != "value": + return + semantic_type.metadata.setdefault("readiness_blockers", []).append( + self._blocker( + "c_opaque_struct_by_value", + "Opaque C structs can only cross wrapper boundaries through explicit pointer or handle policy.", + {"owner": semantic_type.name, "type": semantic_type.name}, + ) + ) + def _project_metadata(self, project: CProject) -> dict[str, Any]: metadata: dict[str, Any] = { "source_language": "c", @@ -1133,11 +1236,15 @@ def _diagnostic_code(code: str) -> str: @staticmethod def _module_name(c_file: CFile) -> str: if c_file.filename: - stem = Path(c_file.filename).stem + return CToIRConverter._module_name_for_filename(c_file.filename) else: stem = "c_module" return CToIRConverter._identifier(stem or "c_module") + @staticmethod + def _module_name_for_filename(filename: str) -> str: + return CToIRConverter._identifier(Path(filename).stem or "c_module") + @staticmethod def _identifier(name: str) -> str: text = _IDENTIFIER_RE.sub("_", str(name)).strip("_") diff --git a/semantics/fortran2ir.py b/semantics/fortran2ir.py index c751e2a32..79f9d823e 100644 --- a/semantics/fortran2ir.py +++ b/semantics/fortran2ir.py @@ -1,7 +1,9 @@ from __future__ import annotations import ast +from collections.abc import Iterable from copy import deepcopy +from dataclasses import dataclass import re from pathlib import Path @@ -11,13 +13,16 @@ FortranDerivedType, FortranFile, FortranModule, + FortranProject, FortranProgram, FortranProcedureSignature, FortranSubmodule, + FortranUseMapping, FortranVariable, ) from .models import ( + EXTERNAL_TYPE_REF_METADATA, SemanticArgument, SemanticArrayContract, SemanticClass, @@ -76,6 +81,13 @@ } +@dataclass(frozen=True) +class _DerivedTypeContext: + module: str | None = None + uses: dict[str, list[FortranUseMapping]] | None = None + local_types: frozenset[str] = frozenset() + + def _normalize_compile_time_values( compile_time_values: dict[str, int | str] | None, ) -> dict[str, str]: @@ -139,27 +151,39 @@ def __init__( self, type_map: dict[tuple[str, str | None], str] | None = None, compile_time_values: dict[str, int | str] | None = None, + wrapped_derived_types: Iterable[tuple[str, str]] | None = None, ): self.type_map = FORTRAN_TYPE_MAP if type_map is None else type_map self.compile_time_values = _normalize_compile_time_values(compile_time_values) + self.wrapped_derived_types = { + (str(module).lower(), str(name).lower()) + for module, name in (wrapped_derived_types or []) + } def visit(self, node, **context): """Dispatch one parsed Fortran model to the matching conversion method.""" + if isinstance(node, FortranProject): + return self.visit_project(node) if isinstance(node, FortranFile): return self.visit_file(node) if isinstance(node, FortranModule): return self.visit_module(node) if isinstance(node, FortranProcedureSignature): - return self.visit_procedure(node, visibility=context.get("visibility", "public")) + return self.visit_procedure( + node, + visibility=context.get("visibility", "public"), + derived_type_context=context.get("derived_type_context"), + ) if isinstance(node, FortranDerivedType): return self.visit_derived_type( node, procedure_lookup=context.get("procedure_lookup", {}), + derived_type_context=context.get("derived_type_context"), ) if isinstance(node, FortranArgument): - return self.visit_argument(node) + return self.visit_argument(node, derived_type_context=context.get("derived_type_context")) if isinstance(node, FortranVariable): - return self.visit_variable(node) + return self.visit_variable(node, derived_type_context=context.get("derived_type_context")) raise TypeError(f"Unsupported Fortran parse object: {type(node)!r}") def first_module(self, parsed): @@ -177,10 +201,29 @@ def first_module(self, parsed): raise TypeError(f"Unsupported Fortran parse object: {type(parsed)!r}") def visit_file(self, parsed_file: FortranFile) -> SemanticModule: - return self.visit_module(self.first_module(parsed_file)) + converter = self._with_additional_wrapped_types(self._wrapped_types_from_file(parsed_file)) + return converter.visit_module(self.first_module(parsed_file)) + + def visit_project(self, project: FortranProject) -> list[SemanticModule]: + converter = self._with_additional_wrapped_types(self._wrapped_types_from_project(project)) + return [ + module + for parsed_file in project.files + for module in converter.visit_file_modules(parsed_file) + ] - def visit_variable(self, var: FortranVariable) -> SemanticType: + def visit_variable( + self, + var: FortranVariable, + *, + derived_type_context: _DerivedTypeContext | None = None, + ) -> SemanticType: semantic_name = self._semantic_type_name(var) + derived_type_ref = self._derived_type_ref(var, derived_type_context) + metadata = {} + if derived_type_ref is not None: + semantic_name, ref_metadata = derived_type_ref + metadata[EXTERNAL_TYPE_REF_METADATA] = ref_metadata shape = [self._resolve_compile_time_text(dim) for dim in var.shape] storage = self._array_storage_contract(var, shape) if var.rank > 0 else None semantic_type = SemanticType( @@ -188,6 +231,7 @@ def visit_variable(self, var: FortranVariable) -> SemanticType: rank=var.rank, dtype=semantic_name, shape=list(storage.array.shape if storage is not None and storage.array is not None else shape), + metadata=metadata, storage=storage, origin=self._variable_origin(var), ) @@ -199,8 +243,9 @@ def visit_argument( arg: FortranArgument | FortranVariable, *, intent: str | None = None, + derived_type_context: _DerivedTypeContext | None = None, ) -> SemanticArgument: - semantic_type = self.visit_variable(arg) + semantic_type = self.visit_variable(arg, derived_type_context=derived_type_context) resolved_intent = intent if intent is not None else getattr(arg, "intent", "in") resolved_intent = str(resolved_intent).lower().replace(" ", "") if resolved_intent == "unknown": @@ -225,8 +270,9 @@ def visit_data_member( var: FortranArgument | FortranVariable, *, intent: str = "in", + derived_type_context: _DerivedTypeContext | None = None, ) -> SemanticArgument: - semantic_type = self.visit_variable(var) + semantic_type = self.visit_variable(var, derived_type_context=derived_type_context) if semantic_type.storage is not None and semantic_type.storage.array is not None: semantic_type.storage.array.allocatable = getattr(var, "allocatable", False) semantic_type.storage.array.pointer = getattr(var, "pointer", False) @@ -243,13 +289,16 @@ def visit_procedure( self, proc: FortranProcedureSignature, visibility: str = "public", + *, + derived_type_context: _DerivedTypeContext | None = None, ) -> SemanticFunction: - arguments = [self.visit_argument(arg) for arg in proc.arguments] + context = self._procedure_derived_type_context(proc, derived_type_context) + arguments = [self.visit_argument(arg, derived_type_context=context) for arg in proc.arguments] return SemanticFunction( name=proc.name, native_name=proc.name, arguments=arguments, - return_type=self.visit_variable(proc.result) if proc.result else None, + return_type=self.visit_variable(proc.result, derived_type_context=context) if proc.result else None, projection=self._procedure_projection(proc, arguments), visibility=visibility, origin=SemanticOrigin( @@ -264,12 +313,21 @@ def visit_derived_type( self, dtype: FortranDerivedType, procedure_lookup: dict[str, SemanticFunction] | None = None, + *, + derived_type_context: _DerivedTypeContext | None = None, ) -> SemanticClass: lookup = procedure_lookup or {} + context = derived_type_context or _DerivedTypeContext( + module=dtype.module, + local_types=frozenset({dtype.name.lower()}), + ) return SemanticClass( name=dtype.name, native_name=dtype.name, - fields=[self.visit_data_member(field, intent="in") for field in dtype.fields], + fields=[ + self.visit_data_member(field, intent="in", derived_type_context=context) + for field in dtype.fields + ], methods=self._bound_methods(dtype, lookup), base_classes=self._base_classes(dtype), visibility=getattr(dtype, "visibility", "public"), @@ -282,17 +340,23 @@ def visit_derived_type( ) def visit_module(self, module: FortranModule) -> SemanticModule: + context = self._module_derived_type_context(module) semantic_functions = [ self.visit_procedure( proc, visibility=self._symbol_visibility(module, proc.name), + derived_type_context=context, ) for proc in module.procedures ] procedure_lookup = {func.name: func for func in semantic_functions} semantic_classes = [ - self.visit_derived_type(dtype, procedure_lookup=procedure_lookup) + self.visit_derived_type( + dtype, + procedure_lookup=procedure_lookup, + derived_type_context=context, + ) for dtype in module.derived_types ] for semantic_cls in semantic_classes: @@ -302,7 +366,10 @@ def visit_module(self, module: FortranModule) -> SemanticModule: name=module.name, functions=semantic_functions, classes=semantic_classes, - variables=[self.visit_data_member(var, intent="in") for var in getattr(module, "variables", [])], + variables=[ + self.visit_data_member(var, intent="in", derived_type_context=context) + for var in getattr(module, "variables", []) + ], imports=self._module_imports(module), origin=SemanticOrigin( source_language="fortran", @@ -318,10 +385,11 @@ def visit_file_modules( *, standalone_module_name: str | None = None, ) -> list[SemanticModule]: - modules = [self.visit_module(module) for module in parsed_file.modules] + converter = self._with_additional_wrapped_types(self._wrapped_types_from_file(parsed_file)) + modules = [converter.visit_module(module) for module in parsed_file.modules] if parsed_file.procedures: modules.append( - self.procedures_to_semantic_module( + converter.procedures_to_semantic_module( parsed_file.procedures, name=standalone_module_name or self._standalone_module_name(parsed_file), ) @@ -356,6 +424,8 @@ def derived_type_to_semantic_class( return self.visit_derived_type(dtype, procedure_lookup=procedure_lookup) def module_to_semantic_module(self, module) -> SemanticModule: + if isinstance(module, FortranFile): + return self.visit_file(module) return self.visit_module(self.first_module(module)) @staticmethod @@ -387,6 +457,131 @@ def file_to_semantic_modules( standalone_module_name=standalone_module_name, ) + def project_to_semantic_modules(self, project: FortranProject) -> list[SemanticModule]: + return self.visit_project(project) + + def _with_additional_wrapped_types( + self, + wrapped_types: Iterable[tuple[str, str]], + ) -> "FortranToIRConverter": + merged = self.wrapped_derived_types | { + (str(module).lower(), str(name).lower()) + for module, name in wrapped_types + } + if merged == self.wrapped_derived_types: + return self + return FortranToIRConverter( + type_map=self.type_map, + compile_time_values=self.compile_time_values, + wrapped_derived_types=merged, + ) + + @staticmethod + def _wrapped_types_from_file(parsed_file: FortranFile) -> set[tuple[str, str]]: + return { + (dtype.module.lower(), dtype.name.lower()) + for module in parsed_file.modules + for dtype in module.derived_types + if dtype.module + } + + @staticmethod + def _wrapped_types_from_project(project: FortranProject) -> set[tuple[str, str]]: + return { + (dtype.module.lower(), dtype.name.lower()) + for dtype in project.derived_types.values() + if dtype.module + } + + @staticmethod + def _module_derived_type_context(module: FortranModule) -> _DerivedTypeContext: + return _DerivedTypeContext( + module=module.name, + uses=module.uses, + local_types=frozenset(dtype.name.lower() for dtype in module.derived_types), + ) + + @staticmethod + def _procedure_derived_type_context( + proc: FortranProcedureSignature, + parent: _DerivedTypeContext | None, + ) -> _DerivedTypeContext: + uses = dict(parent.uses or {}) if parent is not None else {} + uses.update(proc.uses) + return _DerivedTypeContext( + module=proc.module or (parent.module if parent is not None else None), + uses=uses, + local_types=parent.local_types if parent is not None else frozenset(), + ) + + def _derived_type_ref( + self, + var: FortranVariable, + context: _DerivedTypeContext | None, + ) -> tuple[str, dict[str, object]] | None: + if var.base_type.lower() != "derived": + return None + local_name = str(var.kind) + if not local_name: + return None + + origin_module, source_name = self._resolve_derived_type_origin(local_name, context) + local_type = bool( + context is not None + and context.module + and local_name.lower() in context.local_types + ) + if local_type or origin_module is None: + return None + wrapped = bool( + origin_module + and (origin_module.lower(), source_name.lower()) in self.wrapped_derived_types + ) + return local_name, { + "name": source_name, + "local_name": local_name, + "origin_module": origin_module, + "wrapped": wrapped, + "representation": "wrapped" if wrapped else "opaque", + } + + def _resolve_derived_type_origin( + self, + local_name: str, + context: _DerivedTypeContext | None, + ) -> tuple[str | None, str]: + lname = local_name.lower() + if context is None: + return None, local_name + if lname in context.local_types: + return context.module, local_name + + explicit: list[tuple[str, str]] = [] + wildcard_modules: list[str] = [] + for module_name, mappings in (context.uses or {}).items(): + if not mappings: + wildcard_modules.append(module_name) + continue + for mapping in mappings: + if mapping.local_name.lower() == lname: + explicit.append((module_name, mapping.source)) + + if len(explicit) == 1: + return explicit[0] + if len(explicit) > 1: + return None, local_name + + wrapped_wildcards = [ + module_name + for module_name in wildcard_modules + if (module_name.lower(), lname) in self.wrapped_derived_types + ] + if len(wrapped_wildcards) == 1: + return wrapped_wildcards[0], local_name + if len(wildcard_modules) == 1: + return wildcard_modules[0], local_name + return None, local_name + def _semantic_type_name(self, var: FortranVariable) -> str: base_type = var.base_type.lower() if base_type == "unknown": @@ -1077,10 +1272,14 @@ def resolve_semantic_compile_time_values( def _converter_for( compile_time_values: dict[str, int | str] | None = None, + wrapped_derived_types: Iterable[tuple[str, str]] | None = None, ) -> FortranToIRConverter: - if compile_time_values is None: + if compile_time_values is None and wrapped_derived_types is None: return _DEFAULT_CONVERTER - return FortranToIRConverter(compile_time_values=compile_time_values) + return FortranToIRConverter( + compile_time_values=compile_time_values, + wrapped_derived_types=wrapped_derived_types, + ) _DEFAULT_CONVERTER = FortranToIRConverter() @@ -1090,8 +1289,9 @@ def fortran_module_to_semantic_module( module, *, compile_time_values: dict[str, int | str] | None = None, + wrapped_derived_types: Iterable[tuple[str, str]] | None = None, ) -> SemanticModule: - return _converter_for(compile_time_values).module_to_semantic_module(module) + return _converter_for(compile_time_values, wrapped_derived_types).module_to_semantic_module(module) def fortran_file_to_semantic_modules( @@ -1099,12 +1299,21 @@ def fortran_file_to_semantic_modules( *, standalone_module_name: str | None = None, compile_time_values: dict[str, int | str] | None = None, + wrapped_derived_types: Iterable[tuple[str, str]] | None = None, ) -> list[SemanticModule]: - return _converter_for(compile_time_values).file_to_semantic_modules( + return _converter_for(compile_time_values, wrapped_derived_types).file_to_semantic_modules( parsed_file, standalone_module_name=standalone_module_name, ) +def fortran_project_to_semantic_modules( + project: FortranProject, + *, + compile_time_values: dict[str, int | str] | None = None, +) -> list[SemanticModule]: + return _converter_for(compile_time_values).project_to_semantic_modules(project) + + if __name__ == "__main__": pass diff --git a/semantics/models.py b/semantics/models.py index 4084b492d..719d5a292 100644 --- a/semantics/models.py +++ b/semantics/models.py @@ -5,6 +5,9 @@ from typing import Optional, Any +EXTERNAL_TYPE_REF_METADATA = "external_type_ref" + + # ============================================================ # Semantic Constraints # ============================================================ @@ -484,3 +487,31 @@ class SemanticModule: metadata: dict[str, Any] = field(default_factory=dict) origin: SemanticOrigin = field(default_factory=SemanticOrigin, compare=False) + + +def _iter_semantic_type_tree(semantic_type: SemanticType | None): + if semantic_type is None: + return + yield semantic_type + if semantic_type.name == "Callable": + arguments = semantic_type.metadata.get("arguments") + if isinstance(arguments, list): + for argument in arguments: + yield from _iter_semantic_type_tree(argument) + yield from _iter_semantic_type_tree(semantic_type.metadata.get("return")) + + +def _iter_module_semantic_types(module: SemanticModule): + for variable in module.variables: + yield from _iter_semantic_type_tree(variable.semantic_type) + for cls in module.classes: + for field in cls.fields: + yield from _iter_semantic_type_tree(field.semantic_type) + for method in cls.methods: + for argument in method.arguments: + yield from _iter_semantic_type_tree(argument.semantic_type) + yield from _iter_semantic_type_tree(method.return_type) + for function in module.functions: + for argument in function.arguments: + yield from _iter_semantic_type_tree(argument.semantic_type) + yield from _iter_semantic_type_tree(function.return_type) diff --git a/semantics/pyi_parser.py b/semantics/pyi_parser.py index 6bca20ebb..11a021695 100644 --- a/semantics/pyi_parser.py +++ b/semantics/pyi_parser.py @@ -1,10 +1,12 @@ from __future__ import annotations import ast +from collections.abc import Iterable from dataclasses import dataclass, field from pathlib import Path from .models import ( + EXTERNAL_TYPE_REF_METADATA, ProjectionMapping, SemanticArgument, SemanticArrayContract, @@ -17,9 +19,10 @@ SemanticModule, SemanticStorageContract, SemanticType, + _iter_module_semantic_types, ) -__all__ = ("convert_pyi_to_ir", "load_pyi_file", "parse_pyi_text") +__all__ = ("convert_pyi_to_ir", "load_pyi_file", "load_pyi_modules", "parse_pyi_text") def load_pyi_file(path: str | Path, *, module_name: str | None = None, encoding: str = "utf-8") -> SemanticModule: @@ -31,13 +34,43 @@ def load_pyi_file(path: str | Path, *, module_name: str | None = None, encoding: ) +def load_pyi_modules( + paths: str | Path | Iterable[str | Path], + *, + encoding: str = "utf-8", +) -> list[SemanticModule]: + raw_paths = [paths] if isinstance(paths, (str, Path)) else list(paths) + expanded: dict[Path, str | None] = {} + for raw_path in raw_paths: + path = Path(raw_path) + if path.is_dir(): + for item in path.rglob("*.pyi"): + if not item.is_file(): + continue + module_name = ".".join(item.relative_to(path).with_suffix("").parts) + previous = expanded.get(item) + if previous is not None and previous != module_name: + raise ValueError(f"Ambiguous module name for {item}: {previous!r} or {module_name!r}") + expanded[item] = module_name + else: + expanded.setdefault(path, None) + return _reconcile_external_type_refs( + [ + load_pyi_file(path, module_name=module_name, encoding=encoding) + for path, module_name in sorted(expanded.items()) + ] + ) + + def convert_pyi_to_ir(source: str, *, module_name: str = "") -> SemanticModule: return parse_pyi_text(source, module_name=module_name) def parse_pyi_text(source: str, *, module_name: str = "", filename: str = "") -> SemanticModule: tree = ast.parse(source or "\n", filename=filename) - return _PyiAstParser(module_name=module_name).parse(tree) + module = _PyiAstParser(module_name=module_name).parse(tree) + _annotate_imported_external_type_refs(module) + return module @dataclass @@ -71,13 +104,15 @@ def import_name(self, node: ast.Import) -> str: def class_def(self, node: ast.ClassDef, *, visibility: str) -> SemanticClass: body = _ClassBodyVisitor(self) body.visit_body(node.body) + base_classes = [ast.unparse(base) for base in node.bases] return SemanticClass( name=node.name, native_name=node.name, fields=body.fields, methods=body.methods, - base_classes=[ast.unparse(base) for base in node.bases], + base_classes=base_classes, + metadata={"representation": "opaque"} if "Opaque" in base_classes else {}, visibility=visibility, ) @@ -907,3 +942,64 @@ def generic_visit(self, node: ast.AST) -> None: def _node_text(node: ast.AST) -> str: text = ast.unparse(node) return text.splitlines()[0] if text else type(node).__name__ + + +def _annotate_imported_external_type_refs(module: SemanticModule) -> None: + imported = _imported_type_refs(module) + for semantic_type in _iter_module_semantic_types(module): + imported_ref = imported.get(semantic_type.name) + if imported_ref is None: + continue + origin_module, source_name, local_name = imported_ref + semantic_type.metadata.setdefault( + EXTERNAL_TYPE_REF_METADATA, + { + "name": source_name, + "local_name": local_name, + "origin_module": origin_module, + "wrapped": False, + "representation": "opaque", + }, + ) + + +def _imported_type_refs(module: SemanticModule) -> dict[str, tuple[str, str, str]]: + imported: dict[str, tuple[str, str, str]] = {} + for imp in module.imports: + if isinstance(imp, SemanticImport): + for item in imp.items: + local_name = item.target or item.source + imported[local_name] = (imp.module, item.source, local_name) + continue + for item in imp.split(","): + module_name, _, alias = item.strip().partition(" as ") + visible_name = alias or module_name + imported[visible_name] = (module_name, visible_name, visible_name) + + for semantic_type in _iter_module_semantic_types(module): + if "." not in semantic_type.name: + continue + module_name, type_name = semantic_type.name.rsplit(".", 1) + visible_module = module_name.split(".", 1)[0] + imported_module = imported.get(visible_module) + if imported_module is not None: + imported[semantic_type.name] = (imported_module[0], type_name, semantic_type.name) + return imported + + +def _reconcile_external_type_refs(modules: list[SemanticModule]) -> list[SemanticModule]: + definitions = { + (module.name, cls.name): cls + for module in modules + for cls in module.classes + } + for module in modules: + for semantic_type in _iter_module_semantic_types(module): + ref = semantic_type.metadata.get(EXTERNAL_TYPE_REF_METADATA) + if not isinstance(ref, dict): + continue + cls = definitions.get((ref.get("origin_module"), ref.get("name"))) + wrapped = cls is not None and "Opaque" not in cls.base_classes + ref["wrapped"] = wrapped + ref["representation"] = "wrapped" if wrapped else "opaque" + return modules diff --git a/semantics/pyi_printer.py b/semantics/pyi_printer.py index 61fdb642c..fb5ba9baf 100644 --- a/semantics/pyi_printer.py +++ b/semantics/pyi_printer.py @@ -1,9 +1,12 @@ from __future__ import annotations +from collections.abc import Iterable +from copy import deepcopy import keyword import re from .models import ( + EXTERNAL_TYPE_REF_METADATA, ProjectionMapping, SemanticArgument, SemanticArrayContract, @@ -15,6 +18,7 @@ SemanticMethod, SemanticModule, SemanticType, + _iter_module_semantic_types, ) @@ -283,11 +287,50 @@ def _class_body(self, cls: SemanticClass) -> str: return "\n\n".join(body_parts) def _append_imports(self, sections: list[str], module: SemanticModule) -> None: - for imp in module.imports: + imports = self._effective_imports(module) + for imp in imports: sections.append(self._emit_import(imp)) - if module.imports: + if imports: sections.append("") + @staticmethod + def _effective_imports(module: SemanticModule) -> list[str | SemanticImport]: + imports = list(module.imports) + imported_items = { + (imp.module, item.source, item.target or item.source) + for imp in imports + if isinstance(imp, SemanticImport) + for item in imp.items + } + synthetic: dict[str, list[SemanticImportItem]] = {} + for semantic_type in _iter_module_semantic_types(module): + ref = semantic_type.metadata.get(EXTERNAL_TYPE_REF_METADATA) + if not isinstance(ref, dict): + continue + origin_module = ref.get("origin_module") + source_name = ref.get("name") + local_name = ref.get("local_name") or source_name + if not all(isinstance(value, str) and value for value in (origin_module, source_name, local_name)): + continue + key = (origin_module, source_name, local_name) + if key in imported_items: + continue + synthetic.setdefault(origin_module, []).append( + SemanticImportItem( + source=source_name, + target=local_name if local_name != source_name else None, + ) + ) + imported_items.add(key) + imports.extend( + SemanticImport( + module=module_name, + items=sorted(items, key=lambda item: (item.source, item.target or "")), + ) + for module_name, items in sorted(synthetic.items()) + ) + return imports + @staticmethod def _emit_import(imp: str | SemanticImport) -> str: if isinstance(imp, str): @@ -439,5 +482,81 @@ def emit_module(module: SemanticModule) -> str: return _DEFAULT_PRINTER.emit_module(module) +def opaque_dependency_modules( + modules: SemanticModule | Iterable[SemanticModule], + *, + available_modules: Iterable[SemanticModule] | None = None, +) -> list[SemanticModule]: + source_modules = _module_list(modules) + known_modules = _module_list(available_modules) if available_modules is not None else source_modules + known_classes = { + (module.name, cls.name) + for module in known_modules + for cls in module.classes + } + dependencies: dict[str, set[str]] = {} + for module in source_modules: + for semantic_type in _iter_module_semantic_types(module): + ref = semantic_type.metadata.get(EXTERNAL_TYPE_REF_METADATA) + if not isinstance(ref, dict) or ref.get("representation") != "opaque": + continue + origin_module = ref.get("origin_module") + type_name = ref.get("name") + if not isinstance(origin_module, str) or not isinstance(type_name, str): + continue + if (origin_module, type_name) in known_classes: + continue + dependencies.setdefault(origin_module, set()).add(type_name) + return [ + SemanticModule( + name=module_name, + classes=[ + SemanticClass( + name=type_name, + native_name=type_name, + base_classes=["Opaque"], + metadata={"representation": "opaque"}, + ) + for type_name in sorted(type_names) + ], + ) + for module_name, type_names in sorted(dependencies.items()) + ] + + +def emit_module_stubs( + modules: SemanticModule | Iterable[SemanticModule], + *, + available_modules: Iterable[SemanticModule] | None = None, +) -> dict[str, str]: + source_modules = _module_list(modules) + emitted_modules: dict[str, SemanticModule] = {} + for module in source_modules: + if module.name in emitted_modules: + raise ValueError(f"Cannot emit duplicate semantic module '{module.name}'") + emitted_modules[module.name] = deepcopy(module) + + for dependency in opaque_dependency_modules( + source_modules, + available_modules=available_modules, + ): + target = emitted_modules.setdefault(dependency.name, SemanticModule(name=dependency.name)) + existing = {cls.name for cls in target.classes} + target.classes.extend(cls for cls in dependency.classes if cls.name not in existing) + + return { + module_name: emit_module(module).strip() + for module_name, module in emitted_modules.items() + } + + +def _module_list(modules: SemanticModule | Iterable[SemanticModule] | None) -> list[SemanticModule]: + if modules is None: + return [] + if isinstance(modules, SemanticModule): + return [modules] + return list(modules) + + if __name__ == "__main__": pass diff --git a/semantics/readiness.py b/semantics/readiness.py index d127575af..0877d1859 100644 --- a/semantics/readiness.py +++ b/semantics/readiness.py @@ -5,6 +5,7 @@ from typing import Iterable from .models import ( + EXTERNAL_TYPE_REF_METADATA, SemanticArgument, SemanticClass, SemanticFunction, @@ -13,7 +14,7 @@ SemanticModule, SemanticType, ) -from .pyi_parser import load_pyi_file +from .pyi_parser import load_pyi_modules __all__ = ("assess_pyi_wrap_readiness", "assess_semantic_wrap_readiness") @@ -54,8 +55,9 @@ def assess_pyi_wrap_readiness( encoding: str = "utf-8", ) -> dict: """Load one or more edited .pyi files and assess semantic wrap-readiness.""" - expanded = _expand_pyi_paths(paths) - modules = [load_pyi_file(path, encoding=encoding) for path in expanded] + raw_paths = [paths] if isinstance(paths, (str, Path)) else list(paths) + expanded = _expand_pyi_paths(raw_paths) + modules = load_pyi_modules(raw_paths, encoding=encoding) return assess_semantic_wrap_readiness(modules, source=[str(path) for path in expanded]) @@ -338,7 +340,7 @@ def _check_type( ) return - if not self.index.is_known_type(type_name, module): + if not self.index.is_known_type(type_name, module) and not _is_external_type_ref(semantic_type): self._add_blocker( "unresolved_semantic_types", "Some semantic type references are not declared by the .pyi interface or its imports.", @@ -586,6 +588,18 @@ def _is_constant(semantic_type: SemanticType) -> bool: return any(constraint.name == "Constant" for constraint in semantic_type.constraints) +def _is_external_type_ref(semantic_type: SemanticType) -> bool: + ref = semantic_type.metadata.get(EXTERNAL_TYPE_REF_METADATA) + return ( + isinstance(ref, dict) + and isinstance(ref.get("name"), str) + and bool(ref["name"]) + and isinstance(ref.get("origin_module"), str) + and bool(ref["origin_module"]) + and ref.get("representation") in {"opaque", "wrapped"} + ) + + def _shape_expressions(semantic_type: SemanticType) -> list[str]: expressions = list(semantic_type.shape) storage = semantic_type.storage diff --git a/tests/parser/c/test_c_cli_skeleton.py b/tests/parser/c/test_c_cli_skeleton.py index a512fef14..034fe48d0 100644 --- a/tests/parser/c/test_c_cli_skeleton.py +++ b/tests/parser/c/test_c_cli_skeleton.py @@ -224,6 +224,39 @@ def test_cli_c_pyi_out_requires_explicit_language_and_writes_when_selected(tmp_p assert "def add(" in output.read_text(encoding="utf-8") +def test_cli_c_pyi_out_writes_explicit_multi_header_owner_stubs(tmp_path: Path): + types = tmp_path / "types.h" + api = tmp_path / "api.h" + types.write_text("struct state { int id; };\n", encoding="utf-8") + api.write_text("struct state;\nvoid step(struct state *state);\n", encoding="utf-8") + + result = subprocess.run( + [ + sys.executable, + "-m", + "x2py", + str(types), + str(api), + "--language", + "c", + "--pyi", + "--out", + ], + capture_output=True, + text=True, + check=True, + ) + + assert result.stdout == "" + assert "class state:" in (tmp_path / "types.pyi").read_text(encoding="utf-8") + api_stub = (tmp_path / "api.pyi").read_text(encoding="utf-8") + assert "from types import state" in api_stub + assert "class state" not in api_stub + assert "state: Ptr(state)" in api_stub + readiness = x2py_cli._wrap_readiness_report([str(types), str(api)], language="c") + assert readiness[str(api)]["wrap_readiness"]["wrappable"] is True + + def test_cli_c_input_rejects_explicit_fortran_frontend(tmp_path: Path): header = tmp_path / "api.h" output = tmp_path / "api.pyi" diff --git a/tests/parser/c/test_c_public_api_skeleton.py b/tests/parser/c/test_c_public_api_skeleton.py index 6e6fc42b6..6dda59f14 100644 --- a/tests/parser/c/test_c_public_api_skeleton.py +++ b/tests/parser/c/test_c_public_api_skeleton.py @@ -3,6 +3,10 @@ from pathlib import Path +import pytest + +from c_parser import CParser, parse_c_file, parse_c_project + def test_parse_c_file_accepts_inline_source_and_returns_typed_model(): from c_parser import CFile, parse_c_file @@ -240,6 +244,20 @@ def test_c_parser_instance_entrypoints_match_public_functions(): assert parser.visit_project({"api.h": source}) == parse_c_project({"api.h": source}) +@pytest.mark.parametrize( + "parse,args", + [ + (parse_c_file, ("",)), + (CParser().visit_file, ("",)), + (parse_c_project, ({"api.h": ""},)), + (CParser().visit_project, ({"api.h": ""},)), + ], +) +def test_public_c_parse_rejects_removed_macro_defines_argument(parse, args): + with pytest.raises(TypeError, match="macro_defines"): + parse(*args, macro_defines={"USE_FAST"}) + + def test_c_parse_error_attributes_and_diagnostic_formatting(): from c_parser import CParseError diff --git a/tests/parser/test_cli.py b/tests/parser/test_cli.py index 363637896..8d03d961d 100644 --- a/tests/parser/test_cli.py +++ b/tests/parser/test_cli.py @@ -487,6 +487,71 @@ def test_fortran_parser_cli_semantics_pyi_and_empty_module_report_from_inline_co assert "" in empty_pyi_res.stdout +def test_x2py_semantics_marks_explicit_cross_file_derived_type_as_wrapped(tmp_path: Path): + types_mod = tmp_path / "types_mod.f90" + physics = tmp_path / "physics.f90" + types_mod.write_text( + """ +module types_mod + type :: particle + real :: mass + end type particle +end module types_mod +""", + encoding="utf-8", + ) + physics.write_text( + """ +module physics + use types_mod, only: particle +contains + subroutine move(p) + type(particle), intent(inout) :: p + end subroutine move +end module physics +""", + encoding="utf-8", + ) + + payload = x2py_cli._semantic_report([str(types_mod), str(physics)]) + semantic_type = payload[str(physics)]["semantic_modules"][0]["functions"][0]["arguments"][0]["semantic_type"] + + assert semantic_type["metadata"]["external_type_ref"]["wrapped"] is True + assert "class particle" not in payload[str(physics)]["pyi"] + + readiness = x2py_cli._wrap_readiness_report([str(types_mod), str(physics)]) + readiness_type = readiness[str(physics)]["semantic_modules"][0]["functions"][0]["arguments"][0]["semantic_type"] + + assert readiness_type["metadata"]["external_type_ref"]["wrapped"] is True + + +def test_x2py_pyi_report_writes_opaque_dependency_stub_for_external_type(tmp_path: Path, monkeypatch): + physics = tmp_path / "physics.f90" + physics.write_text( + """ +module physics + use types_mod, only: particle +contains + function create_particle() result(p) + type(particle) :: p + end function create_particle +end module physics +""", + encoding="utf-8", + ) + + payload = x2py_cli._semantic_report([str(physics)]) + + assert payload[str(physics)]["pyi_dependencies"] == { + "types_mod": "class particle(Opaque):\n pass" + } + monkeypatch.setattr(sys, "argv", ["x2py", str(physics), "--pyi", "--out"]) + assert x2py_cli.main() == 0 + + assert (tmp_path / "physics.pyi").exists() + assert (tmp_path / "types_mod.pyi").read_text(encoding="utf-8") == "class particle(Opaque):\n pass\n" + + def test_cli_out_requires_stage_flag(): cmd = [sys.executable, "-m", "x2py", str(TEST_FILE), "--out"] diff --git a/tests/parser/test_preprocessing_cli.py b/tests/parser/test_preprocessing_cli.py index f53925c07..4d10517a5 100644 --- a/tests/parser/test_preprocessing_cli.py +++ b/tests/parser/test_preprocessing_cli.py @@ -125,10 +125,7 @@ def test_preprocessing_config_internal_macros_recipe_and_validation(tmp_path: Pa selected = PreprocessingConfig(defines=["USE_MPI", "VALUE=3"], undefs=["DEBUG"]) assert plain.uses_compiler is False - assert plain.fortran_macro_defines() is None - assert PreprocessingConfig(mode="compiler", defines=["USE_MPI"]).fortran_macro_defines() is None assert plain.fortran_internal_recipe(source) is None - assert selected.fortran_macro_defines() == {"USE_MPI": 1, "VALUE": "3", "DEBUG": 0} assert selected.fortran_internal_recipe(source)["source_path"] == str(source) with pytest.raises(PreprocessingError, match="requires a macro name"): validate_macro_name("=value", "--define") diff --git a/tests/pyi/test_pyi_to_ir.py b/tests/pyi/test_pyi_to_ir.py index eeeed5025..66fab4a09 100644 --- a/tests/pyi/test_pyi_to_ir.py +++ b/tests/pyi/test_pyi_to_ir.py @@ -12,7 +12,7 @@ SemanticModule, SemanticType, ) -from semantics.pyi_parser import _PyiAstParser, convert_pyi_to_ir, load_pyi_file, parse_pyi_text +from semantics.pyi_parser import _PyiAstParser, convert_pyi_to_ir, load_pyi_file, load_pyi_modules, parse_pyi_text from semantics.pyi_printer import emit_module from tests._shared.fixture_outputs import FORTRAN_DATA_DIR, FORTRAN_SUFFIXES from x2py import parse_fortran_file @@ -130,6 +130,84 @@ def test_parse_pyi_text_accepts_import_aliases(): ] +def test_load_pyi_modules_reconciles_opaque_and_edited_external_types(tmp_path: Path): + physics = tmp_path / "physics.pyi" + types_mod = tmp_path / "types_mod.pyi" + physics.write_text( + """ +from types_mod import particle + +def create_particle() -> Ptr(particle): ... + +def move(p: Annotated[Ptr(particle), CompatibleHandle]) -> None: ... +""", + encoding="utf-8", + ) + types_mod.write_text( + """ +class particle(Opaque): + pass +""", + encoding="utf-8", + ) + + modules = {module.name: module for module in load_pyi_modules(tmp_path)} + opaque = modules["types_mod"].classes[0] + create_ref = modules["physics"].functions[0].return_type.metadata["external_type_ref"] + move_type = modules["physics"].functions[1].arguments[0].semantic_type + + assert opaque.metadata == {"representation": "opaque"} + assert create_ref == { + "name": "particle", + "local_name": "particle", + "origin_module": "types_mod", + "wrapped": False, + "representation": "opaque", + } + assert [constraint.name for constraint in move_type.constraints] == ["CompatibleHandle"] + + types_mod.write_text( + """ +class particle: + mass: Float64 +""", + encoding="utf-8", + ) + edited_modules = {module.name: module for module in load_pyi_modules([physics, types_mod])} + edited_ref = edited_modules["physics"].functions[0].return_type.metadata["external_type_ref"] + + assert edited_ref["wrapped"] is True + assert edited_ref["representation"] == "wrapped" + assert edited_modules["types_mod"].classes[0].fields[0].name == "mass" + + +def test_load_pyi_modules_preserves_dotted_module_names_from_directory(tmp_path: Path): + package = tmp_path / "shared" + package.mkdir() + (tmp_path / "physics.pyi").write_text( + """ +from shared.types_mod import particle + +def move(p: Ptr(particle)) -> None: ... +""", + encoding="utf-8", + ) + (package / "types_mod.pyi").write_text( + """ +class particle(Opaque): + pass +""", + encoding="utf-8", + ) + + modules = {module.name: module for module in load_pyi_modules(tmp_path)} + particle_ref = modules["physics"].functions[0].arguments[0].semantic_type.metadata["external_type_ref"] + + assert "shared.types_mod" in modules + assert particle_ref["origin_module"] == "shared.types_mod" + assert particle_ref["representation"] == "opaque" + + def test_convert_pyi_to_ir_and_import_parser_edge_cases(): module = convert_pyi_to_ir("from m import a, b as c\n", module_name="edited") assert module.imports == [ diff --git a/tests/semantics/test_c2ir.py b/tests/semantics/test_c2ir.py index 063bc23a2..007ed778a 100644 --- a/tests/semantics/test_c2ir.py +++ b/tests/semantics/test_c2ir.py @@ -3,7 +3,7 @@ import pytest -from c_parser import parse_c_file +from c_parser import parse_c_file, parse_c_project from c_parser.models import ( CArray, CAtomic, @@ -46,6 +46,7 @@ c_type_to_semantic_type, ) from semantics.readiness import assess_semantic_wrap_readiness +from semantics.pyi_printer import emit_module, emit_module_stubs def _function(module, name): @@ -136,6 +137,7 @@ def test_c2ir_converts_structs_and_opaque_struct_pointers(): assert scale_point.arguments[0].semantic_type.name == "point" assert context_create.return_type.name == "context" assert context_create.return_type.storage.kind == "reference" + assert "class context(Opaque):" in emit_module(module) report = assess_semantic_wrap_readiness(module, source="structs.h") assert report["wrappable"] is True @@ -161,15 +163,76 @@ def test_c2ir_private_include_types_remain_available_as_opaque_handles(): } module = c_file_to_semantic_modules(parsed)[0] - private_context = next(cls for cls in module.classes if cls.name == "private_context") make_context = _function(module, "make_context") use_context = _function(module, "use_context") + stubs = emit_module_stubs(module) - assert private_context.visibility == "private" - assert private_context.base_classes == ["Opaque"] - assert private_context.fields == [] + assert all(cls.name != "private_context" for cls in module.classes) assert make_context.return_type.name == "private_context" assert use_context.arguments[0].semantic_type.name == "private_context" + assert make_context.return_type.metadata["external_type_ref"] == { + "name": "private_context", + "local_name": "private_context", + "origin_module": "private", + "wrapped": False, + "representation": "opaque", + } + assert "from private import private_context" in stubs["api"] + assert stubs["private"] == "class private_context(Opaque):\n pass" + assert assess_semantic_wrap_readiness(module, source="api.h")["wrappable"] is True + + +def test_c2ir_explicit_project_headers_import_types_from_their_owner_module(): + project = parse_c_project( + { + "types.h": "struct state { int id; };\n", + "api.h": "struct state;\nvoid step(struct state *state);\n", + } + ) + modules = {module.name: module for module in c_project_to_semantic_modules(project)} + api = modules["api"] + state = _function(api, "step").arguments[0].semantic_type + stubs = emit_module_stubs(api, available_modules=modules.values()) + + assert all(cls.name != "state" for cls in api.classes) + assert state.metadata["external_type_ref"] == { + "name": "state", + "local_name": "state", + "origin_module": "types", + "wrapped": True, + "representation": "wrapped", + } + assert "from types import state" in stubs["api"] + assert "class state" not in stubs["api"] + assert assess_semantic_wrap_readiness(api, source="api.h")["wrappable"] is True + + +def test_c2ir_private_include_opaque_struct_by_value_remains_blocked(): + parsed = parse_c_file( + """ +# 1 "private.h" 1 +struct private_context { int internal; }; +# 1 "api.h" 2 +void use_context(struct private_context ctx); +""", + filename="api.h", + preprocessing="compiler", + ) + parsed.preprocessing_recipe = { + "included_files": [ + {"path": "api.h", "dependency_kind": "root", "exposure": "public"}, + {"path": "private.h", "dependency_kind": "project", "exposure": "private"}, + ] + } + + module = c_file_to_semantic_modules(parsed)[0] + report = assess_semantic_wrap_readiness(module, source="api.h") + + assert report["wrappable"] is False + assert "c_opaque_struct_by_value" in { + blocker["code"] + for blocker in report["wrappability_blockers"] + } def test_c2ir_converts_enum_constants_and_simple_macro_constants(): diff --git a/tests/semantics/test_fortran2ir.py b/tests/semantics/test_fortran2ir.py index d80d3ff29..63e291744 100644 --- a/tests/semantics/test_fortran2ir.py +++ b/tests/semantics/test_fortran2ir.py @@ -16,6 +16,7 @@ FortranVariable, ) from x2py import parse_fortran_file as parse_fortran_source +from x2py import parse_fortran_project from semantics.fortran2ir import ( FortranToIRConverter, @@ -24,6 +25,7 @@ collect_semantic_compile_time_requirements, fortran_file_to_semantic_modules, fortran_module_to_semantic_module, + fortran_project_to_semantic_modules, resolve_semantic_compile_time_values, ) from semantics import models as semantic_models @@ -1080,6 +1082,70 @@ def test_fortran_file_to_semantic_modules_keeps_standalone_procedures_from_inlin assert func.projection[1].result_position is None +def test_imported_derived_type_is_an_opaque_external_reference_by_default(): + parsed = parse_fortran_source( + """ +module physics + use types_mod, only: particle +contains + subroutine move(p) + type(particle), intent(inout) :: p + end subroutine move +end module physics +""" + ) + + module = fortran_module_to_semantic_module(parsed) + particle = get_function(module, "move").arguments[0].semantic_type + + assert module.classes == [] + assert particle.storage.kind == "reference" + assert particle.metadata["external_type_ref"] == { + "name": "particle", + "local_name": "particle", + "origin_module": "types_mod", + "wrapped": False, + "representation": "opaque", + } + + +def test_explicit_project_target_resolves_imported_derived_type_without_reexport(): + project = parse_fortran_project( + { + "types_mod.f90": """ +module types_mod + type :: particle + real :: mass + end type particle +end module types_mod +""", + "physics.f90": """ +module physics + use types_mod, only: particle +contains + subroutine move(p) + type(particle), intent(inout) :: p + end subroutine move +end module physics +""", + } + ) + + modules = {module.name: module for module in fortran_project_to_semantic_modules(project)} + particle = get_function(modules["physics"], "move").arguments[0].semantic_type + + assert [cls.name for cls in modules["types_mod"].classes] == ["particle"] + assert modules["physics"].classes == [] + assert sum(cls.name == "particle" for module in modules.values() for cls in module.classes) == 1 + assert particle.metadata["external_type_ref"] == { + "name": "particle", + "local_name": "particle", + "origin_module": "types_mod", + "wrapped": True, + "representation": "wrapped", + } + + def test_semantic_function_projection_equality_and_placeholders(): left = SemanticFunction( name="f", diff --git a/tests/semantics/test_pyi_printer.py b/tests/semantics/test_pyi_printer.py index 1eeb0beb8..ebc1934ee 100644 --- a/tests/semantics/test_pyi_printer.py +++ b/tests/semantics/test_pyi_printer.py @@ -8,6 +8,7 @@ from semantics.pyi_printer import ( emit_module, + emit_module_stubs, PyiPrinter, ) from semantics.models import ( @@ -392,6 +393,47 @@ def test_emit_import_renames(): assert "from list_input import delete_input_list as delete_input" in code +def test_emit_imported_derived_type_reference_without_reexporting_class(): + parsed = parse_fortran_source( + """ +module physics + use types_mod, only: particle +contains + subroutine move(p) + type(particle), intent(inout) :: p + end subroutine move +end module physics +""" + ) + module = fortran_module_to_semantic_module(parsed) + stubs = emit_module_stubs(module) + code = stubs["physics"] + + assert "from types_mod import particle" in code + assert "p: Ptr(particle)" in code + assert "class particle" not in code + assert stubs["types_mod"] == "class particle(Opaque):\n pass" + + +def test_emit_bare_use_adds_import_for_opaque_dependency_type(): + parsed = parse_fortran_source( + """ +module physics + use types_mod +contains + subroutine move(p) + type(particle), intent(inout) :: p + end subroutine move +end module physics +""" + ) + stubs = emit_module_stubs(fortran_module_to_semantic_module(parsed)) + + assert "import types_mod" in stubs["physics"] + assert "from types_mod import particle" in stubs["physics"] + assert stubs["types_mod"] == "class particle(Opaque):\n pass" + + def test_emit_structured_import_without_items_as_plain_import(): module = SemanticModule( name="imports", diff --git a/x2py/__init__.py b/x2py/__init__.py index ee071052e..3d149b971 100644 --- a/x2py/__init__.py +++ b/x2py/__init__.py @@ -22,6 +22,7 @@ collect_semantic_compile_time_requirements, fortran_file_to_semantic_modules, fortran_module_to_semantic_module, + fortran_project_to_semantic_modules, resolve_semantic_compile_time_values, ) from semantics.c2ir import ( @@ -35,7 +36,8 @@ c_struct_to_semantic_class, c_type_to_semantic_type, ) -from semantics.pyi_parser import convert_pyi_to_ir, load_pyi_file, parse_pyi_text +from semantics.pyi_parser import convert_pyi_to_ir, load_pyi_file, load_pyi_modules, parse_pyi_text +from semantics.pyi_printer import emit_module_stubs, opaque_dependency_modules from semantics.readiness import assess_pyi_wrap_readiness, assess_semantic_wrap_readiness from .cli import main @@ -45,6 +47,7 @@ "FortranTypeProbeReport", "build_fortran_type_probe_source", "evaluate_fortran_type_requirements", + "emit_module_stubs", "fortran_type_probe_expressions", "probe_fortran_type_expressions", } @@ -91,8 +94,11 @@ def __getattr__(name: str): "fortran_file_to_semantic_modules", "fortran_type_probe_expressions", "fortran_module_to_semantic_module", + "fortran_project_to_semantic_modules", "load_pyi_file", + "load_pyi_modules", "main", + "opaque_dependency_modules", "parse_c_file", "parse_c_project", "parse_fortran_file", diff --git a/x2py/cli.py b/x2py/cli.py index 6b9b94f06..d5756c6f4 100644 --- a/x2py/cli.py +++ b/x2py/cli.py @@ -13,7 +13,7 @@ from fortran_parser.models import FortranParseError from fortran_parser.parser import FortranParser from fortran_parser.cli import _format_report -from semantics.c2ir import c_file_to_semantic_modules +from semantics.c2ir import c_project_to_semantic_modules from semantics.fortran2ir import fortran_file_to_semantic_modules from semantics.pyi_parser import load_pyi_file from semantics.readiness import assess_semantic_wrap_readiness @@ -205,6 +205,18 @@ def _parse_c_path( return parsed +def _parse_c_project( + paths: list[str], + preprocessing: PreprocessingConfig, +): + parser = CParser() + parsed_files = { + str(path): _parse_c_path(parser, path, preprocessing) + for path in expand_c_paths(paths) + } + return parser._build_project(parsed_files) + + def _parse_report(paths: list[str], preprocessing: PreprocessingConfig | None = None) -> dict[str, dict]: preprocessing = preprocessing or PreprocessingConfig() out: dict[str, dict] = {} @@ -233,46 +245,111 @@ def _semantic_report( language: str = "fortran", ) -> dict[str, dict]: from semantics.fortran2ir import fortran_module_to_semantic_module - from semantics.pyi_printer import emit_module + from semantics.pyi_printer import emit_module_stubs preprocessing = preprocessing or PreprocessingConfig() out: dict[str, dict] = {} if language == "c": - parser = CParser() + project = _parse_c_project(paths, preprocessing) + converted_files = { + module.origin.native_name: [module] + for module in c_project_to_semantic_modules(project) + } + available_modules = [ + module + for modules in converted_files.values() + for module in modules + ] for p in expand_c_paths(paths): - parsed = _parse_c_path(parser, p, preprocessing) - modules = c_file_to_semantic_modules(parsed) + modules = converted_files[str(p)] + stubs = emit_module_stubs(modules, available_modules=available_modules) + primary_names = {module.name for module in modules} out[str(p)] = { "semantic_modules": [asdict(module) for module in modules], - "pyi": "\n\n".join(emit_module(module) for module in modules).strip(), + "pyi": "\n\n".join(stubs[module.name] for module in modules).strip(), + } + dependencies = { + module_name: text + for module_name, text in stubs.items() + if module_name not in primary_names } + if dependencies: + out[str(p)]["pyi_dependencies"] = dependencies return out parser = FortranParser() + parsed_files = [] for p in _expand_paths(paths): code, _preprocessing_recipe = _fortran_source_for_path(p, preprocessing) fobj = parser.visit_file(code, filename=str(p)) + parsed_files.append((p, fobj)) + wrapped_derived_types = _fortran_wrapped_derived_types(fobj for _p, fobj in parsed_files) + converted_files = [] + for p, fobj in parsed_files: compile_time_values = _fortran_compile_time_values(fobj, preprocessing) modules = [ - fortran_module_to_semantic_module(m, compile_time_values=compile_time_values) + fortran_module_to_semantic_module( + m, + compile_time_values=compile_time_values, + wrapped_derived_types=wrapped_derived_types, + ) for m in fobj.modules ] + converted_files.append((p, modules)) + available_modules = [module for _p, modules in converted_files for module in modules] + for p, modules in converted_files: + stubs = emit_module_stubs(modules, available_modules=available_modules) + primary_names = {module.name for module in modules} out[str(p)] = { "semantic_modules": [asdict(m) for m in modules], - "pyi": "\n\n".join(emit_module(m) for m in modules).strip(), + "pyi": "\n\n".join(stubs[module.name] for module in modules).strip(), + } + dependencies = { + module_name: text + for module_name, text in stubs.items() + if module_name not in primary_names } + if dependencies: + out[str(p)]["pyi_dependencies"] = dependencies return out def _format_pyi_report(semantic_report: dict[str, dict]) -> str: lines: list[str] = [] + emitted_dependencies: set[str] = set() for fname, payload in semantic_report.items(): lines.append(f"File: {fname}") lines.append(payload.get("pyi") or "") lines.append("") + for module_name, text in payload.get("pyi_dependencies", {}).items(): + if module_name in emitted_dependencies: + continue + emitted_dependencies.add(module_name) + lines.append(f"Dependency stub: {module_name}.pyi") + lines.append(text) + lines.append("") return "\n".join(lines).rstrip() +def _write_pyi_dependencies( + semantic_report: dict[str, dict], + *, + output_dir: Path | None = None, +) -> None: + outputs: dict[Path, str] = {} + for fname, payload in semantic_report.items(): + parent = output_dir or Path(fname).parent + for module_name, text in payload.get("pyi_dependencies", {}).items(): + path = parent.joinpath(*module_name.split(".")).with_suffix(".pyi") + existing = outputs.get(path) + if existing is not None and existing != text: + raise ValueError(f"Conflicting generated dependency stub for {path}") + outputs[path] = text + for path, text in outputs.items(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text + "\n", encoding="utf-8") + + def _wrap_readiness_report( paths: list[str], preprocessing: PreprocessingConfig | None = None, @@ -282,10 +359,13 @@ def _wrap_readiness_report( preprocessing = preprocessing or PreprocessingConfig() out: dict[str, dict] = {} if language == "c": - parser = CParser() + project = _parse_c_project(paths, preprocessing) + converted_files = { + module.origin.native_name: [module] + for module in c_project_to_semantic_modules(project) + } for p in expand_c_paths(paths): - parsed = _parse_c_path(parser, p, preprocessing) - modules = c_file_to_semantic_modules(parsed) + modules = converted_files[str(p)] out[str(p)] = { "source_kind": "c", "semantic_modules": [asdict(module) for module in modules], @@ -294,18 +374,27 @@ def _wrap_readiness_report( return out parser = FortranParser() - for p in _expand_readiness_paths(paths): + expanded_paths = _expand_readiness_paths(paths) + parsed_files = {} + for p in expanded_paths: + if p.suffix.lower() == ".pyi": + continue + code, _preprocessing_recipe = _fortran_source_for_path(p, preprocessing) + parsed_files[p] = parser.visit_file(code, filename=str(p)) + wrapped_derived_types = _fortran_wrapped_derived_types(parsed_files.values()) + + for p in expanded_paths: if p.suffix.lower() == ".pyi": modules = [load_pyi_file(p)] source_kind = "pyi" else: - code, _preprocessing_recipe = _fortran_source_for_path(p, preprocessing) - parsed = parser.visit_file(code, filename=str(p)) + parsed = parsed_files[p] compile_time_values = _fortran_compile_time_values(parsed, preprocessing) modules = fortran_file_to_semantic_modules( parsed, standalone_module_name=p.stem, compile_time_values=compile_time_values, + wrapped_derived_types=wrapped_derived_types, ) source_kind = "fortran" @@ -317,6 +406,16 @@ def _wrap_readiness_report( return out +def _fortran_wrapped_derived_types(parsed_files) -> set[tuple[str, str]]: + return { + (dtype.module.lower(), dtype.name.lower()) + for parsed in parsed_files + for module in parsed.modules + for dtype in module.derived_types + if dtype.module + } + + def _fortran_compile_time_values( parsed, preprocessing: PreprocessingConfig, @@ -756,9 +855,11 @@ def main() -> int: if args.out: pyi_text = "\n\n".join((report.get("pyi") or "") for report in (semantic_payload or {}).values()).strip() Path(args.out).write_text(pyi_text + "\n", encoding="utf-8") + _write_pyi_dependencies(semantic_payload or {}, output_dir=Path(args.out).parent) else: for fname, report in (semantic_payload or {}).items(): Path(fname).with_suffix(".pyi").write_text((report.get("pyi") or "") + "\n", encoding="utf-8") + _write_pyi_dependencies(semantic_payload or {}) else: if args.out: Path(args.out).write_text(json.dumps(payload, indent=2), encoding="utf-8") diff --git a/x2py/preprocessing.py b/x2py/preprocessing.py index c208f350a..04ccb82ee 100644 --- a/x2py/preprocessing.py +++ b/x2py/preprocessing.py @@ -272,20 +272,6 @@ class PreprocessingConfig: def uses_compiler(self) -> bool: return self.mode == "compiler" - def fortran_macro_defines(self) -> dict[str, int | str] | None: - """Return legacy parser macros only for non-production internal tests.""" - if self.uses_compiler: - return None - if not self.defines and not self.undefs: - return None - result: dict[str, int | str] = {} - for define in self.defines: - name, value = define.split("=", 1) if "=" in define else (define, 1) - result[name] = value - for undef in self.undefs: - result[undef] = 0 - return result - def fortran_internal_recipe(self, path: Path) -> dict[str, object] | None: if self.uses_compiler or not (self.defines or self.undefs): return None From c55aeaf86ddc96ea01f6723f5d8f34cb10ae5e48 Mon Sep 17 00:00:00 2001 From: said Date: Sun, 31 May 2026 12:21:37 +0100 Subject: [PATCH 12/13] update docs --- README.md | 2 + c_parser/parser.py | 64 +++++++++- docs/c_parser/c_parser_architecture.md | 14 ++- docs/c_parser/c_parser_reference.md | 24 ++++ docs/fortran/fortran_parser.md | 15 ++- .../parser_implementation_reference.md | 9 +- fortran_parser/parser.py | 117 +++++++++++++++++- tests/parser/c/test_c_cli_skeleton.py | 44 +++++++ tests/parser/c/test_c_public_api_skeleton.py | 3 + tests/semantics/test_pyi_printer.py | 6 + .../semantics/test_semantic_wrap_readiness.py | 32 +++++ x2py/__init__.py | 2 +- x2py/cli.py | 99 ++++++++++----- 13 files changed, 385 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 08a2daf82..ec5a68047 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,8 @@ Public API entrypoints include: - `x2py.parse_c_project(files, include_dirs=None, preprocessing="raw", encoding="utf-8") -> CProject` - `x2py.fortran_file_to_semantic_modules(parsed_file, standalone_module_name=None) -> list[SemanticModule]` - `x2py.fortran_project_to_semantic_modules(project) -> list[SemanticModule]` +- `x2py.c_file_to_semantic_modules(parsed_file) -> list[SemanticModule]` +- `x2py.c_project_to_semantic_modules(project) -> list[SemanticModule]` - `x2py.emit_module_stubs(module_or_modules) -> dict[str, str]` - `x2py.load_pyi_modules(path_or_paths, encoding="utf-8") -> list[SemanticModule]` - `x2py.assess_semantic_wrap_readiness(semantic_ir, source=None) -> dict` diff --git a/c_parser/parser.py b/c_parser/parser.py index f8e94fd54..059ea3b0e 100644 --- a/c_parser/parser.py +++ b/c_parser/parser.py @@ -9,6 +9,24 @@ parse_c_file(...) -> CParser.visit_file(...) -> CFile parse_c_project(...) -> CParser.visit_project(...) -> CProject +Recommended reading order for maintainers: + +1. Start from the module-level wrappers: `parse_c_file` and `parse_c_project`. +2. Read `CParser.visit_file`, `visit_project`, and `visit_parsed_project`. +3. Follow `_parse_translation_unit`, which dispatches one top-level C segment. +4. Read the declaration/declarator helpers used by each dispatched segment. +5. Finish with `_build_project` and the index helpers. + +`CParser` is organized in that same order: + +- public visitor entrypoints; +- source locations, diagnostics, macro provenance, and redeclaration merging; +- declaration-specifier and compiler-extension lexical helpers; +- recursive declarator grammar and parameter helpers; +- function and aggregate visitors; +- translation-unit dispatch and project assembly; +- thin module-level wrappers backed by `_DEFAULT_PARSER`. + One source file follows this path: source text/path @@ -376,6 +394,14 @@ class CParser: The instance carries no parse stack; per-call input and preprocessing configuration flow explicitly through `visit_file` and `visit_project`. See the module sketch and developer tutorial tests for the helper path. + + Class section map: + - public file/project visitor entrypoints; + - source-location, diagnostic, macro, and redeclaration helpers; + - declaration-specifier and compiler-extension helpers; + - recursive declarator and parameter grammar helpers; + - function and aggregate visitors; + - translation-unit dispatch and project assembly. """ # ------------------------------------------------------------------ @@ -509,7 +535,7 @@ def visit_project( ) for name, source in files.items() } - return self._build_project(parsed_files) + return self.visit_parsed_project(parsed_files) paths: list[Path] = [] root: Path | None = None @@ -535,7 +561,22 @@ def visit_project( preprocessing=preprocessing, encoding=encoding, ) - return self._build_project(parsed_files) + return self.visit_parsed_project(parsed_files) + + def visit_parsed_project(self, files: Mapping[str, CFile]) -> CProject: + """Assemble already parsed translation units into one `CProject`. + + This visitor is useful when an orchestration layer preprocesses each + source first and attaches recipe metadata before project resolution. + + Example: + >>> parser = CParser() + >>> parsed = parser.visit_file("int answer(void);", filename="api.h") + >>> project = parser.visit_parsed_project({"api.h": parsed}) + >>> sorted(project.functions) + ['answer'] + """ + return self._build_project(dict(files)) # ------------------------------------------------------------------ # Source locations, diagnostics, and macro provenance @@ -547,6 +588,7 @@ def _mark_preprocessed_declarations(parsed: CFile) -> None: seen_types: set[int] = set() def mark_type(type_: CType | None) -> None: + """Mark one reachable type graph as originating in preprocessed text.""" if type_ is None: return if isinstance(type_, CComposedType): @@ -582,6 +624,7 @@ def mark_type(type_: CType | None) -> None: mark_type(type_.type) def mark_variable(variable: CVariable) -> None: + """Mark a variable and recursively mark its declared type.""" variable.origin = "preprocessed" mark_type(variable.type) @@ -653,11 +696,13 @@ def _source_location(self, segment: CTopLevelSegment) -> CSourceLocation: @staticmethod def _could_start_c_external_declaration(text: str) -> bool: + """Return whether `text` begins like a C external declaration.""" stripped = text.lstrip() return bool(stripped) and (stripped[0].isalpha() or stripped[0] == "_") @staticmethod def _raise_for_invalid_top_level_syntax(segment: CTopLevelSegment) -> None: + """Raise a focused syntax error for a segment that cannot begin C.""" text = segment.text.strip() if not text: return @@ -3285,7 +3330,12 @@ def parse_c_file( preprocessing: str = "raw", encoding: str = "utf-8", ) -> CFile: - """Parse one C source string/path using the default parser instance.""" + """Parse one C source string/path using the default parser instance. + + Example: + >>> parse_c_file("int answer(void);", filename="api.h").functions[0].name + 'answer' + """ return _DEFAULT_PARSER.visit_file( source_or_path, filename=filename, @@ -3302,7 +3352,13 @@ def parse_c_project( preprocessing: str = "raw", encoding: str = "utf-8", ) -> CProject: - """Parse multiple C files or a directory using the default parser instance.""" + """Parse multiple C files or a directory using the default parser instance. + + Example: + >>> project = parse_c_project({"api.h": "int answer(void);"}) + >>> sorted(project.functions) + ['answer'] + """ return _DEFAULT_PARSER.visit_project( files, include_dirs=include_dirs, diff --git a/docs/c_parser/c_parser_architecture.md b/docs/c_parser/c_parser_architecture.md index 66352de9f..f6bcca07d 100644 --- a/docs/c_parser/c_parser_architecture.md +++ b/docs/c_parser/c_parser_architecture.md @@ -28,6 +28,8 @@ Implemented now: - `c_parser/` package exists and is included in package discovery. - `c_parser.models` defines JSON-stable parser dataclasses and `CParseError`. - `c_parser.parser` exposes `CParser`, `parse_c_file`, and `parse_c_project`. + `CParser.visit_parsed_project` is the documented orchestration hook for + assembling translation units that were parsed individually. - `x2py` re-exports `parse_c_file` and `parse_c_project`, matching its Fortran file/project entrypoint style. - `c_parser.parser` keeps parser helpers on `CParser`, matching the Fortran @@ -284,6 +286,7 @@ Current and planned responsibilities: JSON output. - `c_parser/parser.py` - Implemented: `CParser`, `parse_c_file`, `parse_c_project`, + `CParser.visit_parsed_project`, translation-unit visiting, declaration/function visitors, grammar-shaped recursive declarator parsing, concrete `CType` construction, and aggregate member extraction. Helper methods live on `CParser` rather than as broad @@ -305,8 +308,9 @@ Current and planned responsibilities: extension coverage. - `c_parser/project.py` - Compatibility export for project parsing. - - Project behavior is implemented through `CParser.visit_project`, including - `.c`/`.h`/`.i` discovery, include graphs, and header/source association. + - Project behavior is implemented through `CParser.visit_project` and + `CParser.visit_parsed_project`, including `.c`/`.h`/`.i` discovery, + include graphs, and header/source association. - `c_parser/type_resolver.py` - Resolves tag and typedef references, typedef chains/cycles, and aggregate references across parsed project files. @@ -344,6 +348,7 @@ Implemented companion class: class CParser: def visit_file(...): ... def visit_project(...): ... + def visit_parsed_project(...): ... ``` These entrypoints are exposed from both `c_parser` and `x2py.__init__`, using @@ -351,6 +356,11 @@ the same top-level file/project invocation pattern already provided for Fortran. C semantic conversion is exposed separately through `semantics.c2ir` and top-level `x2py` compatibility helpers. +`visit_parsed_project` is a class-level orchestration hook rather than an +additional module-level wrapper. It is used when a caller preprocesses each +translation unit first, attaches recipe metadata, and then asks the parser to +resolve the resulting file set as one `CProject`. + ## Core Model Families All declared types inherit from `CType`, which stores `qualifiers` and diff --git a/docs/c_parser/c_parser_reference.md b/docs/c_parser/c_parser_reference.md index 58c7926bc..dbbcf6e6b 100644 --- a/docs/c_parser/c_parser_reference.md +++ b/docs/c_parser/c_parser_reference.md @@ -308,6 +308,30 @@ For cross targets, provide a runner, for example `--runner=qemu-aarch64 The C semantic converter accepts this report as target context. The parser model remains source-faithful and does not embed host ABI assumptions. +## Parser Organization Notes + +`c_parser/parser.py` is intentionally ordered for maintainers. Read it from +top to bottom in these sections: + +1. Parser constants, private grammar dataclasses, and small path helpers. +2. `CParser` public visitors: `visit_file`, `visit_project`, and + `visit_parsed_project`. +3. Source-location, diagnostic, macro-provenance, and redeclaration helpers. +4. Declaration-specifier and compiler-extension lexical helpers. +5. Recursive declarator grammar and parameter helpers. +6. Function and aggregate visitors. +7. Translation-unit dispatch and project assembly. +8. Thin module-level wrappers: `parse_c_file` and `parse_c_project`. + +Helper methods remain on `CParser` when they depend on parser state. Their +docstrings describe the narrow parsing responsibility and include examples +where call shape or grammar behavior is not obvious. + +`visit_parsed_project(files)` assembles translation units that a caller has +already parsed individually. The x2py CLI uses it after compiler preprocessing +and recipe attachment. Most callers should use `parse_c_project(...)`, which +handles source loading before delegating to the same project assembly path. + ## Public API Implemented top-level and package entrypoints: diff --git a/docs/fortran/fortran_parser.md b/docs/fortran/fortran_parser.md index 00aba5cc1..a60600fe6 100644 --- a/docs/fortran/fortran_parser.md +++ b/docs/fortran/fortran_parser.md @@ -85,14 +85,14 @@ Supported public API: ## Parser organization notes `fortran_parser/parser.py` is now intentionally organized into clearly labeled -sections so maintainers can navigate the file by concern instead of by history: +sections and carries an embedded maintainer guide. Start with the thin public +wrappers at the bottom, then read the class from top to bottom: -- Regex/constants and parser-wide type aliases -- Module-level helper blocks (source-form rules, preprocessor logic, - diagnostics, compile-time expression resolution, dependency ordering) +- Regex/constants, parser-wide type aliases, private unit dataclasses, and the + compile-time resolver - `FortranParser` internals grouped by domain: - - internal visitor entrypoints (`visit_file`, `visit_project`). The public - API remains the module-level wrappers listed above. + - public visitor entrypoints (`visit_file`, `visit_project`). The supported + module-level API remains the wrappers listed above. - source-unit visitors for files, modules, submodules, programs, procedures, interfaces, derived types, and block data - recursive source-unit slicing (`header`, specification part, execution @@ -105,6 +105,9 @@ sections so maintainers can navigate the file by concern instead of by history: - Thin module-level convenience wrappers that delegate to a shared parser instance +Parser methods carry focused docstrings, with examples where a compatibility +visitor or lexical helper is easier to understand from a concrete call. + `visit_file` is the central orchestration path. It first slices the source into direct file-level units, then each unit visitor parses only its own substring and recursively slices direct children. This is the key parser design: each diff --git a/docs/fortran/parser_implementation_reference.md b/docs/fortran/parser_implementation_reference.md index 4de571b01..3533df91b 100644 --- a/docs/fortran/parser_implementation_reference.md +++ b/docs/fortran/parser_implementation_reference.md @@ -335,7 +335,11 @@ Dedicated tests for the error handling system: - `fortran_parser/lexer.py` - line preprocessing, source-form handling, continuation/comment normalization; returns tuples of `(preprocessed_line, original_line_number, original_source_line)` for downstream error reporting. - `fortran_parser/parser.py` - - main grammar subset parser and orchestration functions. + - main grammar subset parser and orchestration functions. The file embeds a + maintainer guide and keeps parser methods documented. Read the thin public + wrappers first, then follow `FortranParser` through public visitors, + grammar-unit visitors, scoped `_helper_*` methods, and low-level lexical + utilities. - `fortran_parser/type_resolver.py` - kind extraction and symbol/expression helpers. - `fortran_parser/models.py` @@ -395,7 +399,8 @@ ask it to implement each of these layers explicitly: Use this as the mental model when changing `fortran_parser/parser.py`: parsing is recursive over source units, and each unit is handled by the same grammar -shape before grammar-specific exceptions are applied. +shape before grammar-specific exceptions are applied. The parser method +docstrings are the local reference for individual helper responsibilities. ```fortran module m diff --git a/fortran_parser/parser.py b/fortran_parser/parser.py index 3e6e9edd3..821e22a31 100644 --- a/fortran_parser/parser.py +++ b/fortran_parser/parser.py @@ -1,4 +1,11 @@ # -*- coding: utf-8 -*- +"""Wrapper-oriented Fortran parser with recursive grammar-unit visitors. + +Read the module-level wrappers at the bottom first, then `FortranParser` +public visitors, source-unit visitors, `_helper_*` scoped parsing methods, and +finally low-level lexical/static utilities. The detailed maintainer guide +below documents the same file order with a control-flow example. +""" from __future__ import annotations import re @@ -12,7 +19,7 @@ from .type_resolver import extract_kind_from_type_spec from .utils import split_csv -""" +_PARSER_ARCHITECTURE_GUIDE = """ Parser architecture quick guide =============================== @@ -172,10 +179,12 @@ class _CompileTimeResolver: """Resolve compile-time expressions against one immutable symbol snapshot.""" def __init__(self, symbols: dict[str, str]): + """Normalize symbol names and initialize the expression cache.""" self.symbols = {name.lower(): str(value) for name, value in symbols.items()} self.cache: dict[tuple[str, bool], str] = {} def resolve(self, expr: str, prefer_symbolic: bool = True, resolving: frozenset[str] = frozenset()) -> str: + """Resolve symbols in one expression and fold integer-only results.""" text = expr.strip() if not text: return expr @@ -195,6 +204,7 @@ def resolve(self, expr: str, prefer_symbolic: bool = True, resolving: frozenset[ changed = False def replace_symbol(match: re.Match[str]) -> str: + """Replace one resolvable symbol while detecting cycles.""" nonlocal changed token = match.group(0) key = token.lower() @@ -467,6 +477,12 @@ def visit_project( return project def visit_fortran_module(self, code: _SourceOrLines, filename: str | None = None) -> FortranModule: + """Parse exactly one module unit from inline source or normalized lines. + + Example: + >>> FortranParser().visit_fortran_module("module m\\nend module m\\n").name + 'm' + """ _lines, root_scope, all_units = self._helper_prepare_source_units(code, filename) module_units = [unit for unit in all_units if unit.kind == "module"] if not module_units and any(unit.kind == "procedure" for unit in all_units): @@ -484,6 +500,7 @@ def visit_fortran_module(self, code: _SourceOrLines, filename: str | None = None return self.visit_module_unit(unit, parent_scope=root_scope, filename=filename) def visit_fortran_submodule(self, code: _SourceOrLines, filename: str | None = None) -> FortranSubmodule: + """Parse exactly one submodule unit from inline source or normalized lines.""" _lines, root_scope, all_units = self._helper_prepare_source_units(code, filename) unit = self._expect_single_parse_result( [unit for unit in all_units if unit.kind == "submodule"], @@ -494,6 +511,7 @@ def visit_fortran_submodule(self, code: _SourceOrLines, filename: str | None = N return self.visit_submodule_unit(unit, parent_scope=root_scope, filename=filename) def visit_fortran_interface(self, code: _SourceOrLines, filename: str | None = None) -> FortranInterface: + """Parse exactly one interface block, including nested procedure declarations.""" unit, scope = self._expect_single_parse_result( self._collect_interface_source_units(code, filename), parser_name="visit_fortran_interface", @@ -503,6 +521,7 @@ def visit_fortran_interface(self, code: _SourceOrLines, filename: str | None = N return self.visit_interface_unit(unit, parent_scope=scope, filename=filename) def visit_fortran_derived_type(self, code: _SourceOrLines, filename: str | None = None) -> FortranDerivedType: + """Parse exactly one derived-type block and its wrapper-relevant fields.""" unit, scope = self._expect_single_parse_result( self._collect_derived_type_source_units(code, filename), parser_name="visit_fortran_derived_type", @@ -512,6 +531,7 @@ def visit_fortran_derived_type(self, code: _SourceOrLines, filename: str | None return self.visit_derived_type_unit(unit, parent_scope=scope, filename=filename) def visit_fortran_program(self, code: _SourceOrLines, filename: str | None = None) -> FortranProgram: + """Parse exactly one program unit and its specification declarations.""" _lines, root_scope, all_units = self._helper_prepare_source_units(code, filename) unit = self._expect_single_parse_result( [unit for unit in all_units if unit.kind == "program"], @@ -522,6 +542,7 @@ def visit_fortran_program(self, code: _SourceOrLines, filename: str | None = Non return self.visit_program_unit(unit, parent_scope=root_scope, filename=filename) def visit_fortran_block_data_unit(self, code: _SourceOrLines, filename: str | None = None) -> FortranBlockData: + """Parse exactly one block-data unit and its specification declarations.""" _lines, root_scope, all_units = self._helper_prepare_source_units(code, filename) unit = self._expect_single_parse_result( [unit for unit in all_units if unit.kind == "block_data"], @@ -960,6 +981,7 @@ def _collect_interface_source_units( interfaces: list[tuple[_SourceUnit, _ParserScope]] = [] def collect(scope: _ParserScope, child_units: list[_SourceUnit]) -> None: + """Walk non-execution children and retain interface units.""" for child in child_units: if child.kind == "interface": interfaces.append((child, scope)) @@ -1001,6 +1023,7 @@ def _collect_derived_type_source_units( types: list[tuple[_SourceUnit, _ParserScope]] = [] def collect(scope: _ParserScope, child_units: list[_SourceUnit]) -> None: + """Walk nested grammar units and retain derived-type units.""" for child in child_units: if child.kind == "derived_type": types.append((child, scope)) @@ -1101,6 +1124,7 @@ def _handle_procedure_preprocessor_line( @staticmethod def _procedure_preprocessor_condition_set(pp_condition_stack: list[tuple[int, int]]) -> frozenset[str]: + """Serialize the active raw CPP branch stack for sibling comparison.""" return frozenset(f"g{group_id}:b{branch_id}" for group_id, branch_id in pp_condition_stack) def _helper_validate_possible_unit_header( @@ -1198,6 +1222,7 @@ def _helper_validate_file_scope_unparsed_lines(self, lines: _PreprocessedLines, @staticmethod def _is_allowed_unparsed_file_scope_line(line: str) -> bool: + """Return whether a file-scope line is intentionally metadata-only.""" stripped = line.strip() return ( stripped.startswith("#") @@ -1214,6 +1239,7 @@ def _raise_invalid_fortran_syntax_line( lineno: int | None, source_line: str | None, ) -> None: + """Raise the shared invalid-syntax diagnostic for one source line.""" raise FortranParseError( f"Invalid Fortran syntax in {context}: {line.strip()}", filename=filename, @@ -2019,6 +2045,7 @@ def _helper_unit_end_matches(kind: str, line: str) -> bool: @staticmethod def _is_contains_transition(line: str) -> bool: + """Return whether `line` starts a unit's `contains` region.""" return line.lower() == "contains" # ------------------------------------------------------------------ @@ -2032,6 +2059,7 @@ def _parse_module_header( lineno: int | None = None, source_line: str | None = None, ) -> FortranModule | None: + """Parse a module header or reject a malformed module-like line.""" module_match = _REGEX["module"].match(line) if not module_match: if line.lower().startswith("module ") and not re.match(r"^module\s+(procedure|subroutine|function)\b", line, re.IGNORECASE): @@ -2046,6 +2074,7 @@ def _parse_module_header( return FortranModule(name=module_match.group("name"), filename=filename) def _parse_submodule_header(self, line: str, filename: str | None) -> FortranSubmodule | None: + """Parse a submodule header, including parent and optional ancestor.""" match = _REGEX["submodule"].match(line) if not match: return None @@ -2058,18 +2087,21 @@ def _parse_submodule_header(self, line: str, filename: str | None) -> FortranSub ) def _parse_program_header(self, line: str, filename: str | None) -> FortranProgram | None: + """Parse a program header when `line` starts a program unit.""" match = _REGEX["program"].match(line) if not match: return None return FortranProgram(name=match.group("name"), filename=filename) def _parse_block_data_header(self, line: str, filename: str | None) -> FortranBlockData | None: + """Parse a named or unnamed block-data header.""" match = _REGEX["block_data"].match(line) if not match: return None return FortranBlockData(name=match.group("name"), filename=filename) def _parse_derived_type_start(self, line: str) -> tuple[str, list[str]] | None: + """Parse modern or legacy derived-type opening syntax.""" stripped = line.strip() tm = _REGEX["derived_type"].match(stripped) if tm: @@ -2087,6 +2119,7 @@ def _init_derived_type( *, current_module: str | None, ) -> FortranDerivedType | None: + """Build a derived-type model from one recognized opening line.""" parsed_type = self._parse_derived_type_start(line) if not parsed_type: return None @@ -2110,6 +2143,7 @@ def _init_derived_type( @staticmethod def _parse_interface_header(line: str) -> tuple[bool, str | None]: + """Return whether `line` opens an interface and its optional name.""" lower = line.lower() if not (lower.startswith("interface") or lower.startswith("abstract interface")): return False, None @@ -2126,6 +2160,7 @@ def _parse_procedure_header( lineno: int | None = None, source_line: str | None = None, ): + """Build procedure scope state from a subroutine or function header.""" module_proc = _REGEX["module_procedure_impl"].match(line) if module_proc and not in_interface: name = module_proc.group("name") @@ -2202,6 +2237,7 @@ def _raise_if_unparsed_procedure_header( lineno: int | None, source_line: str | None, ) -> None: + """Reject malformed lines that still resemble procedure headers.""" stripped = line.strip() if not stripped: return @@ -2227,6 +2263,7 @@ def _raise_if_unparsed_procedure_header( @staticmethod def _add_interface_attribute(sig: FortranProcedureSignature, interface_name: str | None) -> None: + """Attach a stable `interface(name)` marker to a procedure signature.""" if not interface_name: return iface_attr = f"interface({interface_name})" @@ -2235,6 +2272,7 @@ def _add_interface_attribute(sig: FortranProcedureSignature, interface_name: str @staticmethod def _resolve_derived_type_extensions(types: list[FortranDerivedType]) -> None: + """Link same-file `extends(parent)` names to parsed type models.""" by_name = {t.name.lower(): t for t in types} for dtype in types: if isinstance(dtype.extends, str): @@ -2282,6 +2320,7 @@ def _helper_scope_for_model( @staticmethod def _scope_key(name: str) -> str: + """Normalize a case-insensitive Fortran scope key.""" return name.lower() def _new_procedure_scope_state( @@ -2292,6 +2331,7 @@ def _new_procedure_scope_state( typed_symbols: set[str] | None = None, explicit_result: bool = False, ) -> dict: + """Create mutable procedure parsing state shared by spec-line helpers.""" state = { "signature": signature, "symbols": symbols, @@ -2314,9 +2354,11 @@ def _new_procedure_scope_state( return state def _proc_scope_get_symbol(self, proc_state: dict, name: str) -> FortranArgument | None: + """Return one procedure symbol by case-insensitive name.""" return proc_state["symbols"].get(self._scope_key(name)) def _proc_scope_symbol_is_declared(self, proc_state: dict, name: str) -> bool: + """Return whether a procedure symbol already has an explicit type.""" return self._scope_key(name) in proc_state["typed_symbols"] def _proc_scope_mark_declared_symbol( @@ -2328,6 +2370,7 @@ def _proc_scope_mark_declared_symbol( line_number: int | None = None, source_line: str | None = None, ) -> str: + """Record an explicitly typed procedure symbol and reject duplicates.""" key = self._scope_key(name) if key in proc_state["typed_symbols"]: raise FortranParseError( @@ -2341,6 +2384,7 @@ def _proc_scope_mark_declared_symbol( return key def _proc_scope_add_external_symbol(self, proc_state: dict, name: str) -> str: + """Record an external procedure symbol and update a matching dummy.""" key = self._scope_key(name) proc_state.setdefault("external_symbols", set()).add(key) arg = self._proc_scope_get_symbol(proc_state, key) @@ -2349,12 +2393,15 @@ def _proc_scope_add_external_symbol(self, proc_state: dict, name: str) -> str: return key def _proc_scope_add_include(self, proc_state: dict, include_path: str) -> None: + """Record one procedure-local include path.""" proc_state.setdefault("includes", []).append(include_path) def _proc_scope_add_imports(self, proc_state: dict, names: list[str]) -> None: + """Record interface imports visible inside a procedure declaration.""" proc_state.setdefault("imports", set()).update(self._scope_key(n) for n in names if n.strip()) def _proc_scope_set_declared_local_type(self, proc_state: dict, name: str, meta: dict) -> None: + """Store type metadata for a declared local symbol.""" key = self._scope_key(name) proc_state["declared_local_types"][key] = { "base_type": meta["base_type"], @@ -2374,6 +2421,7 @@ def _proc_scope_add_local_parameter( register_implicit_if_missing: bool = False, legacy: bool = False, ) -> None: + """Store one local parameter expression and validate declaration policy.""" key = self._scope_key(name) if require_declared and not self._proc_scope_symbol_is_declared(proc_state, key): raise FortranParseError( @@ -2406,6 +2454,7 @@ def _insert_unique_scope_symbol( label: str, filename: str | None = None, ) -> None: + """Insert one scope symbol or raise a duplicate-name diagnostic.""" if key in scope: raise FortranParseError( f"Duplicate symbol '{key}' in {label}.", @@ -2744,6 +2793,7 @@ def _parse_derived_type_contains_line( lineno: int | None = None, source_line: str | None = None, ) -> None: + """Parse type-bound procedure and generic bindings after `contains`.""" proc_binding = _REGEX["procedure_binding"].match(line) if proc_binding: binding_names = split_csv(proc_binding.group("names")) @@ -2887,6 +2937,7 @@ def _parse_declaration_left( *, parse_character_star: bool = True, ) -> tuple[dict, list[str]] | None: + """Parse a declaration prefix into normalized metadata and attributes.""" star_kind = self._find_legacy_star_kind(left) char_star = _REGEX["char_star"].match(left) if parse_character_star else None if char_star: @@ -3031,6 +3082,7 @@ def _helper_push_declaration_to_scope( @staticmethod def _new_decl_meta(base_type: str, kind: str | None) -> dict: + """Return default declaration metadata for one normalized base type.""" return { "base_type": base_type, "kind": kind or "", @@ -3048,6 +3100,7 @@ def _new_decl_meta(base_type: str, kind: str | None) -> dict: @staticmethod def _apply_decl_attrs(meta: dict, attrs: list[str], *, include_intent: bool = False) -> None: + """Merge declaration attributes into normalized metadata.""" for a in attrs: la = a.lower() if include_intent and la.startswith("intent") and "(" in la and ")" in la: @@ -3073,6 +3126,7 @@ def _apply_decl_attrs(meta: dict, attrs: list[str], *, include_intent: bool = Fa @staticmethod def _normalize_declared_name(name: str, meta: dict) -> str: + """Strip legacy entity-local spelling from a declared symbol name.""" normalized_name = re.sub(r"^\*\s*[0-9]+\s*", "", name).strip() if meta["base_type"] == "character" and "*" in normalized_name: # Legacy CHARACTER declarations may carry entity-local length @@ -3083,6 +3137,7 @@ def _normalize_declared_name(name: str, meta: dict) -> str: @staticmethod def _strip_legacy_star_kind_prefix(left: str) -> str: + """Remove an intrinsic `type*kind` prefix from a legacy declaration.""" return re.sub( r"^(integer|real|complex|logical)\s*\*\s*[0-9]+\s*", "", @@ -3092,6 +3147,7 @@ def _strip_legacy_star_kind_prefix(left: str) -> str: @staticmethod def _var(entry: str): + """Split one declaration entity into its name and inline dimensions.""" e = entry.strip() if not e: # pragma: no cover - split_csv omits empty declaration entities for valid declarations. return "", [] @@ -3105,6 +3161,7 @@ def _var(entry: str): @staticmethod def _apply(arg: FortranArgument, meta: dict, shape: list[str]): + """Apply normalized declaration metadata to an argument-like model.""" arg.base_type = meta["base_type"] arg.kind = meta["kind"] or "" arg.intent = meta["intent"] @@ -3124,6 +3181,7 @@ def _apply(arg: FortranArgument, meta: dict, shape: list[str]): @staticmethod def _split_dim_bounds(dim: str) -> tuple[str | None, str | None]: + """Normalize one dimension into lower and upper bound text.""" part = dim.strip() if not part: # pragma: no cover - empty dimensions are invalid Fortran and not emitted by split_csv. return None, None @@ -3136,6 +3194,7 @@ def _split_dim_bounds(dim: str) -> tuple[str | None, str | None]: @staticmethod def _extract_bounds(shape: list[str]) -> tuple[list[str | None], list[str | None]]: + """Extract parallel lower/upper bound lists from serialized dimensions.""" lbounds: list[str | None] = [] ubounds: list[str | None] = [] for dim in shape: @@ -3187,6 +3246,7 @@ def _handle_proc_external_line(self, line: str, proc_state: dict) -> bool: @staticmethod def _is_ignored_proc_spec_line(line: str) -> bool: + """Return whether a procedure spec line is intentionally metadata-only.""" ignored_patterns = ( r"^(function|subroutine)\b", r"^intrinsic\b", @@ -3282,6 +3342,7 @@ def _handle_proc_parameter_line( @staticmethod def _looks_like_unknown_proc_declaration(line: str) -> bool: + """Return whether an unparsed procedure line still looks declarative.""" m_first = re.match(r"^([A-Za-z_][A-Za-z0-9_]*)", line.strip()) first_word = m_first.group(1).lower() if m_first else "" non_decl_starts = { @@ -3342,6 +3403,7 @@ def _handle_unknown_proc_declaration( # ------------------------------------------------------------------ def _finalize_proc(self, state: dict) -> FortranProcedureSignature: + """Validate and freeze one procedure signature from mutable scope state.""" sig = state["signature"] symbols = state["symbols"] local_params = state.get("local_params", {}) @@ -3440,6 +3502,7 @@ def _finalize_proc(self, state: dict) -> FortranProcedureSignature: @staticmethod def _validate_all_args_declared(sig: FortranProcedureSignature, filename: str | None, *, explicit_result: bool) -> None: + """Require explicit argument/result types when `implicit none` is active.""" for arg in sig.arguments: if arg.base_type == "unknown": raise FortranParseError( @@ -3462,6 +3525,7 @@ def _validate_all_args_declared(sig: FortranProcedureSignature, filename: str | @staticmethod def _validate_function_result(sig: FortranProcedureSignature, filename: str | None) -> None: + """Validate function result existence and argument-name separation.""" if sig.result is None: # pragma: no cover - function signatures are constructed with result objects. raise FortranParseError( f"Function '{sig.name}' has no result variable.", @@ -3486,6 +3550,7 @@ def _validate_variable_declarations( owner_name: str | None, filename: str | None, ) -> None: + """Reject incompatible duplicate variables in one module-like scope.""" seen: dict[str, FortranArgument] = {} display_name = owner_name or "" for var in variables: @@ -3510,6 +3575,7 @@ def _validate_variable_declarations( @staticmethod def _variable_scope_label(scope) -> tuple[str, str | None]: + """Return the diagnostic owner label for a module-like scope.""" if isinstance(scope, FortranSubmodule): return "submodule", scope.name if isinstance(scope, FortranProgram): @@ -3520,6 +3586,7 @@ def _variable_scope_label(scope) -> tuple[str, str | None]: @staticmethod def _validate_module_variables(module: FortranModule | FortranSubmodule, filename: str | None) -> None: + """Validate variables declared by a module or submodule.""" owner_kind, owner_name = FortranParser._variable_scope_label(module) FortranParser._validate_variable_declarations( module.variables, @@ -3530,6 +3597,7 @@ def _validate_module_variables(module: FortranModule | FortranSubmodule, filenam @staticmethod def _apply_module_visibility(module: FortranModule, filename: str | None) -> None: + """Apply module default and explicit public/private visibility rules.""" public_set = {s.lower() for s in module.public_symbols} private_set = {s.lower() for s in module.private_symbols} for var in module.variables: @@ -3549,6 +3617,7 @@ def _apply_module_visibility(module: FortranModule, filename: str | None) -> Non @staticmethod def _validate_derived_type_fields(dtype: FortranDerivedType, filename: str | None) -> None: + """Reject duplicate or unresolved fields in one derived type.""" seen: set[str] = set() for f in dtype.fields: if f.name.lower() in seen: @@ -3573,6 +3642,7 @@ def _validate_no_duplicate_arg_names( line_number: int | None = None, source_line: str | None = None, ) -> None: + """Reject duplicate dummy argument names in one procedure header.""" seen: set[str] = set() for arg in args: key = arg.name.lower() @@ -3587,6 +3657,7 @@ def _validate_no_duplicate_arg_names( seen.add(key) def _collect_module_parameters(self, code: _SourceOrLines, filename: str | None) -> dict[str, dict[str, str]]: + """Collect module specification-part parameter expressions by module.""" lines = self._preprocessed_lines(code, filename) current_module = None in_module_spec_part = False @@ -3627,6 +3698,7 @@ def _collect_module_parameters(self, code: _SourceOrLines, filename: str | None) @staticmethod def _resolve_module_parameter_values(module_params: dict[str, dict[str, str]]) -> dict[str, dict[str, str]]: + """Resolve transitive parameter expressions inside each module.""" resolved: dict[str, dict[str, str]] = {} for module_name, params in module_params.items(): resolver = _CompileTimeResolver(params) @@ -3643,6 +3715,7 @@ def _resolve_signature_kinds( *, resolve_shapes: bool = True, ) -> None: + """Resolve procedure kind and optional shape expressions from scope facts.""" module_params = FortranParser._resolve_module_parameter_values(module_params) symbol_to_value: dict[str, str] = {} if sig.module: @@ -3687,6 +3760,7 @@ def _resolve_module_variable_kinds( module: FortranModule | FortranSubmodule | FortranProgram | FortranBlockData, module_params: dict[str, dict[str, str]], ) -> None: + """Resolve kind, value, and shape facts for module-like variables.""" module_params = FortranParser._resolve_module_parameter_values(module_params) symbol_to_value: dict[str, str] = {} if getattr(module, "name", None): @@ -3727,6 +3801,7 @@ def _resolve_kind_expression( *, resolver: _CompileTimeResolver | None = None, ) -> str: + """Resolve one kind or character-length expression against symbols.""" active_resolver = resolver or _CompileTimeResolver(symbols) text = expr.strip() if text.lower().startswith("len="): @@ -3736,6 +3811,7 @@ def _resolve_kind_expression( @staticmethod def _resolve_symbol_reference(expr: str, symbols: dict[str, str]) -> str: + """Follow direct symbol aliases until a stable expression is reached.""" out = expr.strip() seen: set[str] = set() while out.lower() in symbols and out.lower() not in seen: @@ -3745,6 +3821,7 @@ def _resolve_symbol_reference(expr: str, symbols: dict[str, str]) -> str: @staticmethod def _collect_relevant_local_params(sig: FortranProcedureSignature, local_params: dict[str, str]) -> dict[str, str]: + """Keep only local parameters reachable from a wrapper signature.""" if not local_params: return {} if not sig.arguments and sig.result is None: @@ -3769,6 +3846,7 @@ def _collect_relevant_local_params(sig: FortranProcedureSignature, local_params: @staticmethod def _extract_symbol_names(expr: str) -> set[str]: + """Extract lowercase identifier tokens from one expression.""" keywords = {"and", "or", "not"} return { token.lower() @@ -3778,6 +3856,7 @@ def _extract_symbol_names(expr: str) -> set[str]: @staticmethod def _normalize_parameter_value(value: str) -> str | None: + """Return a stable literal value or `None` for unresolved expressions.""" parsed_int = FortranParser._safe_eval_int_expr(value) if parsed_int is not None: return str(parsed_int) @@ -3796,6 +3875,7 @@ def _normalize_parameter_value(value: str) -> str | None: @staticmethod def _is_literal_parameter_value(value: str) -> bool: + """Return whether `value` is a supported serialized literal.""" text = value.strip() if not text: return False @@ -3820,6 +3900,7 @@ def _resolve_variables( base_types: dict[str, str] | None = None, symbolic_values: dict[str, str | None] | None = None, ) -> dict[str, FortranVariable]: + """Build resolved parameter variables from a symbol-expression map.""" base_types = base_types or {} symbolic_values = symbolic_values or {} valued: dict[str, FortranVariable] = {} @@ -3861,6 +3942,7 @@ def _safe_eval_int_expr(expr: str) -> int | None: allowed_unary = (ast.UAdd, ast.USub) def _eval(n): + """Evaluate one allowed AST node, returning None when unsupported.""" if isinstance(n, ast.Expression): return _eval(n.body) if isinstance(n, ast.Constant) and isinstance(n.value, (int, float, str, bool)): @@ -3964,6 +4046,7 @@ def _eval(n): @staticmethod def _topological_files(file_deps: dict[str, set[str]]) -> list[str]: + """Return dependency-first file order while tolerating dependency cycles.""" in_degree = {f: 0 for f in file_deps} for f, deps in file_deps.items(): for d in deps: @@ -4047,6 +4130,7 @@ def _expect_single_parse_result( @staticmethod def _source_form(filename: str | None) -> str: + """Infer fixed, modern, or unknown source form from a filename suffix.""" if not filename: return "unknown" ext = Path(filename).suffix.lower() @@ -4058,6 +4142,7 @@ def _source_form(filename: str | None) -> str: @staticmethod def _infer_implicit_base_type(symbol_name: str) -> str: + """Apply the default Fortran I-N integer implicit typing rule.""" first = symbol_name.strip()[:1].lower() if "i" <= first <= "n": return "integer" @@ -4065,6 +4150,7 @@ def _infer_implicit_base_type(symbol_name: str) -> str: @staticmethod def _find_legacy_star_kind(type_left: str) -> tuple[str, str] | None: + """Return base type and width from a legacy `type*kind` prefix.""" m = re.match(r"^(integer|real|complex|logical|character)\s*\*\s*([0-9]+)\b", type_left, re.IGNORECASE) if not m: return None @@ -4072,6 +4158,7 @@ def _find_legacy_star_kind(type_left: str) -> tuple[str, str] | None: @staticmethod def _parse_type_prefix(prefix: str) -> tuple[str, str | None] | None: + """Parse a standalone intrinsic, derived, or class type prefix.""" txt = prefix.strip() if not txt: return None @@ -4128,6 +4215,7 @@ def _looks_like_existing_source_path(source: str | Path) -> bool: @staticmethod def _split_submodule_parent(parent_spec: str) -> tuple[str, str | None]: + """Split `ancestor:parent` or plain `parent` submodule syntax.""" parts = [p.strip() for p in parent_spec.split(":", 1)] if len(parts) == 2: ancestor, parent = parts @@ -4136,6 +4224,7 @@ def _split_submodule_parent(parent_spec: str) -> tuple[str, str | None]: @staticmethod def _attrs(prefix: str, tail: str) -> list[str]: + """Collect supported procedure header attributes.""" attrs = [t.lower() for t in prefix.split() if t.lower() in _ATTR_PREFIX_WORDS] if _REGEX["bind_c"].search(tail): attrs.append("bind(c)") @@ -4143,6 +4232,7 @@ def _attrs(prefix: str, tail: str) -> list[str]: @staticmethod def _looks_like_procedure_header(line: str) -> bool: + """Return whether a line resembles a subroutine or function header.""" stripped = line.strip() if not stripped: return False @@ -4154,10 +4244,12 @@ def _looks_like_procedure_header(line: str) -> bool: @staticmethod def _is_openmp_directive(line: str) -> bool: + """Return whether a line begins an OpenMP sentinel directive.""" return line.lstrip().lower().startswith("!$omp") @staticmethod def _is_openmp_declarative_directive(line: str) -> bool: + """Return whether an OpenMP directive belongs in a specification part.""" directive = line.lstrip()[5:].strip().lower() if FortranParser._is_openmp_directive(line) else "" return directive.startswith( ( @@ -4172,6 +4264,7 @@ def _is_openmp_declarative_directive(line: str) -> bool: @staticmethod def _looks_like_declaration_or_spec(line: str) -> bool: + """Return whether a line resembles a specification-part statement.""" stripped = line.strip() if not stripped: return False @@ -4193,6 +4286,7 @@ def _looks_like_declaration_or_spec(line: str) -> bool: @staticmethod def _is_statement_function_statement(line: str) -> bool: + """Return whether a line has legacy statement-function syntax.""" stripped = line.strip() return bool( re.match( @@ -4204,6 +4298,7 @@ def _is_statement_function_statement(line: str) -> bool: @staticmethod def _is_ignored_spec_statement(line: str) -> bool: + """Return whether a recognized specification statement needs no model.""" return bool( _REGEX["include"].match(line) or re.match( @@ -4215,6 +4310,12 @@ def _is_ignored_spec_statement(line: str) -> bool: @staticmethod def _parse_use_statement(line: str) -> tuple[str, list[FortranUseMapping]] | None: + """Parse a `use` statement into its module and explicit symbol mappings. + + Example: + >>> FortranParser._parse_use_statement("use kinds, only: local => remote") + ('kinds', [FortranUseMapping(source='remote', target='local')]) + """ match = _REGEX["use"].match(line) if not match: return None @@ -4240,6 +4341,7 @@ def _parse_use_statement(line: str) -> tuple[str, list[FortranUseMapping]] | Non @staticmethod def _is_executable_statement_start(line: str) -> bool: + """Return whether a line starts execution rather than specification.""" stripped = line.strip() if not stripped: # pragma: no cover - callers skip blank lines before executable checks. return False @@ -4301,6 +4403,12 @@ def parse_fortran_file( filename: str | None = None, encoding: str = "utf-8", ) -> FortranFile: + """Parse one Fortran source string or path with the shared parser. + + Example: + >>> parse_fortran_file("subroutine ping()\\nend subroutine ping\\n").procedures[0].name + 'ping' + """ return _DEFAULT_PARSER.visit_file( source_or_path, filename=filename, @@ -4309,4 +4417,11 @@ def parse_fortran_file( def parse_fortran_project(files, *, encoding: str = "utf-8") -> FortranProject: + """Parse explicit Fortran sources or a directory into one project model. + + Example: + >>> project = parse_fortran_project({"types.f90": "module types\\nend module types\\n"}) + >>> sorted(project.modules) + ['types'] + """ return _DEFAULT_PARSER.visit_project(files, encoding=encoding) diff --git a/tests/parser/c/test_c_cli_skeleton.py b/tests/parser/c/test_c_cli_skeleton.py index 034fe48d0..7454f5d95 100644 --- a/tests/parser/c/test_c_cli_skeleton.py +++ b/tests/parser/c/test_c_cli_skeleton.py @@ -187,6 +187,50 @@ def test_cli_c_wrap_readiness_human_output_for_header(tmp_path: Path): assert "Wrappable: yes" in res.stdout +def test_cli_c_wrap_readiness_directory_includes_native_and_pyi_inputs(tmp_path: Path): + header = tmp_path / "api.h" + pyi = tmp_path / "solver.pyi" + header.write_text("int add(int a, int b);\n", encoding="utf-8") + pyi.write_text("def fill(n: Int32) -> None: ...\n", encoding="utf-8") + cmd = [ + sys.executable, + "-m", + "x2py", + str(tmp_path), + "--language", + "c", + "--wrap-readiness", + "--json", + ] + + res = subprocess.run(cmd, capture_output=True, text=True, check=True) + payload = json.loads(res.stdout) + + assert payload[str(header)]["source_kind"] == "c" + assert payload[str(pyi)]["source_kind"] == "pyi" + + +def test_cli_c_wrap_readiness_accepts_language_neutral_pyi_input(tmp_path: Path): + pyi = tmp_path / "solver.pyi" + pyi.write_text("def fill(n: Int32) -> None: ...\n", encoding="utf-8") + cmd = [ + sys.executable, + "-m", + "x2py", + str(pyi), + "--language", + "c", + "--wrap-readiness", + "--json", + ] + + res = subprocess.run(cmd, capture_output=True, text=True, check=True) + payload = json.loads(res.stdout) + + assert payload[str(pyi)]["source_kind"] == "pyi" + assert payload[str(pyi)]["wrap_readiness"]["wrappable"] is True + + def test_cli_c_pyi_human_output_for_header(tmp_path: Path): header = tmp_path / "api.h" header.write_text("int add(int a, int b);\n", encoding="utf-8") diff --git a/tests/parser/c/test_c_public_api_skeleton.py b/tests/parser/c/test_c_public_api_skeleton.py index 6dda59f14..030151dff 100644 --- a/tests/parser/c/test_c_public_api_skeleton.py +++ b/tests/parser/c/test_c_public_api_skeleton.py @@ -242,6 +242,9 @@ def test_c_parser_instance_entrypoints_match_public_functions(): assert parser.visit_file(source, filename="api.h") == parse_c_file(source, filename="api.h") assert parser.visit_project({"api.h": source}) == parse_c_project({"api.h": source}) + assert parser.visit_parsed_project( + {"api.h": parser.visit_file(source, filename="api.h")} + ) == parse_c_project({"api.h": source}) @pytest.mark.parametrize( diff --git a/tests/semantics/test_pyi_printer.py b/tests/semantics/test_pyi_printer.py index ebc1934ee..d0c8b581c 100644 --- a/tests/semantics/test_pyi_printer.py +++ b/tests/semantics/test_pyi_printer.py @@ -1,5 +1,6 @@ import pytest +import x2py from x2py import parse_fortran_file as parse_fortran_source from semantics.fortran2ir import ( @@ -30,6 +31,11 @@ # Helpers # ============================================================ +def test_x2py_public_api_exports_module_stub_emitter(): + assert "emit_module_stubs" in x2py.__all__ + assert x2py.emit_module_stubs is emit_module_stubs + + def generate_pyi(source: str) -> str: fmod = parse_fortran_source(source) diff --git a/tests/semantics/test_semantic_wrap_readiness.py b/tests/semantics/test_semantic_wrap_readiness.py index 028d5adef..91efc6c95 100644 --- a/tests/semantics/test_semantic_wrap_readiness.py +++ b/tests/semantics/test_semantic_wrap_readiness.py @@ -390,6 +390,38 @@ def test_x2py_main_wrap_readiness_json_directory_expands_fortran_and_pyi(tmp_pat assert payload[str(pyi)]["source_kind"] == "pyi" +def test_wrap_readiness_report_reconciles_edited_pyi_file_set(tmp_path: Path): + physics = tmp_path / "physics.pyi" + types_mod = tmp_path / "types_mod.pyi" + physics.write_text( + """ +from types_mod import particle + +def create_particle() -> Ptr(particle): ... +""", + encoding="utf-8", + ) + types_mod.write_text( + """ +class particle: + mass: Float64 +""", + encoding="utf-8", + ) + + payload = x2py_cli._wrap_readiness_report([str(tmp_path)]) + modules = { + module["name"]: module + for module in payload[str(physics)]["semantic_modules"] + } + particle_ref = modules["physics"]["functions"][0]["return_type"]["metadata"]["external_type_ref"] + + assert particle_ref["origin_module"] == "types_mod" + assert particle_ref["wrapped"] is True + assert particle_ref["representation"] == "wrapped" + assert payload[str(physics)]["wrap_readiness"]["n_modules"] == 2 + + def test_x2py_main_semantic_readiness_blocker_formatting(): text = x2py_cli._format_semantic_readiness( { diff --git a/x2py/__init__.py b/x2py/__init__.py index 3d149b971..7a060cd06 100644 --- a/x2py/__init__.py +++ b/x2py/__init__.py @@ -47,7 +47,6 @@ "FortranTypeProbeReport", "build_fortran_type_probe_source", "evaluate_fortran_type_requirements", - "emit_module_stubs", "fortran_type_probe_expressions", "probe_fortran_type_expressions", } @@ -90,6 +89,7 @@ def __getattr__(name: str): "c_type_to_semantic_type", "collect_semantic_compile_time_requirements", "convert_pyi_to_ir", + "emit_module_stubs", "evaluate_fortran_type_requirements", "fortran_file_to_semantic_modules", "fortran_type_probe_expressions", diff --git a/x2py/cli.py b/x2py/cli.py index d5756c6f4..b3b3856f4 100644 --- a/x2py/cli.py +++ b/x2py/cli.py @@ -15,7 +15,7 @@ from fortran_parser.cli import _format_report from semantics.c2ir import c_project_to_semantic_modules from semantics.fortran2ir import fortran_file_to_semantic_modules -from semantics.pyi_parser import load_pyi_file +from semantics.pyi_parser import load_pyi_modules from semantics.readiness import assess_semantic_wrap_readiness from x2py.preprocessing import ( PreprocessingConfig, @@ -93,6 +93,17 @@ def _expand_readiness_paths(paths: list[str]) -> list[Path]: return sorted(set(expanded)) +def _expand_pyi_paths(paths: list[str]) -> list[Path]: + expanded: list[Path] = [] + for raw in paths: + p = Path(raw) + if p.is_dir(): + expanded.extend(_collect_pyi_extensions(p)) + elif p.suffix.lower() == ".pyi": + expanded.append(p) + return sorted(set(expanded)) + + def _resolve_language( paths: list[str], requested: str | None, @@ -214,7 +225,7 @@ def _parse_c_project( str(path): _parse_c_path(parser, path, preprocessing) for path in expand_c_paths(paths) } - return parser._build_project(parsed_files) + return parser.visit_parsed_project(parsed_files) def _parse_report(paths: list[str], preprocessing: PreprocessingConfig | None = None) -> dict[str, dict]: @@ -359,53 +370,81 @@ def _wrap_readiness_report( preprocessing = preprocessing or PreprocessingConfig() out: dict[str, dict] = {} if language == "c": - project = _parse_c_project(paths, preprocessing) - converted_files = { - module.origin.native_name: [module] - for module in c_project_to_semantic_modules(project) - } - for p in expand_c_paths(paths): - modules = converted_files[str(p)] - out[str(p)] = { - "source_kind": "c", - "semantic_modules": [asdict(module) for module in modules], - "wrap_readiness": assess_semantic_wrap_readiness(modules, source=str(p)), + c_paths = [ + path + for path in expand_c_paths(paths) + if path.suffix.lower() != ".pyi" + ] + if c_paths: + project = _parse_c_project([str(path) for path in c_paths], preprocessing) + converted_files = { + module.origin.native_name: [module] + for module in c_project_to_semantic_modules(project) } + for p in c_paths: + modules = converted_files[str(p)] + out[str(p)] = { + "source_kind": "c", + "semantic_modules": [asdict(module) for module in modules], + "wrap_readiness": assess_semantic_wrap_readiness(modules, source=str(p)), + } + out.update(_pyi_readiness_report(paths)) return out parser = FortranParser() - expanded_paths = _expand_readiness_paths(paths) + expanded_paths = [ + path + for path in _expand_readiness_paths(paths) + if path.suffix.lower() != ".pyi" + ] parsed_files = {} for p in expanded_paths: - if p.suffix.lower() == ".pyi": - continue code, _preprocessing_recipe = _fortran_source_for_path(p, preprocessing) parsed_files[p] = parser.visit_file(code, filename=str(p)) wrapped_derived_types = _fortran_wrapped_derived_types(parsed_files.values()) for p in expanded_paths: - if p.suffix.lower() == ".pyi": - modules = [load_pyi_file(p)] - source_kind = "pyi" - else: - parsed = parsed_files[p] - compile_time_values = _fortran_compile_time_values(parsed, preprocessing) - modules = fortran_file_to_semantic_modules( - parsed, - standalone_module_name=p.stem, - compile_time_values=compile_time_values, - wrapped_derived_types=wrapped_derived_types, - ) - source_kind = "fortran" + parsed = parsed_files[p] + compile_time_values = _fortran_compile_time_values(parsed, preprocessing) + modules = fortran_file_to_semantic_modules( + parsed, + standalone_module_name=p.stem, + compile_time_values=compile_time_values, + wrapped_derived_types=wrapped_derived_types, + ) out[str(p)] = { - "source_kind": source_kind, + "source_kind": "fortran", "semantic_modules": [asdict(module) for module in modules], "wrap_readiness": assess_semantic_wrap_readiness(modules, source=str(p)), } + out.update(_pyi_readiness_report(paths)) return out +def _pyi_readiness_report(paths: list[str]) -> dict[str, dict]: + """Load one edited `.pyi` file set and report each interface path.""" + + pyi_paths = _expand_pyi_paths(paths) + if not pyi_paths: + return {} + modules = load_pyi_modules( + [ + raw + for raw in paths + if Path(raw).is_dir() or Path(raw).suffix.lower() == ".pyi" + ] + ) + return { + str(path): { + "source_kind": "pyi", + "semantic_modules": [asdict(module) for module in modules], + "wrap_readiness": assess_semantic_wrap_readiness(modules, source=str(path)), + } + for path in pyi_paths + } + + def _fortran_wrapped_derived_types(parsed_files) -> set[tuple[str, str]]: return { (dtype.module.lower(), dtype.name.lower()) From 95513b7c0a93b6d924d20d638ccf7a858d3d42c4 Mon Sep 17 00:00:00 2001 From: said Date: Sun, 31 May 2026 13:15:49 +0100 Subject: [PATCH 13/13] coverage --- AGENTS.md | 2 + tests/parser/c/test_c_cli_skeleton.py | 31 ++- .../c/test_c_declarations_and_declarators.py | 1 - tests/parser/c/test_c_lexer_preprocessor.py | 54 ++++++ tests/parser/c/test_c_public_api_skeleton.py | 31 +-- tests/parser/test_cli.py | 183 ++++++++++++++++++ .../parser/test_parser_public_entrypoints.py | 5 - tests/parser/test_preprocessing_cli.py | 12 ++ .../parser/test_procedure_and_type_parsing.py | 5 + 9 files changed, 291 insertions(+), 33 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 5f01225ca..8e1581af2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,4 +11,6 @@ Ignore: - *.json Do not spend context window or analysis on those files unless explicitly requested. +When updating tests, remove obsolete tests that only assert removed/old implementation behavior does not exist. Do not preserve rejection or absence checks for API/features that were intentionally removed unless explicitly requested. +When investigating coverage failures, mirror the GitHub Actions workflow before deciding the fix: run coverage with `COVERAGE_PROCESS_START=pyproject.toml`, combine parallel data with `python -m coverage combine`, then run `python -m coverage report`. Do not assume a plain local coverage run matches CI, especially when subprocess tests are involved. When you create a commit add this prefix to the message to know that you did push the commit "codex: ..." diff --git a/tests/parser/c/test_c_cli_skeleton.py b/tests/parser/c/test_c_cli_skeleton.py index 7454f5d95..027038b03 100644 --- a/tests/parser/c/test_c_cli_skeleton.py +++ b/tests/parser/c/test_c_cli_skeleton.py @@ -3,11 +3,15 @@ import json import os +import runpy import subprocess import sys from pathlib import Path from types import SimpleNamespace +import pytest + +from c_parser import CParseError from c_parser import cli as c_parser_cli from x2py import cli as x2py_cli from x2py.preprocessing import PreprocessingConfig @@ -113,6 +117,31 @@ def test_attach_preprocessing_recipe_filters_invalid_and_duplicate_macros(): assert parsed.macros[2].source_location.line == 4 +def test_c_parser_cli_helpers_errors_and_module_entrypoint(monkeypatch, capsys): + monkeypatch.setenv("C_PARSER_TEST_FLAG", " on ") + assert c_parser_cli._env_flag("C_PARSER_TEST_FLAG") is True + monkeypatch.delenv("C_PARSER_TEST_FLAG") + assert c_parser_cli._env_flag("C_PARSER_TEST_FLAG") is False + + assert c_parser_cli._diagnostic_color_enabled(disabled=True) is False + monkeypatch.setenv("NO_COLOR", "1") + assert c_parser_cli._diagnostic_color_enabled(disabled=False) is False + monkeypatch.delenv("NO_COLOR") + assert c_parser_cli._diagnostic_color_enabled(disabled=False) is True + + def fail_parse(_paths): + raise CParseError("invalid", filename="bad.h", line_number=1, column=1, source_line="@@@") + + monkeypatch.setattr(c_parser_cli, "parse_c_report", fail_parse) + assert c_parser_cli.main(["bad.h", "--no-color"]) == 1 + assert "bad.h:1:1: error[CPARSE_ERROR]: invalid" in capsys.readouterr().err + + monkeypatch.setattr(c_parser_cli, "main", lambda _argv=None: 0) + with pytest.raises(SystemExit) as exc_info: + runpy.run_module("c_parser.__main__", run_name="__main__") + assert exc_info.value.code == 0 + + def test_cli_c_parse_json_out_writes_file_and_suppresses_stdout(tmp_path: Path): header = tmp_path / "api.h" output = tmp_path / "report.json" @@ -485,7 +514,6 @@ def test_c_parser_cli_module_handles_directory_loader_and_output_modes(tmp_path: def test_c_parser_module_entrypoint_and_compatibility_exports(tmp_path: Path): import c_parser.__main__ as c_module_entrypoint - import c_parser.utils as c_utils from c_parser.parser import parse_c_project from c_parser.project import parse_c_project as compatibility_parse_c_project @@ -501,7 +529,6 @@ def test_c_parser_module_entrypoint_and_compatibility_exports(tmp_path: Path): assert json.loads(result.stdout)[str(header)]["functions"][0]["name"] == "run" assert c_module_entrypoint.main is c_parser_cli.main assert compatibility_parse_c_project is parse_c_project - assert c_utils.__all__ == () def test_c_parser_module_formats_parse_errors_without_traceback(tmp_path: Path): diff --git a/tests/parser/c/test_c_declarations_and_declarators.py b/tests/parser/c/test_c_declarations_and_declarators.py index c97f759d2..b51950d68 100644 --- a/tests/parser/c/test_c_declarations_and_declarators.py +++ b/tests/parser/c/test_c_declarations_and_declarators.py @@ -498,7 +498,6 @@ def test_function_type_discards_placeholder_parameter_names(): signature = type_.components[1] assert isinstance(signature, CFunctionType) assert len(signature.parameter_types) == 2 - assert not hasattr(signature, "parameters") def test_conflicting_function_pointer_typedefs_report_diagnostic(): diff --git a/tests/parser/c/test_c_lexer_preprocessor.py b/tests/parser/c/test_c_lexer_preprocessor.py index 85d29d57a..fb9511b01 100644 --- a/tests/parser/c/test_c_lexer_preprocessor.py +++ b/tests/parser/c/test_c_lexer_preprocessor.py @@ -71,6 +71,60 @@ def test_top_level_split_helpers_ignore_nested_commas_and_function_bodies(): ] +def test_c_lexer_covers_linemarker_escapes_top_level_strings_and_eof_records(): + from c_parser import parse_c_file + from c_parser.lexer import ( + CLogicalRecord, + _unescape_linemarker_filename, + lex_c_source, + line_mappings_for_source, + normalize_c_source, + split_top_level_c_source, + ) + from c_parser.preprocessor import _record_location + + assert _unescape_linemarker_filename(r"a\nb\rc\td\\e\"f\x") == "a\nb\rc\td\\e\"fx" + assert _unescape_linemarker_filename("tail\\") == "tail\\" + + mappings = line_mappings_for_source( + '#line 7\nint local;\n# 3 "dir\\\\api\\".h"\nint named;\n', + filename="generated.i", + use_linemarkers=True, + ) + assert mappings[1].filename == "generated.i" + assert mappings[1].line == 7 + assert mappings[3].filename == 'dir\\api".h' + assert mappings[3].line == 3 + + segments = split_top_level_c_source('"literal";\nint unfinished', filename="odd.c") + assert [(segment.text, segment.terminator) for segment in segments] == [ + ('"literal"', ";"), + ("int unfinished", "eof"), + ] + + normalized = normalize_c_source("#define API \\\n", filename="defs.h") + assert normalized.records[0].text == "#define API" + assert _record_location( + CLogicalRecord(text="#define API", filename="defs.h", original_source_lines=()) + ).source_line is None + assert _record_location( + CLogicalRecord(text="define API", filename="defs.h", original_source_lines=("define API",)) + ).column == 1 + + token = lex_c_source(r'char *s = "unterminated\\', filename="bad.c")[-1] + assert token.kind == "string" + assert token.text == r'"unterminated\\' + + parsed = parse_c_file( + '#line 11 "dir\\\\api\\".h"\n' + 'struct __attribute__((annotate("tag\\"ged"))) named { int value; };\n', + filename="generated.i", + preprocessing="preprocessed", + ) + assert parsed.structs[0].name == "named" + assert parsed.structs[0].source_location.filename == 'dir\\api".h' + + def test_raw_mode_records_includes_without_expanding_them(): from c_parser import parse_c_file diff --git a/tests/parser/c/test_c_public_api_skeleton.py b/tests/parser/c/test_c_public_api_skeleton.py index 030151dff..c14c13a43 100644 --- a/tests/parser/c/test_c_public_api_skeleton.py +++ b/tests/parser/c/test_c_public_api_skeleton.py @@ -3,8 +3,6 @@ from pathlib import Path -import pytest - from c_parser import CParser, parse_c_file, parse_c_project @@ -226,14 +224,6 @@ def test_unresolved_typedef_reference_metadata_is_preserved_in_json(): assert result_type["type"] is None -def test_public_c_parser_entrypoints_do_not_include_parser_side_readiness(): - import c_parser - - assert hasattr(c_parser, "parse_c_file") - assert hasattr(c_parser, "parse_c_project") - assert not hasattr(c_parser, "assess_c_wrap_readiness") - - def test_c_parser_instance_entrypoints_match_public_functions(): from c_parser import CParser, parse_c_file, parse_c_project @@ -247,22 +237,8 @@ def test_c_parser_instance_entrypoints_match_public_functions(): ) == parse_c_project({"api.h": source}) -@pytest.mark.parametrize( - "parse,args", - [ - (parse_c_file, ("",)), - (CParser().visit_file, ("",)), - (parse_c_project, ({"api.h": ""},)), - (CParser().visit_project, ({"api.h": ""},)), - ], -) -def test_public_c_parse_rejects_removed_macro_defines_argument(parse, args): - with pytest.raises(TypeError, match="macro_defines"): - parse(*args, macro_defines={"USE_FAST"}) - - def test_c_parse_error_attributes_and_diagnostic_formatting(): - from c_parser import CParseError + from c_parser import CArray, CComposedType, CInt, CParseError, CPointer, CSourceLocation err = CParseError( "unexpected token", @@ -283,6 +259,11 @@ def test_c_parse_error_attributes_and_diagnostic_formatting(): assert "2 | int broken(;" in diagnostic assert "note: parser raised at" in diagnostic + assert CSourceLocation(filename="api.h").display == "api.h" + composed = CComposedType(components=[CPointer(), CArray(bound="4"), CInt()]) + assert composed.pointer_depth == 1 + assert composed.array_rank == 1 + def test_c_parse_error_color_and_no_color_formatting(): from c_parser import CParseError diff --git a/tests/parser/test_cli.py b/tests/parser/test_cli.py index 8d03d961d..30fe850e3 100644 --- a/tests/parser/test_cli.py +++ b/tests/parser/test_cli.py @@ -3,6 +3,7 @@ from dataclasses import dataclass import json import os +import runpy import subprocess import sys import types @@ -11,7 +12,9 @@ import pytest from fortran_parser import cli as fortran_parser_cli +from x2py import FortranParseError from x2py import cli as x2py_cli +from x2py.preprocessing import PreprocessingConfig, PreprocessingDiagnostic, PreprocessingError TEST_FILE = Path(__file__).parent.parent / "data" / "fortran" / "general" / "basic_subroutine.f90" @@ -552,6 +555,186 @@ def test_x2py_pyi_report_writes_opaque_dependency_stub_for_external_type(tmp_pat assert (tmp_path / "types_mod.pyi").read_text(encoding="utf-8") == "class particle(Opaque):\n pass\n" +def test_x2py_pyi_report_formats_and_rejects_conflicting_dependency_stubs(): + report = { + "first.f90": { + "pyi": "def first() -> None: ...", + "pyi_dependencies": {"shared": "class shared(Opaque):\n pass"}, + }, + "second.f90": { + "pyi": "def second() -> None: ...", + "pyi_dependencies": {"shared": "class shared(Opaque):\n pass"}, + }, + } + + text = x2py_cli._format_pyi_report(report) + + assert text.count("Dependency stub: shared.pyi") == 1 + assert "def first() -> None: ..." in text + with pytest.raises(ValueError, match="Conflicting generated dependency stub"): + x2py_cli._write_pyi_dependencies( + { + "first.f90": {"pyi_dependencies": {"shared": "class shared:\n pass"}}, + "second.f90": {"pyi_dependencies": {"shared": "class shared:\n value: int"}}, + } + ) + + +def test_x2py_main_formats_preprocessing_errors_with_and_without_diagnostics(monkeypatch, capsys): + monkeypatch.setattr(sys, "argv", ["x2py", str(TEST_FILE), "--parse"]) + + def fail_with_diagnostic(_paths, _preprocessing): + raise PreprocessingError( + "compiler failed", + category="PREPROCESSOR_FAILED", + diagnostics=[ + PreprocessingDiagnostic( + category="PREPROCESSOR_FAILED", + message="bad include", + path="source.F90", + line=9, + ) + ], + ) + + monkeypatch.setattr(x2py_cli, "_parse_report", fail_with_diagnostic) + assert x2py_cli.main() == 1 + assert "source.F90:9: error[PREPROCESSOR_FAILED]: bad include" in capsys.readouterr().err + + def fail_without_diagnostic(_paths, _preprocessing): + raise PreprocessingError("plain failure", category="PREPROCESSOR_FAILED") + + monkeypatch.setattr(x2py_cli, "_parse_report", fail_without_diagnostic) + assert x2py_cli.main() == 1 + assert "x2py: error[PREPROCESSOR_FAILED]: plain failure" in capsys.readouterr().err + + +def test_x2py_cli_helpers_cover_language_and_preprocessing_edges(tmp_path: Path, monkeypatch): + class ErrorParser: + def error(self, message): + raise ValueError(message) + + def args(**overrides): + values = { + "defines": [], + "undefs": [], + "preprocess": "raw", + "compiler": None, + "compile_commands": None, + "preprocessor_adapter": "auto", + "preprocess_template": None, + "include_dirs": [], + "std": None, + "compiler_args": [], + "include_exposure": "reachable-project", + "public_includes": [], + "private_includes": [], + "language": "fortran", + } + values.update(overrides) + return types.SimpleNamespace(**values) + + parser = ErrorParser() + api_h = tmp_path / "api.h" + api_h.write_text("int add(int x);\n", encoding="utf-8") + stub = tmp_path / "api.pyi" + stub.write_text("def add(x: Int32) -> Int32: ...\n", encoding="utf-8") + (tmp_path / "notes.txt").write_text("ignore", encoding="utf-8") + + assert x2py_cli._expand_pyi_paths([str(tmp_path), str(stub)]) == [stub] + assert x2py_cli._resolve_language([str(api_h)], "c", parser) == "c" + with pytest.raises(ValueError, match="incompatible with --language fortran"): + x2py_cli._resolve_language([str(api_h)], "fortran", parser) + with pytest.raises(ValueError, match="requires explicit --language c"): + x2py_cli._resolve_language([str(api_h)], None, parser) + with pytest.raises(ValueError, match="Cannot determine"): + x2py_cli._resolve_language([str(tmp_path / "notes.txt")], None, parser) + + with pytest.raises(ValueError, match="--compiler requires --preprocess compiler"): + x2py_cli._build_preprocessing_config(args(compiler="gfortran"), parser) + with pytest.raises(ValueError, match="--preprocess-template requires"): + x2py_cli._build_preprocessing_config( + args( + preprocess="compiler", + compiler="cc", + preprocess_template="{compiler} -E {source}", + ), + parser, + ) + with pytest.raises(ValueError, match="requires --compiler"): + x2py_cli._build_preprocessing_config(args(preprocess="compiler"), parser) + with pytest.raises(ValueError, match="--compile-commands requires"): + x2py_cli._build_preprocessing_config(args(compile_commands="compile_commands.json"), parser) + with pytest.raises(ValueError, match="raw C mode records source macros"): + x2py_cli._build_preprocessing_config(args(language="c", defines=["USE_FAST"]), parser) + with pytest.raises(ValueError, match="internal Fortran parsing does not evaluate CPP branches"): + x2py_cli._build_preprocessing_config(args(defines=["USE_FAST"]), parser) + with pytest.raises(ValueError, match="-I/--include-dir affects Fortran only"): + x2py_cli._build_preprocessing_config(args(include_dirs=["include"]), parser) + + class Recipe: + def to_dict(self): + return {"mode": "compiler"} + + def preprocess(path, *, language, config): + assert path == api_h.with_suffix(".f90") + assert language == "fortran" + assert config.compiler == "gfortran" + return "subroutine work()\nend subroutine work\n", Recipe() + + source = api_h.with_suffix(".f90") + source.write_text("subroutine ignored()\nend subroutine ignored\n", encoding="utf-8") + monkeypatch.setattr(x2py_cli, "run_compiler_preprocessor_with_recipe", preprocess) + code, recipe = x2py_cli._fortran_source_for_path( + source, + PreprocessingConfig(mode="compiler", compiler="gfortran"), + ) + assert "subroutine work" in code + assert recipe == {"mode": "compiler"} + report = x2py_cli._parse_report( + [str(source)], + PreprocessingConfig(mode="compiler", compiler="gfortran"), + ) + assert report[str(source)]["preprocessing_recipe"] == {"mode": "compiler"} + + +def test_x2py_and_fortran_module_entrypoints_and_debug_errors(monkeypatch, capsys): + original_fortran_main = fortran_parser_cli.main + monkeypatch.setattr(x2py_cli, "main", lambda: 0) + with pytest.raises(SystemExit) as x2py_exit: + runpy.run_module("x2py.__main__", run_name="__main__") + assert x2py_exit.value.code == 0 + + monkeypatch.setattr(fortran_parser_cli, "main", lambda: 0) + with pytest.raises(SystemExit) as fortran_exit: + runpy.run_module("fortran_parser.__main__", run_name="__main__") + assert fortran_exit.value.code == 0 + monkeypatch.setattr(fortran_parser_cli, "main", original_fortran_main) + + def fail_parse(_paths): + raise FortranParseError("bad", filename="bad.f90", line_number=1, source_line="bad") + + monkeypatch.setattr(fortran_parser_cli, "_parse_paths", fail_parse) + monkeypatch.setattr(sys, "argv", ["fortran_parser", "bad.f90", "--no-color"]) + assert fortran_parser_cli.main() == 1 + assert "bad.f90:1:1: error[PARSE_ERROR]: bad" in capsys.readouterr().err + monkeypatch.setenv("FORTRAN_PARSER_DEBUG", "1") + with pytest.raises(FortranParseError): + fortran_parser_cli.main() + + +def test_x2py_main_debug_reraises_preprocessing_errors(monkeypatch): + monkeypatch.setattr(sys, "argv", ["x2py", str(TEST_FILE), "--parse"]) + monkeypatch.setenv("X2PY_DEBUG", "1") + + def fail_parse(_paths, _preprocessing): + raise PreprocessingError("plain failure", category="PREPROCESSOR_FAILED") + + monkeypatch.setattr(x2py_cli, "_parse_report", fail_parse) + with pytest.raises(PreprocessingError): + x2py_cli.main() + + def test_cli_out_requires_stage_flag(): cmd = [sys.executable, "-m", "x2py", str(TEST_FILE), "--out"] diff --git a/tests/parser/test_parser_public_entrypoints.py b/tests/parser/test_parser_public_entrypoints.py index f186a7002..d9892418d 100644 --- a/tests/parser/test_parser_public_entrypoints.py +++ b/tests/parser/test_parser_public_entrypoints.py @@ -81,11 +81,6 @@ def test_file_path_and_unknown_filename_public_parse_paths(tmp_path): assert parsed_from_path.procedures[0].name == "from_path" assert parsed_unknown_suffix.format == "unknown" -@pytest.mark.parametrize("parse", [parse_fortran_file, FortranParser().visit_file]) -def test_public_file_parse_rejects_removed_macro_defines_argument(parse): - with pytest.raises(TypeError, match="macro_defines"): - parse("", macro_defines={"USE_MPI"}) - def test_public_instance_visitor_entrypoints_use_source_strings(): parser = FortranParser() diff --git a/tests/parser/test_preprocessing_cli.py b/tests/parser/test_preprocessing_cli.py index 4d10517a5..62fa11f2d 100644 --- a/tests/parser/test_preprocessing_cli.py +++ b/tests/parser/test_preprocessing_cli.py @@ -502,6 +502,18 @@ def test_linemarker_dependency_exposure_and_macro_edges(tmp_path: Path): assert preprocessing._unescape_linemarker_filename("trailing\\") == "trailing\\" assert preprocessing._dependency_kind("") == "system" assert preprocessing._mapping_for_generated_line(mappings, mappings[0].generated_line, root) == mappings[0] + fallback = preprocessing._mapping_for_generated_line([], 99, root) + assert fallback.original_path == str(root) + assert fallback.original_line == 99 + no_filename_mappings = preprocessing.parse_linemarker_mappings("#line 42\nint next;\n", filename=str(root)) + assert no_filename_mappings[0].original_path == str(root) + assert no_filename_mappings[0].original_line == 42 + assert preprocessing._included_files_from_linemarkers( + "#line 5\nint next;\n", + root_path=root, + language="c", + config=PreprocessingConfig(), + ) == [files[0]] assert macros[0].name == "BUILTIN" assert macros[0].builtin is True assert by_path[str(root)].dependency_kind == "root" diff --git a/tests/parser/test_procedure_and_type_parsing.py b/tests/parser/test_procedure_and_type_parsing.py index 1caf842aa..f59fb8491 100644 --- a/tests/parser/test_procedure_and_type_parsing.py +++ b/tests/parser/test_procedure_and_type_parsing.py @@ -899,10 +899,15 @@ def test_fortran_variable_spec_expressions_parse_function_calls(): def test_structured_shape_handles_empty_dimensions_and_use_mapping_equality(): + from fortran_parser.type_resolver import extract_kind_from_type_spec + var = FortranVariable(name="empty", shape=[""]) + assert var.shape_info == [{"raw": "", "lower": None, "upper": None}] shape = var.structured_shape assert shape.raw == [""] assert shape.dimensions == [None] + assert extract_kind_from_type_spec("real", "()") is None + assert extract_kind_from_type_spec("real", "(len=5)") is None renamed = FortranUseMapping(source="delete_input_list", target="delete_input") assert renamed == "delete_input"