diff --git a/.github/workflows/parser-reference-guard.yml b/.github/workflows/parser-reference-guard.yml index 4584f7f28..d0ea4f810 100644 --- a/.github/workflows/parser-reference-guard.yml +++ b/.github/workflows/parser-reference-guard.yml @@ -48,11 +48,14 @@ jobs: C_DOC="docs/developer/c-parser-reference.md" FORTRAN_DOC="docs/developer/fortran-parser-reference.md" + PYI_DOC="docs/user/reference/semantic-pyi-format.md" C_DOC_CHANGED=false FORTRAN_DOC_CHANGED=false + PYI_DOC_CHANGED=false C_PARSER_CHANGED=false FORTRAN_PARSER_CHANGED=false + PYI_PARSER_CHANGED=false SHARED_PARSER_CHANGED=false if grep -Fxq "$C_DOC" <<< "$CHANGED_FILES"; then @@ -61,18 +64,25 @@ jobs: if grep -Fxq "$FORTRAN_DOC" <<< "$CHANGED_FILES"; then FORTRAN_DOC_CHANGED=true fi + if grep -Fxq "$PYI_DOC" <<< "$CHANGED_FILES"; then + PYI_DOC_CHANGED=true + fi while IFS= read -r changed_file; do case "$changed_file" in - c_parser/*|x2py/c_parser/*|tests/parser/c/*|tests/parsing/c/*|tests/data/c/*|tests/probes/test_c_types.py) + x2py/parsers/c/*|tests/parser/c/*|tests/parsing/c/*|tests/data/c/*|tests/probes/test_c_types.py) C_PARSER_CHANGED=true ;; - fortran_parser/*|x2py/fortran_parser/*|tests/parser/fortran/*|tests/parsing/fortran/*|tests/data/fortran/*|tests/probes/test_fortran_types.py) + x2py/parsers/fortran/*|tests/parser/fortran/*|tests/parsing/fortran/*|tests/data/fortran/*|tests/probes/test_fortran_types.py) FORTRAN_PARSER_CHANGED=true ;; + x2py/parsers/pyi/*|tests/parsing/pyi/*|tests/pipeline/pyi_builds/*) + PYI_PARSER_CHANGED=true + ;; tests/parser/conftest.py|\ tests/cli/*|\ tests/pipeline/preprocessing/*|\ + x2py/parsers/__init__.py|\ x2py/preprocessing.py) SHARED_PARSER_CHANGED=true ;; @@ -80,11 +90,11 @@ jobs: done <<< "$CHANGED_FILES" if [[ ",${PR_LABELS}," == *",${FORCE_LABEL},"* ]]; then - if [ "$C_DOC_CHANGED" = true ] || [ "$FORTRAN_DOC_CHANGED" = true ]; then + if [ "$C_DOC_CHANGED" = true ] || [ "$FORTRAN_DOC_CHANGED" = true ] || [ "$PYI_DOC_CHANGED" = true ]; then echo "${FORCE_LABEL} label present and at least one parser reference changed." else echo "${FORCE_LABEL} label present, but no parser reference changed." - echo "Update ${C_DOC} or ${FORTRAN_DOC}, or remove ${FORCE_LABEL}." + echo "Update ${C_DOC}, ${FORTRAN_DOC}, or ${PYI_DOC}, or remove ${FORCE_LABEL}." exit 1 fi fi @@ -101,11 +111,17 @@ jobs: FAILED=true fi + if [ "$PYI_PARSER_CHANGED" = true ] && [ "$PYI_DOC_CHANGED" != true ]; then + echo "Semantic .pyi parser-related files changed without updating ${PYI_DOC}." + FAILED=true + fi + if [ "$SHARED_PARSER_CHANGED" = true ] && \ [ "$C_DOC_CHANGED" != true ] && \ - [ "$FORTRAN_DOC_CHANGED" != true ]; then + [ "$FORTRAN_DOC_CHANGED" != true ] && \ + [ "$PYI_DOC_CHANGED" != true ]; then echo "Shared parser workflow files changed without updating a parser reference." - echo "Update ${C_DOC} or ${FORTRAN_DOC}, whichever behavior changed." + echo "Update ${C_DOC}, ${FORTRAN_DOC}, or ${PYI_DOC}, whichever behavior changed." FAILED=true fi @@ -120,5 +136,8 @@ jobs: if [ "$FORTRAN_DOC_CHANGED" = true ]; then echo "${FORTRAN_DOC} changed." fi + if [ "$PYI_DOC_CHANGED" = true ]; then + echo "${PYI_DOC} changed." + fi echo "Parser reference guard passed." diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index cce04b4ff..89813baee 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -23,7 +23,6 @@ on: env: X2PY_GFORTRAN_BINARY: gfortran-13 X2PY_GFORTRAN_PACKAGE: gfortran-13 - X2PY_REAL_LIBRARY_NATIVE_CACHE_PATH: .pytest_cache/x2py/real-library-native jobs: static-analysis: @@ -50,6 +49,8 @@ jobs: run: python -m ruff check . - name: Ruff format run: python -m ruff format --check . + - name: Wrapper-plan generator contracts + run: python tools/check_wrapper_codegen_complexity.py - name: Bandit security scan run: python -m bandit -c pyproject.toml -r c_parser fortran_parser semantics x2py --severity-level medium --confidence-level medium - name: Vulture dead-code scan @@ -66,67 +67,10 @@ jobs: continue-on-error: true run: python -m radon mi c_parser fortran_parser semantics x2py -s - real-library-native-cache: - name: Real-Library Native Cache - if: ${{ !inputs.static_analysis_only }} - needs: static-analysis - runs-on: ubuntu-24.04 - permissions: - contents: read - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 2 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Install test dependencies - run: | - python -m pip install --upgrade pip - python -m pip install -e ".[qa]" - - name: Install pinned GFortran - shell: bash - run: | - if ! command -v "$X2PY_GFORTRAN_BINARY" >/dev/null 2>&1; then - sudo apt-get update - sudo apt-get install --yes "$X2PY_GFORTRAN_PACKAGE" - fi - compiler_dir="$RUNNER_TEMP/x2py-gfortran" - mkdir -p "$compiler_dir" - ln -sf "$(command -v "$X2PY_GFORTRAN_BINARY")" "$compiler_dir/gfortran" - echo "$compiler_dir" >> "$GITHUB_PATH" - "$compiler_dir/gfortran" --version - - name: Capture native cache key facts - id: native-cache-facts - shell: bash - run: | - echo "gfortran_hash=$(gfortran --version | head -n 1 | sha256sum | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" - - name: Restore real-library native cache - id: restore-real-library-native-cache - uses: actions/cache/restore@v4 - with: - path: ${{ env.X2PY_REAL_LIBRARY_NATIVE_CACHE_PATH }} - key: real-library-native-${{ runner.os }}-${{ runner.arch }}-${{ steps.native-cache-facts.outputs.gfortran_hash }}-${{ hashFiles('tests/data/fortran/blas/**', 'tests/data/fortran/lapack/**', 'tests/wrapper/fortran/real_libraries/test_real_blas_lapack.py', 'tools/warm_real_library_native_cache.py') }} - restore-keys: | - real-library-native-${{ runner.os }}-${{ runner.arch }}-${{ steps.native-cache-facts.outputs.gfortran_hash }}- - - name: Warm real-library native cache - env: - PYTHONPATH: . - X2PY_REAL_LIBRARY_NATIVE_CACHE_DIR: ${{ github.workspace }}/${{ env.X2PY_REAL_LIBRARY_NATIVE_CACHE_PATH }} - run: python tools/warm_real_library_native_cache.py - - name: Save real-library native cache - if: steps.restore-real-library-native-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v4 - with: - path: ${{ env.X2PY_REAL_LIBRARY_NATIVE_CACHE_PATH }} - key: real-library-native-${{ runner.os }}-${{ runner.arch }}-${{ steps.native-cache-facts.outputs.gfortran_hash }}-${{ hashFiles('tests/data/fortran/blas/**', 'tests/data/fortran/lapack/**', 'tests/wrapper/fortran/real_libraries/test_real_blas_lapack.py', 'tools/warm_real_library_native_cache.py') }} - test: name: Tests (Python ${{ matrix.python-version }}) if: ${{ !inputs.static_analysis_only }} - needs: [static-analysis, real-library-native-cache] + needs: static-analysis runs-on: ubuntu-24.04 permissions: contents: read @@ -159,26 +103,17 @@ jobs: ln -sf "$(command -v "$X2PY_GFORTRAN_BINARY")" "$compiler_dir/gfortran" echo "$compiler_dir" >> "$GITHUB_PATH" "$compiler_dir/gfortran" --version - - name: Capture native cache key facts - id: native-cache-facts - shell: bash - run: | - echo "gfortran_hash=$(gfortran --version | head -n 1 | sha256sum | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" - - name: Restore real-library native cache - uses: actions/cache/restore@v4 - with: - path: ${{ env.X2PY_REAL_LIBRARY_NATIVE_CACHE_PATH }} - key: real-library-native-${{ runner.os }}-${{ runner.arch }}-${{ steps.native-cache-facts.outputs.gfortran_hash }}-${{ hashFiles('tests/data/fortran/blas/**', 'tests/data/fortran/lapack/**', 'tests/wrapper/fortran/real_libraries/test_real_blas_lapack.py', 'tools/warm_real_library_native_cache.py') }} - restore-keys: | - real-library-native-${{ runner.os }}-${{ runner.arch }}-${{ steps.native-cache-facts.outputs.gfortran_hash }}- - name: Run tests shell: bash env: PYTHONPATH: . HYPOTHESIS_PROFILE: ci X2PY_COVERAGE_REQUESTED: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-coverage')) || (github.event_name == 'workflow_call' && inputs.coverage) }} - X2PY_REAL_LIBRARY_NATIVE_CACHE_DIR: ${{ github.workspace }}/${{ env.X2PY_REAL_LIBRARY_NATIVE_CACHE_PATH }} run: | + pytest_args=( + --ignore=tests/wrapper/fortran/real_libraries/test_real_blas_lapack.py + ) + if [[ "${{ matrix.python-version }}" == "3.12" ]]; then test_paths=(tests) else @@ -186,9 +121,7 @@ jobs: tests/architecture tests/benchmarks tests/cli - tests/codegen tests/docs - tests/lowering tests/naming tests/parsing tests/pipeline @@ -197,6 +130,7 @@ jobs: tests/semantics tests/tools tests/types + tests/wrapper_codegen tests/wrapper/fortran/arrays tests/wrapper/fortran/build_from_pyi tests/wrapper/fortran/build_from_source @@ -212,7 +146,6 @@ jobs: tests/wrapper/fortran/runtime_behavior tests/wrapper/fortran/scalars tests/wrapper/fortran/strings - "tests/wrapper/fortran/real_libraries/test_real_blas_lapack.py::test_full_library_wrapper_imports_every_root_procedure_from_cached_shared_library[blas]" tests/wrapper/fortran/real_libraries/test_stage7_native_bundles.py ) fi @@ -221,11 +154,13 @@ jobs: COVERAGE_PROCESS_START="${{ github.workspace }}/pyproject.toml" \ python -m coverage run -m pytest -q --randomly-seed=1 \ -o junit_family=legacy \ - --junitxml="$RUNNER_TEMP/pytest-results.xml" "${test_paths[@]}" + --junitxml="$RUNNER_TEMP/pytest-results.xml" \ + "${pytest_args[@]}" "${test_paths[@]}" else python -m pytest -q --randomly-seed=1 \ -o junit_family=legacy \ - --junitxml="$RUNNER_TEMP/pytest-results.xml" "${test_paths[@]}" + --junitxml="$RUNNER_TEMP/pytest-results.xml" \ + "${pytest_args[@]}" "${test_paths[@]}" fi - name: Upload coverage data if: matrix.python-version == '3.12' && ((github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-coverage')) || (github.event_name == 'workflow_call' && inputs.coverage)) @@ -239,6 +174,59 @@ jobs: if: failure() run: python tools/print_pytest_failures.py "$RUNNER_TEMP/pytest-results.xml" + real-library-wrappers: + name: Real library wrappers (BLAS + LAPACK) + if: >- + ${{ + !inputs.static_analysis_only && + (github.event_name != 'pull_request' || + !contains(github.event.pull_request.labels.*.name, 'ignore-real-library-wrappers')) + }} + needs: static-analysis + runs-on: ubuntu-24.04 + timeout-minutes: 120 + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 2 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install test dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[qa]" + - name: Install pinned GFortran + shell: bash + run: | + if ! command -v "$X2PY_GFORTRAN_BINARY" >/dev/null 2>&1; then + sudo apt-get update + sudo apt-get install --yes "$X2PY_GFORTRAN_PACKAGE" + fi + compiler_dir="$RUNNER_TEMP/x2py-gfortran" + mkdir -p "$compiler_dir" + ln -sf "$(command -v "$X2PY_GFORTRAN_BINARY")" "$compiler_dir/gfortran" + echo "$compiler_dir" >> "$GITHUB_PATH" + "$compiler_dir/gfortran" --version + - name: Restore compiled native library cache + uses: actions/cache@v4 + with: + path: ${{ runner.temp }}/x2py-real-library-native + key: real-libraries-${{ runner.os }}-gfortran13-${{ hashFiles('tests/data/fortran/blas/**', 'tests/data/fortran/lapack/**') }} + - name: Run full BLAS and LAPACK wrapper tests + env: + PYTHONPATH: . + HYPOTHESIS_PROFILE: ci + X2PY_REAL_LIBRARY_NATIVE_CACHE_DIR: ${{ runner.temp }}/x2py-real-library-native + run: | + python -m pytest -q --randomly-seed=1 \ + "tests/wrapper/fortran/real_libraries/test_real_blas_lapack.py::test_full_library_wrapper_imports_every_root_procedure_from_cached_shared_library[blas]" \ + "tests/wrapper/fortran/real_libraries/test_real_blas_lapack.py::test_full_library_wrapper_imports_every_root_procedure_from_cached_shared_library[lapack]" + coverage-report: name: Coverage Report if: ${{ !inputs.static_analysis_only && ((github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-coverage')) || (github.event_name == 'workflow_call' && inputs.coverage)) }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 793f28ab6..425560d54 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,12 +11,12 @@ This repo includes a CI guard that may require updating parser reference docs when parser-related files change. -- **C parser changes**: if you change `x2py/c_parser/`, `tests/parser/c/`, or +- **C parser changes**: if you change `x2py/parsers/c/`, `tests/parser/c/`, or `tests/data/c/`, update `docs/c_parser.md` when the change affects the documented feature inventory, public API, diagnostics, fixtures, semantic handoff, or maintenance workflow. The guard also treats `tests/probes/test_c_types.py` as C parser related. -- **Fortran parser changes**: if you change `x2py/fortran_parser/`, +- **Fortran parser changes**: if you change `x2py/parsers/fortran/`, `tests/parser/fortran/`, or `tests/data/fortran/`, update `docs/fortran_parser.md` when the change affects the documented feature inventory, public API, diagnostics, fixtures, semantic handoff, or diff --git a/README.md b/README.md index f1b27c8ea..28a711f4b 100644 --- a/README.md +++ b/README.md @@ -47,16 +47,18 @@ Build it with the default output locations: python3 -m x2py scale.f90 ``` -By default, x2py writes the importable `.so` beside the input source and keeps -generated build intermediates under `__x2py__/`: +By default, x2py writes generated build artifacts, including the ABI-suffixed +extension, under `__x2py__/` in the directory where you run the command. A +direct CLI build also writes a stable `.so` import alias there: ```text . scale.f90 scale.so __x2py__/ + scale..so generated-wrapper sources - x2py_runtime/ + binding_support/ ``` Name the Python extension and final `.so` explicitly with `--out NAME`: @@ -72,16 +74,17 @@ Expected result: scale.f90 SCALE.so __x2py__/ + SCALE..so generated-wrapper sources - x2py_runtime/ + binding_support/ ``` For a wrapper build, `--out SCALE` selects the Python module name and the final shared-library filename. This first example is a standalone procedure, so it is exposed directly at the extension root. -Use `--out-dir` when you want the shared library and generated intermediates in -an explicit build directory: +Use `--out-dir` when you want the ABI-specific shared library and generated +intermediates in an explicit build directory: ```bash python3 -m x2py scale.f90 \ @@ -92,10 +95,12 @@ python3 -m x2py scale.f90 \ Expected result: ```text -build/SCALE/ +. SCALE.so - generated-wrapper sources - x2py_runtime/ + build/SCALE/ + SCALE..so + generated-wrapper sources + binding_support/ ``` Generate the semantic `.pyi` contract for the same source: @@ -116,6 +121,8 @@ contracts/ Expected contract (`contracts/__init__.pyi`): ```python +from x2py.contracts import Addr, Arg, Float64, external, native_call + @external @native_call([Addr(Arg(0)), Addr(Arg(1))]) def scale( @@ -124,12 +131,17 @@ def scale( ) -> Float64: ... ``` +The semantic contract does not repeat Fortran `intent`. Source `intent` helps +x2py choose the generated Python arguments and results, while `@native_call` +records the exact native argument order and transport. The compiled native +procedure keeps its own `intent`; editing the `.pyi` changes the wrapper call +contract, not the native procedure declaration. + Then build the shared library from the package-entry `.pyi` contract and the same native implementation source: ```bash python3 -m x2py contracts/__init__.pyi \ - --wrap \ --native-fortran-sources scale.f90 \ --out SCALE \ --out-dir build/SCALE_from_pyi @@ -141,10 +153,12 @@ Use `--out NAME` with wrapper builds when you want the import name and final The `.pyi` build produces the same importable extension shape: ```text -build/SCALE_from_pyi/ +. SCALE.so - generated-wrapper sources - x2py_runtime/ + build/SCALE_from_pyi/ + SCALE..so + generated-wrapper sources + binding_support/ ``` The direct source build exposes the standalone procedure at the extension root: @@ -213,6 +227,8 @@ python3 -m x2py points.f90 --pyi --out contracts Expected contract (`contracts/points.pyi`): ```python +from x2py.contracts import Addr, Arg, Float64, native_call + class point: def __init__( self, @@ -299,8 +315,9 @@ The runtime wrapper mechanism is: Fortran sources -> compiler preprocessing and target-type probing -> Fortran parser - -> semantic IR and readiness validation - -> generated native bridge and Python binding + -> semantic IR construction and readiness validation + -> post-IR policy completion and ordered wrapper plan + -> direct native-bridge and Python-binding lowering -> native compilation and shared-library link -> importable Python extension ``` @@ -310,9 +327,10 @@ Fortran sources Fortran sources -> compiler preprocessing and target-type probing -> Fortran parser - -> semantic IR and readiness validation - -> generated Fortran bind(C) bridge - -> generated C/CPython binding and x2py runtime support + -> semantic IR construction and readiness validation + -> post-IR policy completion and ordered wrapper plan + -> direct Fortran bind(C) bridge lowering + -> direct C/CPython binding lowering and native binding support -> native compilation and shared-library link -> importable Python extension ``` diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md deleted file mode 100644 index a314ba66e..000000000 --- a/THIRD_PARTY_NOTICES.md +++ /dev/null @@ -1,12 +0,0 @@ -# Third-Party Notices - -Some code in this repository was adapted from the Pyccel project. - -Pyccel is licensed under the MIT License: - -Copyright (c) 2017-2020, Pyccel Developers. - -The MIT License permits use, copying, modification, merging, publishing, -distribution, sublicensing, and selling copies of the software, provided that -the copyright notice and permission notice are included in copies or substantial -portions of the software. diff --git a/docs/developer/c-parser-reference.md b/docs/developer/c-parser-reference.md index 33cad4876..bd859a891 100644 --- a/docs/developer/c-parser-reference.md +++ b/docs/developer/c-parser-reference.md @@ -14,7 +14,7 @@ status: maintained X2PY_C_DOCS_END --> ## Parser Organization Notes @@ -471,7 +471,7 @@ X2PY_C_DOCS_END --> @@ -484,7 +484,7 @@ Implemented top-level and package entrypoints: ```python from x2py import parse_c_file, parse_c_project # Equivalent parser-package imports remain available: -# from x2py.c_parser import parse_c_file, parse_c_project +# from x2py.parsers.c import parse_c_file, parse_c_project ``` X2PY_C_DOCS_END --> diff --git a/docs/developer/development-workflow.md b/docs/developer/development-workflow.md index bb50dfde1..5f5c749ac 100644 --- a/docs/developer/development-workflow.md +++ b/docs/developer/development-workflow.md @@ -203,41 +203,41 @@ implementation files. | User-visible area | Main implementation files | Main tests | | --- | --- | --- | -| Fortran parse output | `x2py/fortran_parser/parser.py`, `x2py/fortran_parser/models.py`, `x2py/fortran_parser/lexer.py` | `tests/parsing/fortran/`, `tests/parsing/fortran/test_fortran_fixture_suite.py`, `tests/parsing/fortran/test_error_handling.py` | -| CLI stage selection and output | `x2py/cli.py`, `x2py/fortran_parser/cli.py` | `tests/cli/` | +| Fortran parse output | `x2py/parsers/fortran/parser.py`, `x2py/parsers/fortran/models.py`, `x2py/parsers/fortran/lexer.py` | `tests/parsing/fortran/`, `tests/parsing/fortran/test_fortran_fixture_suite.py`, `tests/parsing/fortran/test_error_handling.py` | +| CLI stage selection and output | `x2py/cli.py`, `x2py/parsers/fortran/cli.py` | `tests/cli/` | | Fortran target type probing and cache | `x2py/probes/fortran_types.py` | `tests/probes/test_fortran_types.py` | | Generated target datatype mapping examples | `x2py/probes/report.py` | `tests/types/test_mapping_report.py`, `tests/docs/test_examples.py` | | Fortran to semantic IR | `x2py/semantics/fortran2ir.py`, `x2py/semantics/models.py` | `tests/semantics/conversion/fortran/` | -| `.pyi` printing | `x2py/codegen/printers/pyi_printer.py` | `tests/codegen/printers/`, `tests/codegen/printers/test_modern_example.py` | -| `.pyi` parsing/loading/editing | `x2py/pyi_parser/parser.py`, `x2py/pipeline/pyi.py`, `x2py/semantics/pyi2ir.py` | `tests/parsing/pyi/`, `tests/pipeline/pyi_builds/test_contract_fixtures.py` | -| Semantic policy completion | `x2py/semantics/policy_completion.py`, `x2py/semantics/ownership.py` | `tests/semantics/policy/`, `tests/lowering/test_semantic_ir.py` | +| `.pyi` printing | `x2py/wrapper_codegen/printers/pyi_printer.py` | `tests/wrapper_codegen/printers/`, `tests/wrapper_codegen/printers/test_modern_example.py` | +| `.pyi` parsing/loading/editing | `x2py/parsers/pyi/parser.py`, `x2py/pipeline/pyi.py`, `x2py/semantics/pyi2ir.py` | `tests/parsing/pyi/`, `tests/pipeline/pyi_builds/test_contract_fixtures.py` | +| Semantic policy completion | `x2py/semantics/policy_completion.py`, `x2py/semantics/ownership.py` | `tests/semantics/policy/` | | Readiness reports | `x2py/semantics/readiness.py` | `tests/semantics/readiness/`, `tests/semantics/readiness/test_wrap_readiness_fixture_suite.py` | | Fortran wrapper orchestration | `x2py/pipeline/build.py` | `tests/wrapper/fortran/build_from_source/test_build_modes.py`, `tests/wrapper/fortran/multiple_files/test_multi_source_builds.py` | -| Semantic IR to codegen AST | `x2py/semantics/ir2ast.py` | `tests/lowering/test_semantic_ir.py`, `tests/wrapper/` | -| Native compilation and runtime support | `x2py/compiling/`, `x2py/stdlib/x2py_runtime/` | `tests/wrapper/fortran/build_from_source/test_runtime_abi.py`, `tests/wrapper/fortran/build_from_source/test_build_modes.py` | +| Wrapper planning and direct lowering | `x2py/wrapper_codegen/plan.py`, `x2py/wrapper_codegen/planner.py`, `x2py/wrapper_codegen/generator.py` | `tests/wrapper_codegen/`, `tests/wrapper/` | +| Native compilation and binding support | `x2py/compiling/`, `x2py/binding_support/` | `tests/wrapper/fortran/build_from_source/test_runtime_abi.py`, `tests/wrapper/fortran/build_from_source/test_build_modes.py` | | Executable Markdown examples | `README.md`, `docs/*.md` | `tests/docs/test_examples.py` | -### Codegen Class Organization +### Wrapper Generator Class Organization Organize generators and printers using `FortranParser` in -`x2py/fortran_parser/parser.py` as the structural reference. A developer +`x2py/parsers/fortran/parser.py` as the structural reference. A developer should be able to read each class from top to bottom in the same order that data moves through it: @@ -278,21 +278,27 @@ module-level function only to preserve an old internal call path. ### `.pyi` Contract Internals User-visible `.pyi` syntax is first parsed to Python AST by -`x2py/pyi_parser/parser.py`, loaded from text/files by +`x2py/parsers/pyi/parser.py`, loaded from text/files by `x2py/pipeline/pyi.py`, converted to semantic IR by `x2py/semantics/pyi2ir.py`, and printed by -`x2py/codegen/printers/pyi_printer.py`. The converter and printer operate on +`x2py/wrapper_codegen/printers/pyi_printer.py`. The converter and printer operate on `x2py/semantics/models.py`. Important implementation rules: - `Addr(T)` and `Addr(T)` are storage contracts, not just pretty syntax. - Array subscriptions such as `Float64[n]` are semantic array contracts. -- `Annotated[..., ORDER_F]` and `ORDER_ANY` are array storage metadata. +- `Annotated[..., ORDER_F]` and `ORDER_ANY` are non-default array storage + metadata. Plain multidimensional Fortran `.pyi` arrays use `ORDER_F`; do not + print or retain that default marker in a generated contract. `Allocatable[T[...]]` and `Pointer[T[...]]` are descriptor-handle wrappers around the array storage contract. Output and writeback behavior is represented by writable storage plus `Returns["name", T]` when a Python result is projected. + - `Final[T]` is the public constant spelling. Do not reintroduce `Constant` as user-facing `.pyi` syntax. - `@native_call` is projection metadata. Use it only when the Python-visible @@ -300,11 +306,20 @@ Important implementation rules: - Generated stubs should preserve behavior-changing native contracts while staying compact; exact source intent that does not change execution can stay in semantic IR instead of the printed `.pyi`. +- Use `SourceName("...")` only when a source identifier cannot be used as the + Python target. Do not infer source identifiers from normalized Python names. +- Binding locals derived from a Python-visible argument must use the reserved + `bound_` namespace. Generated binding sources include Python, standard-library, + optional descriptor, NumPy, and runtime headers, so their imported identifier + sets are not a stable public-name vocabulary. +- Omit `Polymorphic` only for the passed-object dummy of a type-bound procedure, + where the binding itself restores that native fact. Ordinary `class(T)` + arguments must retain it. When changing `.pyi` syntax: 1. Add or update parser tests in `tests/parsing/pyi/`. -2. Add or update printer tests in `tests/codegen/printers/`. +2. Add or update printer tests in `tests/wrapper_codegen/printers/`. 3. Update fixture tests only if the public generated contract changes. 4. Update [Basic wrapper tutorial](../user/tutorials/basic-wrapper.md) or [Verified examples cookbook](../user/examples/verified-cookbook.md) if users need to write or read the new syntax. @@ -419,8 +434,9 @@ CLI args -> preprocessing config and source loading -> parser models -> semantic IR + -> post-IR policy completion -> inspection: .pyi printing / .pyi loading / readiness report - -> Fortran build: codegen AST / native bridge / CPython binding / extension + -> Fortran build: WrapperPlan / direct bridge and binding lowering / extension ``` X2PY_C_DOCS_END --> @@ -446,7 +462,7 @@ C files and directories require explicit language selection. Keep this behavior tested in `tests/cli/` whenever stage selection changes. X2PY_C_DOCS_END --> -The package-specific `x2py/fortran_parser/cli.py` remains for the Fortran parser +The package-specific `x2py/parsers/fortran/cli.py` remains for the Fortran parser package entrypoint. New cross-language user behavior normally belongs in `x2py/cli.py`. @@ -676,7 +692,7 @@ CLI `.pyi` readiness: ```text .pyi path(s) or directory - -> x2py/pyi_parser/parser.py + -> x2py/parsers/pyi/parser.py -> x2py/pipeline/pyi.py pyi_paths_to_semantic_modules(...) -> x2py/semantics/pyi2ir.py -> SemanticModule list @@ -713,7 +729,7 @@ report = assess_semantic_wrap_readiness(modules, source="interfaces") Use the `.pyi` helpers by input shape: -- `parse_pyi_text(source, filename=...)` from `x2py.pyi_parser` for parser-only +- `parse_pyi_text(source, filename=...)` from `x2py.parsers.pyi` for parser-only AST parsing. - `convert_pyi_to_ir(tree, module_name=..., source=...)` from `pyi2ir.py` for AST-to-IR conversion. @@ -828,8 +844,8 @@ ordered source paths -> compile-time expression and storage probes -> fortran_project_to_semantic_modules(...) -> merge public semantic modules - -> semantic_ir_to_codegen_ast(...) - -> Codegen and create_shared_library(...) + -> WrapperPlanner and WrapperCodeGenerator + -> create_shared_library(...) -> WrapperBuildResult ``` @@ -838,16 +854,18 @@ The main ownership boundaries are: - `x2py/pipeline/build.py`: source order, preprocessing/probing, semantic merge, `.pyi` entry-contract loading, native build plan assembly, output placement, direct-versus-Makefile mode, and artifact reporting; -- `x2py/semantics/ir2ast.py`: semantic contract validation and conversion to - codegen models; +- `x2py/wrapper_codegen/planner.py`: projection from completed semantic policy + into validated typed plans; +- `x2py/wrapper_codegen/generator.py`: direct bridge, binding, and source + artifact generation; - `x2py/compiling/`: compiler commands and shared-library linking; and -- `x2py/stdlib/x2py_runtime/`: native runtime support copied into each build. +- `x2py/binding_support/`: native binding support copied into each build. Do not move semantic ownership or projection policy into printers. Do not infer @@ -889,9 +907,9 @@ rather than "what Python wrapper should be generated?" Fortran: -- `x2py/fortran_parser/parser.py` slices the file into grammar units, then parses +- `x2py/parsers/fortran/parser.py` slices the file into grammar units, then parses each unit's specification region. -- `x2py/fortran_parser/models.py` stores `FortranFile`, modules, procedures, +- `x2py/parsers/fortran/models.py` stores `FortranFile`, modules, procedures, variables, derived types, interfaces, programs, submodules, and diagnostics. - Execution bodies are intentionally skipped after the parser has enough signature/source facts. @@ -901,11 +919,11 @@ C: X2PY_C_DOCS_END --> @@ -926,8 +944,8 @@ X2PY_C_DOCS_END --> - `x2py/semantics/fortran2ir.py` maps Fortran procedures, derived types, module variables, kinds, shapes, storage contracts, visibility, imported references, and compile-time values. -- `x2py/codegen/printers/pyi_printer.py` emits editable user contracts. -- `x2py/pyi_parser/parser.py` parses edited contracts to Python AST. +- `x2py/wrapper_codegen/printers/pyi_printer.py` emits editable user contracts. +- `x2py/parsers/pyi/parser.py` parses edited contracts to Python AST. - `x2py/pipeline/pyi.py` converts edited contract text, files, and path sets. - `x2py/semantics/pyi2ir.py` converts parsed `.pyi` AST back into semantic IR. - `x2py/semantics/native_contract.py` validates immutable native scope, ABI, @@ -979,7 +997,7 @@ status-return policy, ownership conversion, or coercion execution. The test ownership is: - loader syntax and error behavior: `tests/parsing/pyi/`; -- printer round-trip shape: `tests/codegen/printers/`; +- printer round-trip shape: `tests/wrapper_codegen/printers/`; - readiness interpretation: `tests/semantics/readiness/`. `tests/parsing/c/test_c_declarations_and_declarators.py`, `tests/parsing/c/test_c_compiler_extensions.py`, or `tests/parsing/c/test_c_structs_unions_enums_typedefs.py`. -2. Implement the parser change in `x2py/c_parser/parser.py`. Add or update model - fields in `x2py/c_parser/models.py` only if the serialized parser contract needs +2. Implement the parser change in `x2py/parsers/c/parser.py`. Add or update model + fields in `x2py/parsers/c/models.py` only if the serialized parser contract needs new facts. 3. If source splitting or raw directive handling changes, update - `x2py/c_parser/lexer.py` and `tests/parsing/c/test_c_lexer_preprocessor.py`. + `x2py/parsers/c/lexer.py` and `tests/parsing/c/test_c_lexer_preprocessor.py`. 4. If project-level resolution changes, update `tests/parsing/c/test_c_project_resolution.py`. 5. If parser JSON changes intentionally, regenerate the relevant project @@ -1110,7 +1128,7 @@ X2PY_C_DOCS_END --> 3. Keep the public semantic dtype names in `x2py/semantics/models.py` stable unless there is a deliberate schema decision. 4. If the emitted `.pyi` annotation changes, update - `tests/codegen/printers/` and `tests/parsing/pyi/`. + `tests/wrapper_codegen/printers/` and `tests/parsing/pyi/`. 5. Update the datatype tables in [Semantic IR reference](../user/reference/semantic-ir.md), and update [Basic wrapper tutorial](../user/tutorials/basic-wrapper.md) or [Verified examples cookbook](../user/examples/verified-cookbook.md) when a visible example changes. @@ -1199,13 +1217,13 @@ Focused verification: ```bash PYTHONPATH=. pytest -q tests/semantics/conversion/fortran/ -PYTHONPATH=. pytest -q tests/codegen/printers/ tests/parsing/pyi/ +PYTHONPATH=. pytest -q tests/wrapper_codegen/printers/ tests/parsing/pyi/ ``` @@ -1216,10 +1234,10 @@ Example target: add a new `Annotated[...]` metadata item or projection helper. 1. Add loader tests in `tests/parsing/pyi/`. 2. Update `x2py/semantics/pyi2ir.py`. Update `x2py/pipeline/pyi.py` when loading or cross-file reconciliation changes. Update - `x2py/pyi_parser/parser.py` only when the raw Python AST parsing boundary + `x2py/parsers/pyi/parser.py` only when the raw Python AST parsing boundary changes. -3. Add printer tests in `tests/codegen/printers/`. -4. Update `x2py/codegen/printers/pyi_printer.py`. +3. Add printer tests in `tests/wrapper_codegen/printers/`. +4. Update `x2py/wrapper_codegen/printers/pyi_printer.py`. 5. Update semantic models in `x2py/semantics/models.py` only if the IR needs a new field or constraint. 6. Update readiness behavior if the new syntax resolves a blocker. @@ -1230,7 +1248,7 @@ Focused verification: ```bash PYTHONPATH=. pytest -q tests/parsing/pyi/ -PYTHONPATH=. pytest -q tests/codegen/printers/ +PYTHONPATH=. pytest -q tests/wrapper_codegen/printers/ PYTHONPATH=. pytest -q tests/semantics/readiness/ ``` @@ -1287,7 +1305,7 @@ diagnostic formatting. 1. Add CLI tests in `tests/cli/` first. 2. Implement shared dispatch and output behavior in `x2py/cli.py`. -3. Keep Fortran package-specific CLI behavior in `x2py/fortran_parser/cli.py`. +3. Keep Fortran package-specific CLI behavior in `x2py/parsers/fortran/cli.py`. 4. If compiler preprocessing behavior changes, update `x2py/pipeline/preprocessing.py` and preprocessing tests. 5. Update [Basic wrapper tutorial](../user/tutorials/basic-wrapper.md) or [Verified examples cookbook](../user/examples/verified-cookbook.md) for @@ -1506,7 +1524,7 @@ Focused tests by concern: - Semantic readiness: `PYTHONPATH=. pytest -q tests/semantics/readiness/` - `.pyi` printer: - `PYTHONPATH=. pytest -q tests/codegen/printers/` + `PYTHONPATH=. pytest -q tests/wrapper_codegen/printers/` - `.pyi` loader and edited stub behavior: `PYTHONPATH=. pytest -q tests/parsing/pyi/` - Semantic and `.pyi` fixtures: @@ -1528,7 +1546,7 @@ python tests/pyi/generate_pyi_fixtures.py ``` Executable examples: `tests/semantics/readiness/`, -`tests/codegen/printers/`, and `tests/parsing/pyi/`. +`tests/wrapper_codegen/printers/`, and `tests/parsing/pyi/`. ### CLI diff --git a/docs/developer/feature-to-code-map.md b/docs/developer/feature-to-code-map.md index f1c00319e..e43f2692b 100644 --- a/docs/developer/feature-to-code-map.md +++ b/docs/developer/feature-to-code-map.md @@ -16,22 +16,22 @@ before documentation may call the behavior supported. | Feature or behavior | Public docs | Main implementation files | Focused tests | Support evidence | | --- | --- | --- | --- | --- | -| Fortran parse output | `docs/developer/fortran-parser-reference.md` | `x2py/fortran_parser/parser.py`, `models.py`, `lexer.py`, `type_resolver.py` | `tests/parsing/fortran/`, `tests/parsing/fortran/test_fortran_fixture_suite.py` | Parser facts and diagnostics match fixtures | -| Semantic `.pyi` generation | `docs/user/reference/semantic-pyi-format.md` | `x2py/codegen/printers/pyi_printer.py` | `tests/codegen/printers/`, `tests/codegen/printers/test_modern_example.py` | Printed `.pyi` round-trips or matches fixtures | -| Semantic `.pyi` conversion and editing | `docs/user/guide/editing-semantic-pyi-contracts.md`, `docs/user/reference/semantic-pyi-format.md`, `docs/user/examples/recipes/semantic-pyi-contracts.md` | `x2py/pyi_parser/parser.py`, `x2py/pipeline/pyi.py`, `x2py/semantics/pyi2ir.py`, `models.py` | `tests/parsing/pyi/`, `tests/pipeline/pyi_builds/test_contract_fixtures.py` | Edited contracts parse to Python AST, then become semantic IR with preserved native facts | +| Fortran parse output | `docs/developer/fortran-parser-reference.md` | `x2py/parsers/fortran/parser.py`, `models.py`, `lexer.py`, `type_resolver.py` | `tests/parsing/fortran/`, `tests/parsing/fortran/test_fortran_fixture_suite.py` | Parser facts and diagnostics match fixtures | +| Semantic `.pyi` generation | `docs/user/reference/semantic-pyi-format.md` | `x2py/wrapper_codegen/printers/pyi_printer.py` | `tests/wrapper_codegen/printers/`, `tests/wrapper_codegen/printers/test_modern_example.py` | Printed `.pyi` round-trips or matches fixtures | +| Semantic `.pyi` conversion and editing | `docs/user/guide/editing-semantic-pyi-contracts.md`, `docs/user/reference/semantic-pyi-format.md`, `docs/user/examples/recipes/semantic-pyi-contracts.md` | `x2py/parsers/pyi/parser.py`, `x2py/pipeline/pyi.py`, `x2py/semantics/pyi2ir.py`, `models.py` | `tests/parsing/pyi/`, `tests/pipeline/pyi_builds/test_contract_fixtures.py` | Edited contracts parse to Python AST, then become semantic IR with preserved native facts | | Readiness blockers | `docs/user/reference/diagnostic-codes.md`, `docs/user/reference/semantic-pyi-format.md` | `x2py/semantics/readiness.py` | `tests/semantics/readiness/`, readiness fixture tests | Unsupported or incomplete contracts fail before codegen | | Fortran wrapper orchestration | `docs/user/guide/fortran-wrapper.md`, `docs/user/examples/recipes/build-and-import-cli.md`, `docs/user/examples/recipes/build-multiple-fortran-sources.md` | `x2py/pipeline/build.py` | `tests/wrapper/fortran/build_from_source/test_build_modes.py`, multi-source wrapper tests | Builds report artifacts and compile/link as documented | -| Semantic IR to codegen AST | `docs/user/guide/fortran-wrapper.md` | `x2py/semantics/ir2ast.py`, `x2py/semantics/ownership.py` | `tests/lowering/test_semantic_ir.py`, `tests/wrapper/fortran/` | Runtime policy is explicit and unsupported cases block | -| Native compilation and runtime support | `docs/user/guide/fortran-wrapper.md`, `docs/user/examples/recipes/generate-editable-makefile.md`, `docs/developer/build-system.md`, `docs/developer/quality-assurance.md` | `x2py/compiling/`, `x2py/stdlib/x2py_runtime/` | `tests/wrapper/fortran/build_from_source/test_runtime_abi.py`, build-mode tests | Generated sources compile, link, import, and clean up correctly | +| Completed semantic policy to wrapper artifacts | `docs/user/guide/fortran-wrapper.md` | `x2py/semantics/policy_completion.py`, `x2py/wrapper_codegen/plan.py`, `planner.py`, `generator.py` | `tests/semantics/policy/`, `tests/wrapper_codegen/`, `tests/wrapper/fortran/` | Runtime policy is explicit, the typed plan is complete, and generated artifacts compile and run | +| Native compilation and binding support | `docs/user/guide/fortran-wrapper.md`, `docs/user/examples/recipes/generate-editable-makefile.md`, `docs/developer/build-system.md`, `docs/developer/quality-assurance.md` | `x2py/compiling/`, `x2py/binding_support/` | `tests/wrapper/fortran/build_from_source/test_runtime_abi.py`, build-mode tests | Generated sources compile, link, import, and clean up correctly | | Source documentation structure | `docs/developer/source-map.md` | `docs/`, package README files, `tests/docs/test_structure.py` | documentation structure and example tests | Pages have metadata, audience separation, and source coverage checks | @@ -41,9 +41,9 @@ X2PY_C_DOCS_END --> For a feature change, start with the implementation file named in the feature map and read only the downstream files that the change actually crosses. For example, a CLI output change normally starts and ends in `x2py/cli.py`, while a -wrapper output-projection change must move through -`x2py/semantics/ir2ast.py`, `x2py/semantics/ownership.py`, the bridge generator, -and the CPython binding generator. +wrapper output-projection change must move through semantic policy completion, +the typed wrapper planner, and the selected bridge and binding implementation +methods. X2PY_C_DOCS_END --> When the user-visible behavior changes, update the public docs in the same row @@ -54,9 +54,9 @@ this routing page tied to the source hotspots and package README files. | User workflow | Start in code | Do not mark supported until | | --- | --- | --- | -| Wrapping functions and subroutines | `x2py/semantics/fortran2ir.py`, `x2py/semantics/ir2ast.py`, bridge and binding generators | Runtime tests compile, import, call, and verify return and failure behavior | +| Wrapping functions and subroutines | `x2py/semantics/fortran2ir.py`, policy completion, `x2py/wrapper_codegen/planner.py`, bridge and binding generators | Runtime tests compile, import, call, and verify return and failure behavior | | Wrapping modules and module variables | parser module facts, semantic module conversion, naming policy, wrapper generators | Python-visible names, accessors, and unsupported module constructs are tested | -| Arrays and allocatables | semantic array contracts, `ir2ast`, ownership policy, bridge/binding array handlers | dtype, shape, rank, contiguity, mutation, returned arrays, and failure paths are tested | +| Arrays and allocatables | semantic array contracts, ownership policy, typed wrapper plans, bridge/binding array handlers | dtype, shape, rank, contiguity, mutation, returned arrays, and failure paths are tested | | Pointer arguments | semantic metadata, ownership policy, bridge/binding pointer handlers | Owner, lifetime, association, and blocked cases are explicit and tested | | Optional arguments | parser optional attributes, semantic arguments, binding argument parsing | Present/absent calls and unsupported combinations are tested | | Generic interfaces | parser interface facts, semantic overload sets, `FunctionOverloadSet`, binding dispatch | Overload selection and ambiguity failures are tested at runtime | @@ -65,7 +65,7 @@ this routing page tied to the source hotspots and package README files. diff --git a/docs/developer/fortran-parser-reference.md b/docs/developer/fortran-parser-reference.md index bcc6dd31f..0bf105a9e 100644 --- a/docs/developer/fortran-parser-reference.md +++ b/docs/developer/fortran-parser-reference.md @@ -93,7 +93,7 @@ Supported public API: ## Parser organization notes -`x2py/fortran_parser/parser.py` is now intentionally organized into clearly labeled +`x2py/parsers/fortran/parser.py` is now intentionally organized into clearly labeled sections and carries embedded implementation guidance. Start with the thin public wrappers at the bottom, then read the class from top to bottom: @@ -116,11 +116,11 @@ wrappers at the bottom, then read the class from top to bottom: Parser methods carry focused docstrings, with examples where a grammar visitor or lexical helper is easier to understand from a concrete call. -The Fortran parser is now packaged under `x2py.fortran_parser` rather than a +The Fortran parser is now packaged under `x2py.parsers.fortran` rather than a top-level parser package. The package includes its CLI module, lexer, JSON-compatible parse models, project parser, type resolver, and utility helpers. Public callers should use the stable top-level `x2py` parser exports -or `x2py.fortran_parser` package imports. +or `x2py.parsers.fortran` package imports. ## Implementation Inventory And Maintenance @@ -130,10 +130,10 @@ testing workflow, and maintenance guard policy live here. The implementation inventory is maintained across these surfaces: -- `x2py/fortran_parser/parser.py` owns source slicing, declaration extraction, +- `x2py/parsers/fortran/parser.py` owns source slicing, declaration extraction, diagnostics, project ordering, dependency resolution, and compile-time expression resolution. -- `x2py/fortran_parser/models.py` owns parse-only dataclasses and JSON-compatible +- `x2py/parsers/fortran/models.py` owns parse-only dataclasses and JSON-compatible parser facts. - `x2py/semantics/fortran2ir.py` owns conversion from parser facts to semantic IR, including kind mapping, compile-time specialization, storage contracts, @@ -146,9 +146,9 @@ The implementation inventory is maintained across these surfaces: @@ -228,21 +231,12 @@ reports advisory/manual. issues, Ruff formatting drift, Vulture unused test parameters, and the too-strict Radon policy. -**Native artifact cache:** the Quality workflow pins the test runner to -`ubuntu-24.04`, installs `gfortran-13`, and warms -`.pytest_cache/x2py/real-library-native` in a dedicated pre-matrix job. The -Python matrix restores that exact cache before pytest and sets -`X2PY_REAL_LIBRARY_NATIVE_CACHE_DIR` to the restored path. Requested coverage -runs collect Python 3.12 coverage data; a final coverage job combines that -artifact and uploads the XML report. This cache holds the full BLAS/LAPACK -object files, archives, and shared libraries used by the real-library wrapper -tests. Cache keys include the runner OS, runner -architecture, pinned `gfortran` version, BLAS/LAPACK source content, and native -cache helper code. Native object files are not portable across different -platforms, compilers, compiler flags, or source revisions; a key change -intentionally rebuilds them. Cold object builds compile independent sources in -parallel after required module sources; set `X2PY_REAL_LIBRARY_NATIVE_JOBS` to -override the bounded worker count. +**Native artifact cache:** dedicated Python 3.12 BLAS and LAPACK jobs restore a +separate runner-local native cache for each library before executing the full +wrapper test. The ordinary pytest matrix excludes that full corpus while +retaining the lighter native-bundle tests. Requested coverage runs still +collect Python 3.12 coverage data; a final coverage job combines that artifact +and uploads the XML report. **Failure reporting:** each pytest matrix invocation writes `pytest-results.xml`; the final failure-only step runs @@ -335,7 +329,7 @@ The `Fuzz` workflow runs deeper discovery every Monday and by manual dispatch: | --- | --- | --- | --- | | 2026-05-31 | Initial stack integration | Added configuration, CI, documentation, and Hypothesis tests. | Continue staged strictness rollout. | | 2026-05-31 | Bandit | Reviewed low-severity findings and confirmed no medium- or high-severity findings. | Re-review when command trust boundaries change. | -| 2026-05-31 | Hypothesis code generation | Added generated native-name escaping, stable synthetic-import ordering, and semantic-IR-to-Pyi parse-back invariants; fixed quoted `Name(...)` emission. | Keep storing minimized failures. | +| 2026-05-31 | Hypothesis code generation | Added generated native-name escaping, stable synthetic-import ordering, and semantic-IR-to-Pyi parse-back invariants; fixed quoted `SourceName(...)` emission. | Keep storing minimized failures. | | 2026-06-01 | Ruff formatting rollout | Formatted the historical Python tree and changed CI to `ruff format --check .`. | Continue complexity-policy ratchets. | | 2026-06-01 | Radon and Ruff complexity policy | Added `tools/check_radon_policy.py`, made the staged Radon policy blocking in CI, and lowered Ruff McCabe from `50` to `45`. | Continue hotspot refactors and later threshold ratchets toward `20`. | | 2026-06-02 | Historical mutation-derived tests | Added direct Fortran parser contracts and fixed the directory namespace encoding bug. | Keep the tests as normal regression coverage. | diff --git a/docs/developer/repository-structure.md b/docs/developer/repository-structure.md index f46eb4156..570905e10 100644 --- a/docs/developer/repository-structure.md +++ b/docs/developer/repository-structure.md @@ -21,18 +21,14 @@ artifacts used by tests. Navigate by ownership boundary first, then by file. | `x2py/probes/` | Compiler-derived target facts and target type mapping reports. | | `x2py/runtime/` | Python runtime objects used by generated extension modules. | | `x2py/types/` | Cross-layer mappings from resolved semantic types to Python ecosystem types. | -| `x2py/fortran_parser/` | Fortran parser frontend and Fortran parse report helpers. | -| `x2py/semantics/` | Semantic IR, source-to-IR conversion, `.pyi` parsing, readiness, and codegen lowering. | -| `x2py/compiling/` | Native compile objects, compiler command orchestration, runtime support installation, and linking. | -| `x2py/stdlib/` | Native runtime support copied into generated wrapper builds. | +| `x2py/parsers/` | Public namespace for language and semantic-contract frontends and parser models. | +| `x2py/semantics/` | Semantic IR, source-to-IR conversion, `.pyi` parsing, policy completion, and readiness. | +| `x2py/wrapper_codegen/` | Typed wrapper plans, direct native bridge/binding lowering, and source and semantic `.pyi` printers. | +| `x2py/compiling/` | Native compile objects, compiler command orchestration, native support installation, and linking. | +| `x2py/binding_support/` | Bundled header-only native support copied into generated wrapper builds. | | `x2py/naming/` | Unified public-name and generated-symbol policy. | | `x2py/utilities/` | Small shared Python utilities. | - - The major source packages have local README files under `x2py/` for developers reading directly in the source tree. Those README files should link back to the maintained source-navigation docs instead of old top-level docs. @@ -40,19 +36,23 @@ back to the maintained source-navigation docs instead of old top-level docs. Only `x2py/__init__.py`, `x2py/__main__.py`, and `x2py/cli.py` live directly at the package root. Public library symbols are deliberately flattened through `x2py/__init__.py`; internal modules are imported through their owning package. -The one public submodule namespace is `x2py.contracts`, because semantic `.pyi` -files use direct `from x2py.contracts import ...` declarations as part of their -contract syntax. +The deliberate public submodule namespaces are `x2py.contracts`, whose import +path is part of semantic `.pyi` syntax, and `x2py.parsers`, which groups the +language-specific frontends. Stable convenience functions remain flattened +through `x2py/__init__.py`. ## Tests | Path | Purpose | | --- | --- | -| `tests/parser/` | Parser, preprocessing, CLI, and parser fixture tests. | -| `tests/semantics/` | Semantic IR, readiness, type mapping, and lowering tests. | -| `tests/pyi/` | Semantic `.pyi` parser and fixture tests. | +| `tests/parsing/` | Parser and parser fixture tests grouped by source language. | +| `tests/pipeline/` | Preprocessing and semantic `.pyi` build-orchestration tests. | +| `tests/semantics/` | Semantic conversion, completed policy, and readiness tests. | +| `tests/wrapper_codegen/` | Typed planning, direct bridge/binding generation, and source-printer tests. | +| `tests/utilities/` | Shared Python utility tests. | | `tests/wrapper/fortran/` | Runtime wrapper tests that compile, import, call, and check failure paths. | -| `tests/tools/` | Tooling tests, including documentation example and structure checks. | +| `tests/docs/` | Documentation example and structure checks. | +| `tests/tools/` | Repository tooling tests. | ## Package Map @@ -68,17 +69,18 @@ X2PY_C_DOCS_END --> | `x2py/probes/` | Compiler-derived target facts plus mapping reports | `fortran_types.py`, `report.py` | target probe and type mapping report tests | | `x2py/runtime/` | Python runtime objects consumed by generated extensions | `handles.py` | runtime handle and wrapper runtime tests | | `x2py/types/` | Semantic-to-Python ecosystem type mappings | `numpy.py` | `tests/types/test_numpy.py` | -| `x2py/fortran_parser/` | Fortran lexer, recursive parser, models, type resolver, and parser CLI helpers | `parser.py`, `lexer.py`, `models.py`, `type_resolver.py`, `cli.py` | `tests/parser/`, `tests/parser/fortran/`, `docs/developer/fortran-parser-reference.md` | -| `x2py/compiling/` | Native compile objects, compiler command orchestration, shared-library linking, and runtime support installation | `basic.py`, `compilers.py`, `python_wrapper.py`, `runtime_support.py` | `tests/wrapper/fortran/build_from_source/test_build_modes.py`, `tests/wrapper/fortran/build_from_source/test_runtime_abi.py` | -| `x2py/stdlib/` | Native runtime support files copied into generated wrapper builds | `x2py_runtime/` | wrapper runtime tests | -| `x2py/utilities/` | Small shared Python utilities | `metaclasses.py`, `strings.py` | tests that exercise callers | +| `x2py/parsers/` | Public namespace for language and semantic `.pyi` frontends | child parser packages | `tests/parsing/`, parser references, semantic `.pyi` reference | +| `x2py/parsers/fortran/` | Fortran lexer, recursive parser, models, type resolver, and parser CLI helpers | `parser.py`, `lexer.py`, `models.py`, `type_resolver.py`, `cli.py` | `tests/parser/`, `tests/parser/fortran/`, `docs/developer/fortran-parser-reference.md` | +| `x2py/compiling/` | Native compile objects, compiler command execution, shared-library linking, and native support installation; wrapper build orchestration lives in `x2py/pipeline/build.py` | `objects.py`, `compilers.py`, `compiler_profiles.py`, `native_support.py` | `tests/wrapper/fortran/build_from_source/test_build_modes.py`, `tests/wrapper/fortran/build_from_source/test_runtime_abi.py` | +| `x2py/binding_support/` | Bundled header-only native binding support copied into generated wrapper builds | support header | wrapper build tests | +| `x2py/utilities/` | Small shared Python utilities | `strings.py`, `visitor.py` | tests that exercise callers | @@ -96,38 +98,34 @@ update this table, the package README files, and the mechanical checks in | `x2py/pipeline/preprocessing.py` | Compiler-backed source preprocessing and dependency facts. | | `x2py/probes/fortran_types.py` | Fortran kind and storage probing. | | `x2py/semantics/ownership.py` | Central ownership, transfer, destruction, and generated-action policy. | -| `x2py/fortran_parser/parser.py` | Fortran parser project model and diagnostics. | -| `x2py/fortran_parser/cli.py` | Fortran parser report formatting. | +| `x2py/parsers/fortran/parser.py` | Fortran parser project model and diagnostics. | +| `x2py/parsers/fortran/cli.py` | Fortran parser report formatting. | | `x2py/semantics/metadata.py` | Cross-stage semantic metadata keys that survive parser, policy, printer, and lowering boundaries. | | `x2py/semantics/models.py` | Semantic IR dataclasses and core model metadata. | | `x2py/semantics/fortran2ir.py` | Fortran parser facts to semantic modules. | -| `x2py/pyi_parser/parser.py` | Minimal `.pyi` text/file parsing to Python AST. | +| `x2py/parsers/pyi/parser.py` | Minimal `.pyi` text/file parsing to Python AST. | | `x2py/pipeline/pyi.py` | Semantic `.pyi` text/file/path-set conversion and external-type reconciliation. | | `x2py/semantics/pyi2ir.py` | Semantic `.pyi` AST conversion and validation. | -| `x2py/semantics/policy_completion.py` | Post-IR semantic policy completion before readiness and lowering. | +| `x2py/semantics/policy_completion.py` | Post-IR semantic policy completion before readiness and wrapper planning. | | `x2py/semantics/readiness.py` | Support blockers and readiness reporting. | -| `x2py/semantics/ir2ast.py` | Semantic IR to codegen AST lowering. | -| `x2py/codegen/binding_pipeline.py` | Ordered bridge and binding generation. | -| `x2py/codegen/printers/fcode.py` | Fortran source printing. | -| `x2py/codegen/printers/pyi_printer.py` | Semantic `.pyi` printing. | -| `x2py/compiling/basic.py` | Native compile object model. | +| `x2py/wrapper_codegen/plan.py` | Typed, policy-complete wrapper plan records. | +| `x2py/wrapper_codegen/planner.py` | Semantic policy to wrapper-plan conversion. | +| `x2py/wrapper_codegen/generator.py` | Ordered direct bridge, binding, header, and source generation. | +| `x2py/wrapper_codegen/fortran/bridge.py` | Direct Fortran bridge lowering from typed plans. | +| `x2py/wrapper_codegen/c/binding.py` | Direct Python-extension binding lowering from typed plans. | +| `x2py/wrapper_codegen/printers/source_printers.py` | Native binding, header, and Fortran source printing. | +| `x2py/wrapper_codegen/printers/pyi_printer.py` | Semantic `.pyi` printing. | +| `x2py/compiling/objects.py` | Native compile object model. | | `x2py/compiling/compilers.py` | Compiler command execution and tool lookup. | -| `x2py/compiling/python_wrapper.py` | Generated wrapper compilation and shared-library linking. | -| `x2py/compiling/runtime_support.py` | Runtime support installation for generated wrappers. | +| `x2py/compiling/native_support.py` | Native binding support installation for generated wrappers. | | `x2py/naming/policy.py` | Public wrapper names and generated target-language symbols. | -| `x2py/stdlib/` | Runtime support payload copied into generated builds. | +| `x2py/binding_support/` | Native binding support payload copied into generated builds. | ## Layer-To-Layer Route @@ -139,15 +137,16 @@ For source-driven Fortran wrappers, read in this order: x2py/cli.py -> x2py/pipeline/build.py -> x2py/pipeline/preprocessing.py - -> x2py/fortran_parser/parser.py + -> x2py/parsers/fortran/parser.py -> x2py/probes/fortran_types.py -> x2py/semantics/fortran2ir.py -> x2py/semantics/policy_completion.py -> x2py/semantics/readiness.py - -> x2py/semantics/ir2ast.py - -> x2py/codegen/bridges/fortran_to_c.py - -> x2py/codegen/bindings/c_to_python.py - -> x2py/compiling/python_wrapper.py + -> x2py/wrapper_codegen/planner.py + -> x2py/wrapper_codegen/generator.py + -> x2py/wrapper_codegen/fortran/bridge.py + -> x2py/wrapper_codegen/c/binding.py + -> x2py/compiling/compilers.py -> tests/wrapper/fortran/ ``` X2PY_C_DOCS_END --> @@ -155,12 +154,13 @@ X2PY_C_DOCS_END --> For semantic `.pyi` builds, the parser branch is replaced by: ```text -x2py/pyi_parser/parser.py +x2py/parsers/pyi/parser.py -> x2py/pipeline/pyi.py -> x2py/semantics/pyi2ir.py -> x2py/semantics/policy_completion.py -> x2py/semantics/readiness.py - -> x2py/semantics/ir2ast.py + -> x2py/wrapper_codegen/planner.py + -> x2py/wrapper_codegen/generator.py ``` The hardest source packages also have local README files: - `x2py/README.md` -- `x2py/fortran_parser/README.md` +- `x2py/parsers/README.md` +- `x2py/parsers/fortran/README.md` +- `x2py/parsers/pyi/README.md` - `x2py/semantics/README.md` -- `x2py/codegen/README.md` - `x2py/compiling/README.md` Keep these files short. They should tell developers where to enter the code, diff --git a/docs/developer/testing-strategy.md b/docs/developer/testing-strategy.md index cb39f9e3b..55e35e43d 100644 --- a/docs/developer/testing-strategy.md +++ b/docs/developer/testing-strategy.md @@ -22,9 +22,8 @@ Stage and unit tests mirror the implementation pipeline: workflow; - semantic conversion, completed policy, and readiness have separate owners under `tests/semantics/`; -- semantic-to-codegen conversion lives under `tests/lowering/`; -- bridge, binding, and printer generation live under matching - `tests/codegen/` subjects; +- wrapper planning, bridge/binding generation, and source/`.pyi` printing live + under `tests/wrapper_codegen/`; - runtime handles, naming, and type mapping live under `tests/runtime/`, `tests/naming/`, and `tests/types/`; - documentation, repository-tool, and architecture checks live under @@ -37,14 +36,18 @@ crosses subjects. For example: python3 -m pytest -q tests/parsing/fortran python3 -m pytest -q tests/semantics/conversion/fortran python3 -m pytest -q tests/semantics/policy -python3 -m pytest -q tests/lowering -python3 -m pytest -q tests/codegen/bridges +python3 -m pytest -q tests/wrapper_codegen ``` CLI behavior has its own `tests/cli/` owner. Property and regression tests live with their owning stage and retain their markers, so a stage-directory command does not omit them. +The legacy lowering AST and `x2py.codegen` package are removed. Their bridge, +binding, and printer implementation tests are not carried through cutover. Any +still-relevant contract is expressed against completed semantic policy, +`WrapperPlan`/`WrapperCodeGenerator`, or public compiled wrapper behavior. + ## Wrapper runtime features Compiled wrapper tests stay organized by user-visible feature, not by internal @@ -58,14 +61,21 @@ modes execute one shared behavioral assertion body. Modified-contract behavior stays in the same feature subject but uses its intentionally different assertions. -Run all Fortran wrapper subjects except real-library runtime work with: +During the wrapper-plan migration, run all Fortran wrapper subjects except the +deferred full BLAS/LAPACK corpus with: ```bash -python3 -m pytest -q tests/wrapper/fortran --ignore=tests/wrapper/fortran/real_libraries +python3 -m pytest -q tests/wrapper/fortran \ + --ignore=tests/wrapper/fortran/real_libraries/test_real_blas_lapack.py ``` -Do not run LAPACK runtime tests locally unless the task explicitly requests -them. BLAS-only evidence may be selected separately when relevant. +The full LAPACK wrapper test remains CI-only by default. The full BLAS test may +be run locally when its library-scale evidence is needed. One dedicated GitHub +Actions job runs the exact full BLAS and LAPACK nodes together on Python 3.12; +the ordinary Python-version matrix excludes their complete test file. Add the +`ignore-real-library-wrappers` label to a pull request to skip only this +expensive dedicated job. General native-bundle tests remain active in the +ordinary matrix. ## Fixtures and generated expectations diff --git a/docs/maintainer/design/index.md b/docs/maintainer/design/index.md index d55fedc28..7065054df 100644 --- a/docs/maintainer/design/index.md +++ b/docs/maintainer/design/index.md @@ -9,7 +9,7 @@ status: planned-documentation # Design Documents Design documents record long-term technical decisions for maintainers. They do -not by themselves establish runtime support. +not by themselves establish native binding support. ## Pages diff --git a/docs/maintainer/design/wrapper-design-notes.md b/docs/maintainer/design/wrapper-design-notes.md index 43464679c..2953d5ce3 100644 --- a/docs/maintainer/design/wrapper-design-notes.md +++ b/docs/maintainer/design/wrapper-design-notes.md @@ -40,7 +40,7 @@ X2PY_C_DOCS_END --> | Gap | Current risk | Proposed direction | | --- | --- | --- | | Procedure pointers and dummy procedures | A broad `Procedure` type loses enough signature and lifetime information that wrappers cannot safely call or receive callbacks. | Resolve abstract interface signatures into a first-class semantic callable form. Preserve procedure pointer, optional, pass-through, and callback lifetime facts; block wrapper generation until call direction and ownership policy are explicit. | -| Pointer and allocatable ownership | Allocatable and pointer arrays use explicit descriptor handles. Module and derived-field handles borrow their native owner; owned allocatable results retain persistent wrapper-owned descriptor storage. An unallocated or unassociated descriptor remains a present handle whose `to_numpy()` result is `None`. Allocatable `intent(inout)` descriptor arguments accept handles and project the same caller handle, while ordinary arrays and ordinary array results keep NumPy data-buffer semantics. Pointer targets remain non-owning, and unsupported result or reassociation shapes fail readiness. | Complete descriptor kind, handle kind, owner retention, extraction, mutation, release, and operation permissions in post-IR policy before lowering. Route module, field, argument, and result generation through named handle-policy dispatch. Keep borrowed views, detached read-only copies, and descriptor handles distinct; never fall back from an incomplete handle policy to an ndarray copy contract. Block allocatable scalar derived-type replacement until ownership and destruction policy is defined. | +| Pointer and allocatable ownership | Allocatable and pointer arrays use explicit descriptor handles. Module and derived-field handles borrow their native owner; owned allocatable results retain persistent wrapper-owned descriptor storage. An unallocated or unassociated descriptor remains a present handle whose `to_numpy()` result is `None`; otherwise `to_numpy()` returns a current live view and callers use `.copy()` explicitly for independent storage. Allocatable `intent(inout)` descriptor arguments accept handles and project the same caller handle, while ordinary arrays and ordinary array results keep NumPy data-buffer semantics. Rank-zero derived module allocatables/pointers use nullable live member proxies. Wrapper-owned allocatable and pointer derived results use persistent typed holders. Module allocatable dummies use reversible `move_alloc` holder transactions; module pointer dummies use typed association transactions and exact restoration. C transports opaque holder addresses and typed operation pointers, never descriptors. A pointer holder owns its association container, not an unknown native target. | Complete descriptor kind, handle/storage kind, actual declaration, dummy form, owner retention, live extraction or member mechanism, mutation/writeback, release, transaction cleanup, and operation permissions in post-IR policy before lowering. Route module, field, argument, and result generation through named policy dispatch. Keep contiguous-view, descriptor-view, scoped-reference, module-transaction, and typed-holder mechanisms distinct; never fall back from incomplete policy to a copy, fabricated address, or compiler-private descriptor. | | Assumed-rank, assumed-type, and optional descriptor-heavy arguments | Descriptors such as `dimension(..)` and `type(*)` can accept many native shapes that Python cannot infer safely. | Represent descriptor category, rank constraints, element type availability, optional presence, and contiguity. Generate wrappers only for explicit accepted rank/dtype policies or through backend shims that validate descriptors. | | Generic interfaces and operators | Named generics, defined operators, named operators, and defined assignment now preserve explicit concrete-target links. Python cannot intercept `=`, arbitrary named operators, or infer safe in-place mutation. Static extension-type inheritance is represented in Python, and scalar polymorphic input dispatch reuses the same generated overload selection path. | Use Python data-model slots for intrinsic operators, `operator_name`/`r_operator_name` methods for named operators, and mutating `assign` methods for defined assignment. Keep exact dtype/rank/extension-class dispatch and reject indistinguishable signatures during generation. | | Coarrays, teams, events, and directive-driven device/offload behavior | These introduce parallel runtime or device-memory semantics outside normal host wrappers. | Treat as out of the initial wrapper scope. Preserve diagnostics where detected and require a separate runtime design before claiming support. | ## Settled Scope @@ -76,12 +76,21 @@ callbacks, and the metadata needed for readiness decisions. X2PY_C_DOCS_END --> Verbose wrapper builds should print the exact compiler command lines they run, not only the source or target being compiled. The printed command should be shell-quoted so users can copy it to reproduce object compilation, generated -wrapper compilation, runtime support compilation, and final shared-library -linking. +wrapper compilation (including its header-only native binding support), and +final shared-library linking. | CLI request | `x2py/cli.py` | source paths and stage flags | selected stage or wrapper build options | `tests/cli/` | | Build orchestration | `x2py/pipeline/build.py` | ordered Fortran sources or `.pyi` contracts plus explicit native artifacts | `WrapperBuildResult`, `NativeBuildPlan`, and generated artifact plan | wrapper build-mode tests | | Preprocessing | `x2py/pipeline/preprocessing.py` | source path, compiler config | preprocessed source and dependency facts | preprocessing tests | -| Parser project model | `x2py/fortran_parser/parser.py` | preprocessed Fortran source | parser project with modules, procedures, types, visibility | Fortran parser fixture tests | +| Parser project model | `x2py/parsers/fortran/parser.py` | preprocessed Fortran source | parser project with modules, procedures, types, visibility | Fortran parser fixture tests | | Target probes | `x2py/probes/fortran_types.py` | semantic type requirements and compiler flags | resolved kind/storage facts | Fortran type probe tests | | Semantic IR | `x2py/semantics/fortran2ir.py` | parser project and target facts | `SemanticModule` objects | semantic Fortran tests | -| Semantic policy completion | `x2py/semantics/policy_completion.py`, `x2py/semantics/ownership.py` | full semantic modules with signatures and `.pyi` overrides | semantic modules annotated with completed ownership, transfer, and destruction decisions | ownership-policy, readiness, and lowering tests | +| Semantic policy completion | `x2py/semantics/policy_completion.py`, `x2py/semantics/ownership.py` | full semantic modules with signatures and `.pyi` overrides | semantic modules annotated with every ownership, transfer, destruction, mutability, storage, accessor, and projection decision needed by wrapper generation | ownership-policy and readiness tests | | Readiness | `x2py/semantics/readiness.py` | prepared semantic modules | blockers and support status | readiness tests and fixtures | -| Codegen lowering | `x2py/semantics/ir2ast.py` | policy-completed semantic modules | codegen AST consuming completed policy decisions | `tests/lowering/test_semantic_ir.py`, wrapper tests | -| Printing | `x2py/codegen/printers/` | generated ASTs | wrapper source files | generated build artifacts and wrapper tests | -| Compile and link | `x2py/compiling/` | user objects, wrapper sources, runtime support | shared library | wrapper runtime tests | +| Wrapper planning | `x2py/wrapper_codegen/planner.py`, `x2py/wrapper_codegen/plan.py` | policy-completed semantic modules | typed wrapper plans consuming completed decisions without re-inferring policy | `tests/wrapper_codegen/`, wrapper tests | +| Direct bridge and binding lowering | `x2py/wrapper_codegen/fortran/bridge.py`, `x2py/wrapper_codegen/c/binding.py`, `x2py/wrapper_codegen/generator.py` | validated typed wrapper plans | Fortran, C, and header syntax nodes | `tests/wrapper_codegen/`, wrapper tests | +| Wrapper and semantic-contract printing | `x2py/wrapper_codegen/printers/` | wrapper syntax nodes or semantic IR | wrapper source files or semantic `.pyi` text | printer, generated-contract, and wrapper artifact tests | +| Compile and link | `x2py/compiling/`, `x2py/pipeline/build.py` | explicit native objects, then generated bridge objects, then runtime/binding objects and ordered link inputs | shared library | wrapper runtime and build-mode tests | ## Concept Ownership Rules @@ -79,12 +78,12 @@ cross-cutting infrastructure. | --- | --- | --- | --- | | Parser facts | parser packages | Source syntax, native declaration structure, source locations, and parser diagnostics | Wrapper policy, Python API projection, generated names, and compile/link decisions | | Readiness and ownership policy | `x2py/semantics/readiness.py`, `x2py/semantics/policy_completion.py`, and `x2py/semantics/ownership.py` | Semantic policy completion, support blockers, and policy choices for ownership, lifetime, output projection, replacement, and ABI safety | Raw parser syntax, backend-specific statement trees, and hidden lowering-time policy decisions | -| Core codegen AST | `x2py/codegen/models/` and `x2py/semantics/ir2ast.py` outputs | The implementation plan after a semantic contract is accepted: generated functions, variables as storage locations, statements, expressions, control flow, temporaries, scopes, and imports/includes | Source-contract authority, `.pyi` persistence, and readiness-only facts | -| Printers and compilation | `x2py/codegen/printers/`, `x2py/compiling/`, and wrapper orchestration | Text emission, generated artifact layout, compiler commands, native objects, libraries, include directories, and link inputs | Semantic support decisions and generated-AST rewriting policy | +| Typed wrapper plan | `x2py/wrapper_codegen/plan.py` and `x2py/wrapper_codegen/planner.py` | A validated, backend-neutral implementation plan projected from completed semantic decisions | Source-contract authority, policy inference, and target-language statement details | +| Printers and compilation | `x2py/wrapper_codegen/printers/`, `x2py/compiling/`, and wrapper orchestration | Text emission, generated artifact layout, compiler commands, native objects, libraries, include directories, and link inputs | Semantic support decisions and plan rewriting policy | @@ -97,21 +96,21 @@ Use these rules when adding a new notion: than a source fact: for example borrowed versus copied data, visible versus hidden native outputs, replacement rules, destructor ownership, or unsupported ABI combinations. If the decision depends on full signature context, complete - it in `policy_completion.py` before readiness or `ir2ast.py`. + it in `policy_completion.py` before readiness or wrapper planning. - Put it in compiling or wrapping when it describes build inputs or build execution: sources, objects, libraries, library directories, include - directories, compiler flags, link items, runtime support files, and generated + directories, compiler flags, link items, binding support files, and generated artifact paths. Merge or move concepts only when their invariants match: @@ -123,7 +122,7 @@ Merge or move concepts only when their invariants match: - Move a codegen concept into semantics only when it can be represented without a generated body, temporary, scope, include, or target-language expression and the fact is needed for `.pyi`, readiness, or source-free replay. -- Move a semantic concept into codegen only when it does not change the public +- Move a semantic concept into a wrapper plan only when it does not change the public contract, native contract, readiness, or `.pyi` representation and exists only to print or compile wrapper code. @@ -143,8 +142,8 @@ Examples: - Python keyword avoidance for a public name, such as a native `def` routine, belongs to naming policy. The chosen public spelling is stored where the contract needs it, while target-specific helper symbols stay generated. -- Codegen `Scope`, `FunctionDef`, body statements, temporaries, decorators, - includes, and backend datatypes stay out of `x2py/semantics/models.py`. +- Wrapper syntax nodes, body statements, temporaries, includes, and backend + datatypes stay out of `x2py/semantics/models.py`. | --- | --- | --- | | CLI and output routing | `x2py/cli.py`, parser CLI helpers | `docs/developer/source-map.md`, `docs/developer/feature-to-code-map.md` | | Source loading and preprocessing | `x2py/pipeline/preprocessing.py` | `docs/developer/source-map.md`, parser references | -| Editable semantic contracts | `x2py/pyi_parser/parser.py`, `x2py/pipeline/pyi.py`, `x2py/semantics/pyi2ir.py`, `x2py/codegen/printers/pyi_printer.py` | `docs/user/reference/semantic-pyi-format.md` | +| Editable semantic contracts | `x2py/parsers/pyi/parser.py`, `x2py/pipeline/pyi.py`, `x2py/semantics/pyi2ir.py`, `x2py/wrapper_codegen/printers/pyi_printer.py` | `docs/user/reference/semantic-pyi-format.md` | | Readiness | `x2py/semantics/readiness.py` | `docs/user/reference/diagnostic-codes.md` | -| Wrapper policy and lowering | `x2py/semantics/policy_completion.py`, `x2py/semantics/ownership.py`, `x2py/semantics/ir2ast.py` | `docs/user/guide/fortran-wrapper.md`, ownership docs | -| Native build | `x2py/compiling/python_wrapper.py`, `x2py/compiling/runtime_support.py` | compiling package README and build-system docs | +| Wrapper policy and lowering | `x2py/semantics/policy_completion.py`, `x2py/semantics/ownership.py`, `x2py/wrapper_codegen/planner.py`, `x2py/wrapper_codegen/generator.py` | `docs/user/guide/fortran-wrapper.md`, ownership docs | +| Native build | `x2py/pipeline/build.py`, `x2py/compiling/compilers.py`, `x2py/compiling/native_support.py` | compiling package README and build-system docs | ## Semantic `.pyi` Wrapper Pipeline @@ -177,21 +176,22 @@ the Python API. ```text .pyi contract - -> x2py/pyi_parser/parser.py + -> x2py/parsers/pyi/parser.py -> x2py/pipeline/pyi.py -> x2py/semantics/pyi2ir.py -> x2py/semantics/native_contract.py -> x2py/semantics/policy_completion.py -> x2py/semantics/readiness.py - -> x2py/semantics/ir2ast.py - -> bridge, binding, compile, and link pipeline + -> x2py/wrapper_codegen/planner.py + -> x2py/wrapper_codegen/generator.py + -> compile and link pipeline ``` The `.pyi` path must preserve native ABI facts in the semantic contract. Missing native build inputs or contradictory contract facts fail before bridge emission or native compilation. Ownership, transfer, and destruction policy is completed -from the full `.pyi` signature before lowering; `ir2ast.py` consumes that -completed policy and must not invent a different one. +from the full `.pyi` signature before planning; the wrapper planner and backend +generators consume that completed policy and must not invent a different one. ## Shared Semantic Policy Boundary @@ -210,7 +210,7 @@ X2PY_C_DOCS_END --> ```text C parser -> x2py/semantics/c2ir.py Fortran parser -> x2py/semantics/fortran2ir.py -.pyi parser -> x2py/pyi_parser/parser.py -> x2py/pipeline/pyi.py -> x2py/semantics/pyi2ir.py +.pyi parser -> x2py/parsers/pyi/parser.py -> x2py/pipeline/pyi.py -> x2py/semantics/pyi2ir.py -> SemanticModule objects -> x2py/semantics/policy_completion.py -> readiness and lowering @@ -220,8 +220,8 @@ X2PY_C_DOCS_END --> | Ownership, lifetime, ABI, or projection policy is unsafe | `x2py/semantics/ownership.py`, readiness, or `ir2ast` | | Generated code cannot represent a supported contract | bridge or binding generator with focused tests | | Compiler/linker invocation is wrong | `x2py/compiling/` or `x2py/pipeline/build.py` | -| Python runtime behavior is wrong | generated binding, runtime support, or ownership policy | +| Python binding behavior is wrong | generated binding, native support, or ownership policy | diff --git a/docs/maintainer/internal-architecture/wrapper-generation-pipeline.md b/docs/maintainer/internal-architecture/wrapper-generation-pipeline.md index 6292c4a64..8ded379ba 100644 --- a/docs/maintainer/internal-architecture/wrapper-generation-pipeline.md +++ b/docs/maintainer/internal-architecture/wrapper-generation-pipeline.md @@ -2,19 +2,218 @@ title: Wrapper Generation Pipeline audience: maintainers prerequisites: semantic passes, code generation design -related: runtime-layer.md, ownership-tracking.md -status: planned-documentation +related: runtime-layer.md, ownership-tracking.md, ../roadmap/wrapper-plan-migration-checklist.md +status: maintained --- # Wrapper Generation Pipeline - +This page describes the canonical wrapper-plan generation route. It covers the +completed scalar, string, array, native-handle, derived-type, class, callback, +module-state, generic, and build surfaces. -## TODO +## Architectural Boundary -- TODO: Document the full generated bridge and binding pipeline with file - ownership. -- TODO: Link feature support to tests that exercise generated runtime behavior. +All semantic policy must be complete before wrapper planning begins. Post-IR +policy completion owns +object kind, ownership, transfer, destruction, mutability, writeback, +nullability, output projection, release responsibility, storage mode, getter +behavior, native setter assignment, and Python setter exposure. + +Planning projects those completed decisions into one editable `ModulePlan`. +Validation checks that the projections agree. Binding and bridge generation +then dispatch only from completed selectors into small named lowering methods; +they do not reconstruct policy from datatype, `intent`, shape, alias flags, or +local memory checks. + +Native-source `intent` may be consumed while importing a source declaration to +propose default Python argument/result positions. It is not retained in the +semantic `.pyi` or post-IR ownership context. The editable Python signature, +`Returns[...]` projection, and ordered native-call mapping are authoritative. +Bridge entry dummies omit `intent`, leaving their storage permissive; that +contract controls wrapper copy-in, copy-back, and returned values, while the +compiled native procedure's own interface controls native access. + +Within that contract, an explicit native-call list is exhaustive for native +dummy positions. Matching named `Returns[...]` items attach result positions to +visible `Arg(i)` entries automatically; direct function results remain the first +ordinary Python return item, while hidden native output dummies require explicit +`Return(...)` entries. Descriptor reassociation follows the same rule: +`Pointer(Arg(i))` without a projected return uses a call-local adapter and +discards reassociation, while a matching projected return requires storage that +can preserve association writeback. + +Native transport overrides also live on that mapping. Primitive `Arg(i)` is a +value handoff and `Addr(Arg(i))` selects call-local address handoff. Wrapped +derived `Arg(i)` is a typed reference handoff and `Value(Arg(i))` selects exact +typed value handoff. `Returns[...]` never selects either ABI; it only assigns a +Python result position and writeback expectation. A derived `Value(...)` slot +does not expose aggregate layout at the C boundary: C still supplies an opaque +address, the bridge reconstructs the exact native type, and the Fortran compiler +applies the explicit interface's `VALUE` semantics at the typed call. + +Standalone legacy externals use a completed declaration mode. Procedures whose +ABI is valid with an implicit interface, including classic BLAS/LAPACK +subroutines and scalar functions, lower to `external` declarations; optional, +descriptor-rich, polymorphic, or array-result procedures retain explicit +interfaces. The bridge dispatches this completed mode and does not reclassify +the signature. + +The public direct-generation boundary is: + +```python +complete_semantic_policies(module) +plan = WrapperPlanner().build(module) +artifacts = WrapperCodeGenerator().generate(plan) +``` + +`WrapperCodeGenerator.generate()` freezes the plan, runs the shared validator, +runs both backend preflight checks, lowers recursively to C and Fortran syntax +nodes, and asks the source printers to render those nodes. Build integration +compiles the rendered sources; it does not own datatype transfer policy. +Wrapper C/Fortran source printers and the semantic `.pyi` printer share +`x2py/wrapper_codegen/printers/`; no compatibility printer remains under the +legacy codegen package. + +Wrapper builds have no legacy route or fallback. An unsupported completed plan +fails with its exact owner path before either backend emits source. + +## Stable Tree and Datatype-Varying Records + +The shared plan has stable module, namespace, and function orchestration: + +```text +ModulePlan + binding: BindingModulePlan + bridge: BridgeModulePlan + namespaces: NamespacePlan ... + functions: FunctionPlan ... + binding: BindingFunctionPlan + bridge: BridgeFunctionPlan + arguments: ArgumentTransferPlan ... + binding: BindingArgumentPlan + bridge: BridgeArgumentPlan + native_call_slot: NativeCallSlotPlan + results: ResultPlan ... + binding: BindingResultPlan + bridge: BridgeResultPlan + native_call_slot: NativeCallSlotPlan | None + native_call_slots: NativeCallSlotPlan ... + lifecycle actions: LifecycleActionPlan ... + variables: ModuleVariablePlan ... +``` + +Most datatype-specific work belongs to `ArgumentTransferPlan` and +`ResultPlan`. Each is one transfer with explicit binding and bridge views. +`ModuleVariablePlan` is the other intentionally datatype-sensitive surface, +because getter, setter, and native assignment behavior depend on the stored +value. + +`FunctionPlan`, `NamespacePlan`, and `ModulePlan` remain orchestration records. +They own export names, call order, result order, runtime/GIL envelopes, and +aggregation, but not datatype policy. + +Python-facing documentation is also a plan projection. The shared docstring +builder consumes completed namespace, module-variable, class, overload, +argument, result, and lifecycle records and stores the rendered text on the +owning plan nodes. C method-table emission and generated Python class assembly +only attach that text; neither backend infers signatures, ownership, mutation, +nullability, or exception behavior while rendering source. + +`NativeCallSlotPlan` and `LifecycleActionPlan` are subordinate transfer +details. Native slots stay indexed on `FunctionPlan` because native ABI order +can interleave argument slots, result slots, literals, and helpers. Lifecycle +actions stay indexed there because copy-out, cleanup, and release order may +span several arguments and results or differ on failure. Argument and hidden +result slots are the same mutable records referenced from both their transfer +owner and the function-wide index; they are not duplicated policy. + +## One Repeatable Transfer Algorithm + +Use this sequence for scalars, strings, arrays, and future datatype families: + +1. Post-IR policy completion classifies the value with `ObjectKind` and + completes ownership, transfer, storage, nullability, mutability, projection, + barrier actions, data action, and any justified copy reason. +2. Wrapper policy records the backend-neutral transfer and the ordered native + slot. It must report a blocker instead of leaving a semantic choice for a + backend. +3. `WrapperPlanner` mechanically projects one `ArgumentTransferPlan` or + `ResultPlan`, adds symbolic handoff roles, and shares the corresponding + `NativeCallSlotPlan` reference. +4. The shared validator checks graph consistency and common invariants, then + dispatches by the completed `object_kind` to scalar, string, or ordinary- + array validation. +5. Backend preflight dispatches by the same completed kind and action selectors + and rejects combinations it cannot lower. +6. The binding lowers Python extraction or result construction. The bridge + lowers ABI declarations, representation conversion, the ordered native + call, and native result production. Both communicate through planned + symbolic roles. +7. Function-level orchestration applies status handling and ordered lifecycle + actions, aggregates Python results, and returns. Printers and build + integration remain generic. + +When adding a datatype, first extend semantic policy and its transfer record, +then add one named validator and one named lowering method per affected +backend. Do not add a parallel plan hierarchy or datatype branches to module, +namespace, or function traversal. Add a new typed action only when the existing +selectors cannot express a genuine semantic choice. + +## Selector Vocabulary + +The action axes are deliberately orthogonal: + +| Selector | Question answered | Examples | +| --- | --- | --- | +| `ObjectKind` | What kind of object follows this route? | `SCALAR`, `STRING`, `NUMPY_ARRAY` | +| `source_kind` | Where is a result produced? | `direct_return`, `hidden_output` | +| `PythonBarrierAction` | How does the binding cross the Python boundary? | `SCALAR_VALUE`, `STRING_VALUE`, `ARRAY_STORAGE` | +| `NativeBarrierAction` | What native ABI transport is used? | `PASS_VALUE`, `PASS_CALL_LOCAL_ADDRESS`, `PASS_ARRAY_BUFFER` | +| `CodegenAction` | What ownership or transfer operation occurs? | `DIRECT_VALUE`, `CALL_LOCAL_INPUT`, `COPY_IN_OUT`, `COPY_OUT` | +| `BridgeDataAction` | What happens to the representation in the bridge? | `DIRECT_TRANSFER`, `ASSOCIATE_VIEW`, `COPY_REPRESENTATION` | +| `WritebackPhase` | When does a lifecycle operation run? | native mutation, copy-out, cleanup, release | + +Hiddenness is not a transfer operation. A hidden scalar result therefore uses +`source_kind="hidden_output"` with `CodegenAction.DIRECT_VALUE`; hidden strings +and ordinary arrays use the same source kind with `CodegenAction.COPY_OUT`. + +`NativeBarrierAction.PASS_ARRAY_BUFFER` identifies the Phase 6 ordinary-array +data-buffer ABI. Its handoff plan carries data, rank, extents, strides, and +itemsize. `PASS_NATIVE_DESCRIPTOR` is reserved for Phase 7 persistent native +descriptors and handles. Neither backend may substitute one for the other. +Array handoff shapes are completed bridge extents; native source bounds are +temporary import facts and must not appear in semantic `.pyi` or become extent +dependencies. A source dimension such as `0:LDB-1` therefore completes to +extent `LDB`, while the native procedure keeps control of its own indexing +bounds. When `PASS_NATIVE_DESCRIPTOR` also carries +optional absence, the completed optional mode lowers a valid call-local +placeholder descriptor plus a separate presence role. This keeps the bridge +entry ABI valid while presence dispatch omits the native dummy. + +`DatatypeFamily` remains useful after object-kind dispatch for primitive +element spelling and conversion, such as integer versus real scalar types or +the element type of an ordinary array. It must not be used to rediscover +whether the transfer itself is a scalar, string, or array. + +## Maintainer Inspection and Acceptance + +Inspect the real records directly with normal Python prints. The primary path +is `complete_semantic_policies()` -> `WrapperPlanner.build()` -> +`WrapperCodeGenerator.generate()`. Generated artifacts from real passing +`tests/wrapper` cases are the behavioral oracle; plan unit tests cover action +and graph invariants. Production source and semantic-`.pyi` builds both use +this one path; unsupported completed policy is an error before lowering, not a +request to retry a legacy generator. + +A wrapper-generation change is acceptable when: + +- semantic decisions are complete before planning; +- datatype variation is confined to transfer, result, lifecycle, or + module-variable records and their named handlers; +- scalar, string, and array routes use the same planning and validation + sequence; +- binding and bridge consume the same shared roles and native-slot records; +- no backend infers policy or silently falls back to another action; +- focused plan tests, relevant wrapper runtime tests, documentation checks, and + static analysis pass. diff --git a/docs/maintainer/roadmap/documentation-content-checklist.md b/docs/maintainer/roadmap/documentation-content-checklist.md index 31341154a..658df8079 100644 --- a/docs/maintainer/roadmap/documentation-content-checklist.md +++ b/docs/maintainer/roadmap/documentation-content-checklist.md @@ -97,7 +97,7 @@ X2PY_C_DOCS_END --> verification paths, fixture regeneration, documentation examples, wrapper runtime tests, and static-analysis gates. - [ ] `docs/developer/build-system.md`: document native compile model, - generated Makefiles, build manifests, runtime support files, compiler probes, + generated Makefiles, build manifests, native support files, compiler probes, and future packaging boundaries. - [ ] `docs/developer/coding-standards.md`: document Python style, documentation front matter, no-compatibility-layer rule, parser/codegen @@ -127,7 +127,7 @@ X2PY_C_DOCS_END --> preprocessing boundaries, model facts, diagnostics, and fixture strategy. - [ ] `docs/maintainer/design/semantic-analysis.md`: document source-to-IR lowering, `.pyi`-to-IR loading, policy completion, readiness blockers, and invariants. -- [ ] `docs/maintainer/design/runtime-model.md`: document runtime support files, generated +- [ ] `docs/maintainer/design/runtime-model.md`: document native support files, generated wrappers, native state, callbacks, threading, and finalization. - [ ] `docs/maintainer/design/error-propagation-model.md`: document diagnostic categories, Python exception projection, native failure handling, cleanup, and user-facing @@ -136,7 +136,8 @@ X2PY_C_DOCS_END --> policy-completion ownership decisions, transfer actions, mutability, setter exposure, and release responsibility. - [ ] `docs/maintainer/internal-architecture/ast-design.md`: document parser AST, semantic - IR, codegen AST, what each layer may store, and what must not leak across + IR, completed wrapper plans, generated source syntax, what each layer may + store, and what must not leak across layers. - [ ] `docs/maintainer/internal-architecture/semantic-passes.md`: document semantic pass ordering, completed policy decisions, readiness checks, and handoff to @@ -148,7 +149,7 @@ X2PY_C_DOCS_END --> - [ ] `docs/maintainer/internal-architecture/type-system.md`: document scalar kinds, arrays, characters, derived types, pointers, allocatables, callbacks, and unsupported storage forms. -- [ ] `docs/maintainer/internal-architecture/runtime-layer.md`: document runtime support +- [ ] `docs/maintainer/internal-architecture/runtime-layer.md`: document native support installation, extension initialization, callbacks, cleanup, and shared native state. - [ ] `docs/maintainer/internal-architecture/ownership-tracking.md`: document ownership @@ -165,7 +166,7 @@ X2PY_C_DOCS_END --> names. dummies update the supplied wrapper object. Runtime evidence lives in `tests/wrapper/fortran/edit_pyi_contracts/test_native_order_contracts.py`. - [x] Ownership, transfer, and destruction policy is completed after full - signatures are known and before readiness or `ir2ast.py`. The shared post-IR + signatures are known and before readiness or wrapper planning. The shared post-IR entrypoint is `complete_semantic_policies(...)` in `x2py/semantics/policy_completion.py`; direct ownership subpasses stay behind that entrypoint. Readiness and lowering consume completed policy metadata instead of recomputing policy from raw datatypes. Evidence: `tests/semantics/policy/`, - `tests/lowering/test_semantic_ir.py`, + `tests/wrapper_codegen/`, `tests/semantics/readiness/`, and `x2py/semantics/README.md`. - [x] `.pyi` parsing and `.pyi` semantic conversion are separate stages: - `x2py/pyi_parser/parser.py` parses text/files to Python AST, and + `x2py/parsers/pyi/parser.py` parses text/files to Python AST, and `x2py/semantics/pyi2ir.py` converts that AST into `SemanticModule` objects before semantic policy completion runs. Evidence: `tests/parsing/pyi/test_python_ast_contracts.py::test_pyi_parser_returns_python_ast_only`, @@ -519,36 +520,13 @@ X2PY_C_DOCS_END --> `tests/wrapper/CHECKLIST_COVERAGE.md`. bridge and binding code are local emitted-code, ABI, documentation, or object-model mechanics rather than semantic policy selection. Evidence: `x2py/semantics/ownership.py`, - `x2py/codegen/bridges/fortran_to_c.py`, - `x2py/codegen/bindings/c_to_python.py`, + `x2py/wrapper_codegen/fortran/bridge.py`, + `x2py/wrapper_codegen/c/binding.py`, `tests/semantics/policy/`, + `tests/wrapper_codegen/`, `tests/wrapper/fortran/derived_types/test_derived_layout.py`, and `tests/wrapper/fortran/edit_pyi_contracts/test_ownership_contracts.py`. X2PY_C_DOCS_END --> diff --git a/docs/maintainer/roadmap/stage-boundary-enforcement-checklist.md b/docs/maintainer/roadmap/stage-boundary-enforcement-checklist.md deleted file mode 100644 index af35b9b7b..000000000 --- a/docs/maintainer/roadmap/stage-boundary-enforcement-checklist.md +++ /dev/null @@ -1,449 +0,0 @@ ---- -title: Stage Boundary Enforcement Checklist -audience: maintainers -prerequisites: pipeline map, semantic IR, ownership policy -related: ../internal-architecture/pipeline-map.md, ../../user/reference/semantic-ir.md, semantic-pyi-wrapper-checklist.md, index.md -status: active-roadmap ---- - -# Stage Boundary Enforcement Checklist - -This checklist tracks the architectural work required to make every wrapper -pipeline transition explicit, one-way, and mechanically enforced. It is not -enough for the main build path to call functions in the intended order. Each -stage must accept only the preceding stage's completed representation, reject -invalid state without repairing it, and leave all earlier-stage decisions -unchanged. - -The target pipeline is: - -```text -parsed source model - -> semantic IR draft - -> policy-completed semantic IR - -> readiness-approved semantic IR - -> codegen IR - -> bridge IR - -> binding IR - -> emitted sources - -> compilation and link result -``` - -Stages are dependent but monotonic. A later stage may translate or validate the -preceding output. It must not reach backward into parser facts, secretly rerun -an earlier stage, infer a missing semantic decision, or mutate completed policy. - -## Incremental Implementation Protocol - -This checklist is the implementation prompt and the authoritative progress -record. Do not maintain a second copy of its requirements in a separate prompt. - -The normal request for continuing this work is: - -> Implement the next coherent group of unchecked items from the stage boundary -> enforcement checklist. - -A request may also name a phase or a smaller set of items when a particular -boundary should be handled first. - -For every implementation turn: - -1. Read `AGENTS.md` and this entire checklist before selecting work. -2. Inspect the live files and existing changes; do not assume an unchecked item - is still unimplemented or that a checked item is still correct. -3. Select the smallest dependency-closed group that produces a verifiable - architectural improvement. Do not select unrelated boxes merely because - they are nearby. -4. State which checklist items are in scope before editing. -5. Update the relevant maintained architecture or public contract docs before - executable code when behavior or ownership changes. -6. Implement the selected group completely across code, diagnostics, tests, - documentation, and call sites. Do not add compatibility paths for an old - stage API. -7. Run focused verification for the selected boundary, followed by every - repository-required static check for the files changed. -8. Mark an item complete only when its full acceptance criterion has direct - evidence. Leave partially implemented items unchecked and add a short note - identifying the remaining work instead of treating partial progress as - completion. -9. Under the completed item or phase, record the owning files, tests, and exact - verification evidence needed by the next maintainer to audit the claim. -10. Stop at a coherent stage boundary. Report the next dependency-ready group, - but do not begin it unless it was part of the stated scope. - -Each implementation summary must include the changed-stage breakdown required -by `AGENTS.md`, the checklist items completed, tests added or updated, focused -and static verification results, remaining unchecked dependencies, and any -unrelated pre-existing failure. The checklist is complete only when every -non-negotiable requirement and phase acceptance item has direct evidence. - -## Non-Negotiable Contract - -- [ ] Semantic draft, policy-completed IR, readiness-approved IR, and codegen IR - are mechanically distinguishable representations. A mutable metadata boolean - is not the stage boundary. -- [ ] Policy completion returns a new completed representation and does not - mutate the caller's semantic draft. -- [ ] Every semantic decision required by lowering, bridge generation, binding - generation, printing, or build integration is complete before `ir2ast.py`. -- [ ] Completed policy cannot be replaced through a public mutable metadata - dictionary. -- [ ] Readiness accepts only policy-completed IR, does not invoke policy - completion, and does not mutate its input. -- [ ] Successful readiness returns a distinct readiness-approved representation. -- [ ] Lowering accepts only readiness-approved IR and cannot invoke readiness or - policy completion. -- [ ] Bridge and binding generators consume codegen decisions and cannot inspect - parser models or semantic drafts. -- [ ] Printers emit the representation they receive and cannot invoke semantic - policy completion or readiness. -- [ ] High-level orchestration invokes every stage explicitly in the documented - order, with no hidden fallback or compatibility path. - -## Boundary Immutability Model - -The implementation rule is: - -```text -private mutable stage builder - -> validate the stage result - -> deeply freeze the stage output - -> hand the immutable output to the next stage -``` - -Immutability applies to values crossing stage boundaries, not to every local -object used while constructing them. Parsers, converters, lowerers, generators, -symbol tables, scopes, caches, and compiler-command builders may use mutation -inside their owning stage. Those mutable builders must remain private and must -never be accepted as another stage's input. - -- [ ] Parsed-project output is immutable before semantic conversion receives it. -- [ ] Semantic-draft output is immutable before policy completion receives it. -- [ ] Policy-completed output is deeply immutable before readiness receives it. -- [ ] Readiness-approved output is deeply immutable before lowering receives it. -- [ ] Codegen IR is frozen after lowering and before bridge generation. -- [ ] Bridge IR is frozen after bridge generation and before binding generation. -- [ ] Binding IR is frozen after binding generation and before printing. -- [ ] Native build plans and build results are immutable records; execution - state remains private to the compiler/build runner. -- [ ] Generated source payloads cross their boundary as immutable text and - immutable artifact descriptions. -- [ ] Runtime handles are explicitly outside the pipeline-freezing rule because - allocation, association, owner state, and `close()` state are intentionally - mutable at execution time. -- [ ] A frozen dataclass containing a mutable list, set, dictionary, metadata - dictionary, or mutable nested semantic object does not satisfy this contract. -- [ ] Boundary collections use tuples, frozensets, deeply copied read-only - mappings, or equivalently immutable structures. -- [ ] Mutable backing mappings are not retained or exposed after constructing a - read-only view. -- [ ] Completed policy is represented through typed read-only fields or a typed - immutable policy bundle rather than replaceable string-key metadata entries. -- [ ] Mutation of a draft, builder, or source metadata object after a transition - cannot change the frozen output already handed to the next stage. -- [ ] Stage types cannot be forged by setting a marker in a metadata dictionary; - construction flows through the owning transition function. -- [ ] Tests treat Python's normal supported API as the enforcement boundary; - deliberate `object.__setattr__`-style interpreter bypasses are not supported - mutation paths. - -## Phase 0 — Live Boundary Audit - -- [ ] Inventory every production caller of parsing, source-to-IR conversion, - policy completion, readiness, lowering, bridge generation, binding - generation, printing, and compilation. -- [ ] Record the input and output representation of every stage entrypoint. -- [ ] Inventory every mutation of semantic modules, declarations, semantic - types, metadata, policy decisions, export lists, and readiness blockers. -- [ ] Inventory every call to `complete_semantic_policies()` and classify it as - the explicit pipeline transition or a hidden stage invocation to remove. -- [ ] Inventory all `_raise_for_*` and blocker construction in `ir2ast.py` and - classify each check as semantic validity, policy validity, readiness/backend - support, or a genuine lowering invariant. -- [ ] Audit every bridge/binding branch that reads datatype, `intent`, rank, - shape, `is_alias`, memory handling, dotted-variable form, nullability, - storage, ownership, or policy fields. -- [ ] For each audited bridge/binding branch, record whether it is completed - policy dispatch, permitted backend-local mechanics, or policy inference that - must move upstream. -- [ ] Confirm the audit includes source-driven builds, semantic `.pyi` builds, - readiness-only inspection, `.pyi` emission, manifest replay, and Makefile - generation. - -## Phase 1 — Documented Stage-State Model - -- [ ] Update the maintained pipeline map with the exact stage-state types and - allowed transitions. -- [ ] Document which stage owns parser validity, semantic contract validity, - policy validity, readiness/backend support, lowering invariants, and - compilation failures. -- [ ] Document the distinction between semantic policy and backend-local emitted - helper storage. -- [ ] Document which transformations are allowed to be lossy, such as entry - export pruning, and which source representation remains available for - diagnostics. -- [ ] Document whether completed and readiness-approved representations are - immutable snapshots, wrappers around immutable data, or another design with - equivalent mechanical guarantees. -- [ ] Update source-navigation docs so contributors enter each concern through - its owning stage rather than a downstream generator. -- [ ] For every stage package, document what it owns, what it consumes, what it - returns, which internal builders may mutate, what it must never infer, and - which downstream package may import its public output. -- [ ] Document the runtime-handle exception so pipeline immutability is not - incorrectly applied to intentionally stateful native runtime objects. - -## Phase 2 — Explicit Stage Representations - -- [ ] Introduce a semantic draft representation produced by source-to-IR and - semantic `.pyi` conversion. -- [ ] Introduce a policy-completed representation that cannot be confused with - the draft type. -- [ ] Introduce a readiness-approved/wrappable representation that cannot be - constructed by merely setting metadata. -- [ ] Preserve a separate codegen representation for lowering output. -- [ ] Introduce distinct bridge and binding handoff representations when those - stages currently share a mutable model that either stage can rewrite. -- [ ] Give each stage representation a narrow public construction path owned by - its transition function; keep mutable builders private to the stage package. -- [ ] Replace mutable policy metadata storage with an immutable completed policy - bundle or an equivalently sealed representation. -- [ ] Make nested collections and policy maps immutable enough that downstream - code cannot replace a decision indirectly. -- [ ] Ensure mutation of the original draft after completion cannot affect the - completed representation. -- [ ] Remove `POLICY_COMPLETION_PREPARED_METADATA` if the new type makes it - redundant, or limit it to serialized diagnostic provenance rather than using - it as authority. -- [ ] Do not add aliases, coercions, adapters, or compatibility wrappers that - allow an old mutable `SemanticModule` to bypass the new boundary. - -The intended transition API should be equivalent to this shape, although exact -names may follow the settled package design: - -```python -def build_semantic_draft(parsed: ParsedProject) -> SemanticDraft: ... - -def complete_policies(draft: SemanticDraft) -> PolicyCompletedIR: ... - -def validate_readiness(completed: PolicyCompletedIR) -> WrappableSemanticIR: ... - -def lower_to_codegen(wrappable: WrappableSemanticIR) -> CodegenIR: ... - -def generate_bridge(codegen: CodegenIR) -> BridgeIR: ... - -def generate_binding(bridge: BridgeIR) -> BindingIR: ... -``` - -- [ ] Each transition rejects every other stage representation rather than - coercing it or running a missing earlier transition. -- [ ] Each transition returns a new object and leaves its input observably - unchanged. -- [ ] Stage results expose immutable diagnostic/source provenance without - retaining a mutable reference to the preceding builder. -- [ ] Equality or stable fingerprints allow tests to prove that validation and - downstream generation did not mutate an earlier result. - -## Phase 3 — Post-IR Policy Completion - -Policy completion must own all semantic choices needed downstream, including -choices currently reconstructed from raw facts during lowering or codegen. - -- [ ] Complete object kind for every argument, result, field, module variable, - callback boundary, class instance, and hidden native value. -- [ ] Complete ownership, transfer, destruction, borrowed state, target owner, - and release responsibility. -- [ ] Complete mutability, native mutation, writeback, assignment mode, and - replacement behavior. -- [ ] Complete nullability and distinguish omitted arguments from explicit - present-but-null descriptor values. -- [ ] Complete output projection and hidden/identity/copy result behavior. -- [ ] Complete contract-value and boundary storage modes (`stack`, `heap`, or - `alias`). -- [ ] Complete Python barrier action and native barrier action. -- [ ] Complete getter behavior, native setter assignment, and Python setter - exposure for every field and module variable. -- [ ] Complete descriptor/data-buffer array interoperability and all required ABI - selector facts. -- [ ] Complete pass-by-value, pass-by-address, and call-local address behavior. -- [ ] Complete native array handle kind, ownership, operations, extraction, - descriptor interop, and build requirements. -- [ ] Complete callback argument/result ownership and barrier decisions. -- [ ] Complete entry export reachability before ownership decisions for the - retained declarations. -- [ ] Reject attempts to run policy completion on an already-completed - representation. -- [ ] Ensure policy completion produces path-aware blockers for missing or - contradictory facts rather than inserting a downstream fallback. - -## Phase 4 — Readiness As A Mandatory Gate - -- [ ] Make the prepared readiness API accept only policy-completed IR. -- [ ] Remove automatic policy completion from readiness APIs. -- [ ] Make readiness validation non-mutating. -- [ ] Move semantic support and backend-capability blockers out of `ir2ast.py` - into readiness. -- [ ] Keep policy contradictions in policy completion rather than readiness. -- [ ] Keep only genuine target-AST representability invariants in lowering. -- [ ] Return a distinct readiness-approved representation only when there are no - blockers. -- [ ] Preserve a structured blocker report when readiness fails. -- [ ] Require both source and semantic `.pyi` wrapper builds to pass readiness - before lowering. -- [ ] Require manifest replay and Makefile generation to use the same readiness - gate as direct builds. - -## Phase 5 — Mechanical IR-To-AST Lowering - -- [ ] Change `semantic_ir_to_codegen_ast()` to accept only readiness-approved - semantic IR. -- [ ] Add one recursive entry validator that verifies every required completed - policy category before visiting any declaration. -- [ ] Replace optional `.get(...)` access for required getter, setter, result, - class, array-handle, storage, barrier, and interoperability decisions with - required typed access. -- [ ] Move pass-by-value selection from parser-origin inspection into completed - policy. -- [ ] Move descriptor versus data-buffer interoperability selection into - completed policy. -- [ ] Audit array category, source shape, target/addressability, optionality, - projection, and layout conversion so only mechanical representation lowering - remains. -- [ ] Remove readiness decisions and unsupported-contract policy from lowering. -- [ ] Prove lowering does not mutate the readiness-approved input. -- [ ] Preserve backend-local creation of scopes, names, temporaries, imports, - statements, and expressions. - -## Phase 6 — Bridge And Binding Dispatch - -- [ ] Ensure bridge and binding modules cannot import or call - `OwnershipPolicyResolver`, `default_ownership_policy`, policy completion, or - readiness. -- [ ] Route semantic behavior through explicit dispatchers keyed by completed - codegen actions or typed completed policy selectors. -- [ ] Reject missing dispatcher combinations without choosing a default. -- [ ] Remove policy inference based on datatype, `intent`, rank/shape alone, - `is_alias`, local memory handling, dotted variables, parser origin, or missing - policy. -- [ ] Limit branches inside selected implementation methods to emitted-code - mechanics. -- [ ] Represent backend-local helper temporary storage separately from semantic - `OwnershipDecision` so local implementation details cannot be mistaken for - contract policy. -- [ ] Prove bridge generation does not mutate or replace completed policy carried - by codegen IR. -- [ ] Prove binding generation does not mutate or replace completed policy - carried by bridge/codegen IR. -- [ ] Keep low-level printers free of ownership-aware behavior selection. - -## Phase 7 — Printers And Build Orchestration - -- [ ] Remove hidden policy completion from `emit_module_stubs()` and other - printer entrypoints. -- [ ] Make `.pyi` emission orchestration explicitly select the required semantic - stage before invoking the printer. -- [ ] Make direct source builds visibly call parse, semantic conversion, policy - completion, readiness, lowering, bridge/binding generation, and compilation - in order. -- [ ] Make semantic `.pyi` builds visibly call `.pyi` parsing/conversion, native - contract validation, policy completion, readiness, lowering, bridge/binding - generation, and compilation in order. -- [ ] Ensure inspection-only CLI stages stop at their declared representation - and do not mutate it for a later report in the same command. -- [ ] Ensure native array build requirements consume completed/readiness-approved - policy rather than reconstructing descriptor needs. -- [ ] Remove legacy entrypoints or permissive stage coercions instead of keeping - compatibility paths. - -## Phase 8 — Mechanical Architecture Tests - -- [ ] Raw semantic draft is rejected by prepared-readiness entrypoints. -- [ ] Raw semantic draft is rejected by lowering. -- [ ] Policy-completed but readiness-unvalidated IR is rejected by lowering. -- [ ] Removing each required policy category is detected by the recursive stage - validator before lowering starts. -- [ ] Missing getter, setter, return, class-instance, class-self, array-handle, - storage, barrier, and interoperability decisions cannot become `None`. -- [ ] Policy completion does not mutate its input draft. -- [ ] Mutating the draft after completion cannot affect completed IR. -- [ ] Readiness does not mutate completed IR or any completed decision. -- [ ] Lowering does not mutate readiness-approved IR or any completed decision. -- [ ] Bridge generation does not mutate or recompute policy. -- [ ] Binding generation does not mutate or recompute policy. -- [ ] Policy completion rejects already-completed input. -- [ ] Wrapper build orchestration invokes stages in the exact documented order. -- [ ] Every stage transition returns a different object from its input and the - input retains the same stable fingerprint after the call. -- [ ] Parsed-project, semantic-draft, completed, wrappable, codegen, bridge, - binding, build-plan, and build-result boundary collections reject ordinary - supported mutation operations. -- [ ] Frozen stage results contain no reachable mutable list, set, dictionary, - mutable metadata dictionary, or mutable semantic child object. -- [ ] Mutable parser, semantic, lowering, bridge, binding, and build builders are - not exported from their package's public stage API. -- [ ] Structural AST tests reject prohibited resolver/completion imports from - lowering, bridge, binding, and printer packages. -- [ ] Structural AST tests reject prohibited raw-fact policy inference in bridge - and binding code. -- [ ] Structural tests allow narrowly identified backend-local helper planning - without allowing semantic policy construction in generators. -- [ ] Structural dependency tests enforce the permitted package direction: - parsing cannot import semantics; semantics cannot import lowering/codegen; - readiness cannot import lowering; lowering cannot import bridge/binding; - bridge/binding cannot import parsers or semantic policy resolvers; printers - cannot invoke earlier stage transitions. -- [ ] Structural dependency tests allow the high-level pipeline orchestrator to - import and compose stage entrypoints without making orchestration policy - authority. -- [ ] Runtime-handle tests continue to prove intentional allocation, - association, ownership, and close-state mutation despite immutable pipeline - artifacts. -- [ ] Missing dispatcher combinations fail explicitly. -- [ ] Export pruning remains before readiness/lowering, and omitted declarations - never reach codegen. -- [ ] Semantic `.pyi` round-tripping preserves the editable contract. -- [ ] Source, generated-contract, and modified-contract runtime behavior remains - unchanged unless an explicitly documented blocker moves earlier. - -## Phase 9 — Focused Evidence - -- [ ] `tests/semantics/policy/` covers immutable completed - policy, complete recursive decision sets, and strict dispatcher behavior. -- [ ] `tests/semantics/readiness/` covers the completed-to- - validated transition and non-mutating readiness. -- [ ] `tests/lowering/test_semantic_ir.py` covers validated-only lowering and absence - of policy/readiness inference. -- [ ] `tests/codegen/printers/` covers printer-only emission without - hidden policy completion. -- [ ] `tests/parsing/pyi/` covers semantic draft construction and - contract round-tripping. -- [ ] `tests/wrapper/fortran/edit_pyi_contracts/` proves edited policy remains - authoritative through runtime behavior. -- [ ] `tests/architecture/test_dependency_boundaries.py` enforces - dependency and inference restrictions mechanically. -- [ ] Source and semantic `.pyi` build-mode tests prove the mandatory stage - sequence. -- [ ] Focused wrapper tests cover scalar, string, array, descriptor, derived - type, callback, module-variable, and optional-argument boundaries. -- [ ] LAPACK remains excluded from local verification unless separately - authorized. - -## Phase 10 — Verification And Completion Record - -- [ ] Focused semantic, `.pyi`, codegen-structure, build-mode, and wrapper tests - pass. -- [ ] `python3 -m ruff check .` passes. -- [ ] `python3 -m ruff format --check .` passes. -- [ ] Bandit passes with the repository configuration. -- [ ] Vulture passes. -- [ ] The blocking Radon policy passes with an explicit base fallback when local - CI SHA variables are unavailable. -- [ ] Full Radon complexity and maintainability reports are run and recorded as - advisory output. -- [ ] `git diff --check` passes. -- [ ] No compatibility shim, fallback path, or legacy stage entrypoint remains. -- [ ] The final implementation report lists every blocker moved, every stage - representation introduced, every downstream inference removed, and every - remaining limitation. -- [ ] This checklist is moved from active work to completed evidence only after - every requirement above has direct test or structural evidence. diff --git a/docs/maintainer/roadmap/test-suite-organization-checklist.md b/docs/maintainer/roadmap/test-suite-organization-checklist.md index 5a3771ca5..5cda84406 100644 --- a/docs/maintainer/roadmap/test-suite-organization-checklist.md +++ b/docs/maintainer/roadmap/test-suite-organization-checklist.md @@ -25,7 +25,7 @@ has been recorded here. - [x] Exclude LAPACK runtime execution from local verification. - [x] Preserve unrelated dirty-worktree changes. Initial audit: worktree clean. - [x] No executable product behavior changed. The only product-module edit is a - path-only documentation-string update in `x2py/c_parser/parser.py`. + path-only documentation-string update in `x2py/parsers/c/parser.py`. ## Baseline collection evidence @@ -98,7 +98,7 @@ all original cases are accounted for in its destination modules. | `tests/semantics/test_ownership_policy.py` | completed decisions in `tests/semantics/policy/`; generator dispatch cases in `tests/codegen/bridges/` and `tests/codegen/bindings/` | | `tests/semantics/test_c_semantic_readiness.py`, `test_semantic_wrap_readiness.py`, `test_wrap_readiness_fixture_suite.py` | `tests/semantics/readiness/`, with the oversized module split by readiness boundary | | `tests/semantics/test_ir2ast.py`, `test_visitor_protocol.py` | `tests/lowering/` | -| `tests/semantics/test_pyi_printer*.py` | `tests/codegen/printers/`, with the oversized printer module split by emitted concept | +| `tests/semantics/test_pyi_printer*.py` | `tests/wrapper_codegen/printers/`, with the oversized printer module split by emitted concept | | `tests/test_runtime_handles.py` | split under `tests/runtime/handles/` | | `tests/test_naming_policy.py` | `tests/naming/test_policy.py` | | `tests/tools/test_documentation_examples.py`, `test_documentation_structure.py` | `tests/docs/` | @@ -122,7 +122,7 @@ trees are not part of this map and must not move. | `tests/semantics/test_ownership_policy.py` | `tests/semantics/policy/test_accessor_and_storage_policy.py`, `test_native_array_ownership.py`, `test_policy_defaults_and_validation.py`; `tests/lowering/test_array_interop_policy.py`; `tests/codegen/bridges/test_bridge_handle_policy_dispatch.py`; `tests/codegen/bindings/test_binding_handle_policy_dispatch.py` | | `tests/semantics/test_fortran2ir.py` | `tests/semantics/conversion/fortran/test_compile_time_values.py`, `test_fortran_conversion_procedures_and_interfaces.py`, `test_modules_and_imports.py`, `test_types_and_storage.py` | | `tests/parser/test_preprocessing_cli.py` | `tests/pipeline/preprocessing/test_cli.py`, `test_configuration_and_adapters.py`, `test_dependencies_and_includes.py`, `test_execution.py` | -| `tests/semantics/test_pyi_printer.py` | `tests/codegen/printers/test_calls_and_policy_metadata.py`, `test_classes_and_methods.py`, `test_pyi_printer_imports_and_packages.py`, `test_types_and_declarations.py` | +| `tests/semantics/test_pyi_printer.py` | `tests/wrapper_codegen/printers/test_calls_and_policy_metadata.py`, `test_classes_and_methods.py`, `test_pyi_printer_imports_and_packages.py`, `test_types_and_declarations.py` | | `tests/test_runtime_handles.py` | `tests/runtime/handles/test_array_actual_abi.py`, `test_descriptor_abi.py`, `test_factories_and_lifecycle.py`, `test_handle_protocols.py` | | `tests/semantics/test_c2ir.py` | `tests/semantics/conversion/c/test_functions_and_callbacks.py`, `test_projects_and_diagnostics.py`, `test_records_and_enums.py`, `test_types_and_constants.py` | | `tests/semantics/test_semantic_wrap_readiness.py` | `tests/cli/test_wrap_readiness.py`; `tests/semantics/readiness/test_policy_blockers.py`, `test_pyi_readiness.py`, `test_reports.py` | @@ -259,8 +259,8 @@ row therefore accounts for two of the 34 normalized-ID differences. | `tests/semantics/test_c_semantic_readiness.py` | `tests/semantics/readiness/test_c_readiness.py` | | `tests/semantics/test_fortran2ir.py` | `tests/semantics/conversion/fortran/` | | `tests/semantics/test_ir2ast.py` | `tests/lowering/test_semantic_ir.py` | -| `tests/semantics/test_pyi_printer.py` | `tests/codegen/printers/` | -| `tests/semantics/test_pyi_printer_modern_example.py` | `tests/codegen/printers/test_modern_example.py` | +| `tests/semantics/test_pyi_printer.py` | `tests/wrapper_codegen/printers/` | +| `tests/semantics/test_pyi_printer_modern_example.py` | `tests/wrapper_codegen/printers/test_modern_example.py` | | `tests/semantics/test_semantic_wrap_readiness.py` | `tests/semantics/readiness/` | | `tests/tools/test_documentation_examples.py` | `tests/docs/test_examples.py` | | `tests/tools/test_documentation_structure.py` | `tests/docs/test_structure.py` | @@ -304,3 +304,13 @@ only fixture/generator trees remain under `tests/parser/`, `tests/pyi/`, and - [x] Final tree, mapping, split rationale, helper moves, collection evidence, focused results, wrapper verification, static checks, retained locations, and any product failures are recorded here and in the handoff. + +## Post-Cutover Supersession + +The earlier inventory and execution record above documents the historical test +move and intentionally retains its old paths as evidence. After the canonical +wrapper-plan cutover, `tests/codegen/`, `tests/lowering/`, `x2py.codegen`, and +the adjacent legacy lowering/build entrypoints are removed. Still-required +contracts belong to completed semantic-policy tests, `tests/wrapper_codegen/` +plan and direct-generation tests, or compiled public behavior under +`tests/wrapper/fortran/`. diff --git a/docs/maintainer/roadmap/wrapper-plan-migration-checklist.md b/docs/maintainer/roadmap/wrapper-plan-migration-checklist.md new file mode 100644 index 000000000..0d8288e47 --- /dev/null +++ b/docs/maintainer/roadmap/wrapper-plan-migration-checklist.md @@ -0,0 +1,4836 @@ +--- +title: Wrapper Plan Migration Checklist +audience: maintainers +prerequisites: pipeline map, semantic IR, ownership policy +related: ../internal-architecture/pipeline-map.md, ../../user/reference/semantic-ir.md, semantic-pyi-wrapper-checklist.md, index.md +status: active-roadmap +--- + +# Wrapper Plan Migration Checklist + +This file is the canonical implementation contract for wrapper-plan migration. +It replaces the generic semantic-IR wrapper lowering route one eligible module +at a time. The migration changes representation and generation organization; it +does not intentionally change the established Python, native ABI, ownership, or +build behavior of a migrated lane. + +## Canonical Pipeline + +```text +Semantic IR + -> post-IR policy completion + -> WrapperPlanner.build(module) + -> editable ModulePlan + -> WrapperCodeGenerator.generate(plan) + -> freeze and validate the received plan + -> recursively synthesize C binding nodes + -> recursively synthesize Fortran bridge nodes + -> print backend nodes + -> RenderedGeneratedWrapperArtifacts + -> existing build/link orchestration +``` + +There is no public or wrapper-domain representation between `ModulePlan` and +backend syntax nodes. `CModule`, `CHeader`, `CFunction`, `FortranModule`, and +`FortranFunction` are direct printer inputs, not another wrapper planning +stage. + +The public generation boundary is deliberately small: + +```python +complete_semantic_policies(module) +plan = WrapperPlanner().build(module) +artifacts = WrapperCodeGenerator().generate(plan) +``` + +`WrapperCodeGenerator.generate` accepts `ModulePlan` only. It does not accept +semantic modules, build a plan itself, select an alternate lowering route, or +retry a prior route after direct generation begins. + +Semantic `.pyi` generation remains outside this route. A semantic `.pyi` +contract can supply the semantic module consumed by planning, but planning does +not change `.pyi` emission. + +## One Shared Plan, Explicit Backend Views + +`ModulePlan` is one shared semantic-and-ABI contract. It is not a C plan joined +to a Fortran plan and it does not contain backend nodes or source text. + +Every owner that crosses or coordinates the boundary has binding and bridge +child plans in the same editable tree: + +```text +ModulePlan + binding: BindingModulePlan + bridge: BridgeModulePlan + functions: FunctionPlan ... + binding: BindingFunctionPlan + bridge: BridgeFunctionPlan + arguments: ArgumentTransferPlan ... + binding: BindingArgumentPlan + bridge: BridgeArgumentPlan + native_call_slot: NativeCallSlotPlan + transformations: TransformationPlan ... + results: ResultPlan ... + binding: BindingResultPlan + bridge: BridgeResultPlan + native_call_slot: NativeCallSlotPlan | None + transformations: TransformationPlan ... + lifecycle: LifecycleActionPlan ... + binding: BindingLifecyclePlan | None + bridge: BridgeLifecyclePlan | None + native_call_slots: ordered references to argument/result slots plus + function-owned literal or helper slots +``` + +`ArgumentTransferPlan` remains the only argument-owner record; do not add a +generic duplicate `ArgumentPlan`. Its backend-facing child plans are +deliberately distinct and directly editable: + +- `binding: BindingArgumentPlan` describes the Python input, its C conversion + action, and + the C handoff value/role it produces; +- `bridge: BridgeArgumentPlan` describes the C ABI slot, value-versus-address + convention, + native action, and Fortran value that the bridge consumes; +- `native_call_slot` records the exact native-call position and source; +- an argument or hidden result's `native_call_slot` is the same mutable record + referenced from `FunctionPlan.native_call_slots`, not a copied record that a + maintainer must edit twice; +- result and lifecycle records identify later producers, consumers, ordering, + and responsibility through their own binding and bridge views; +- native slots and lifecycle actions are subordinate transfer details, not + parallel datatype-policy systems. They remain indexed on `FunctionPlan` + because native ABI order and success/failure lifecycle order may span more + than one argument or result. A function-owned literal, status helper, or + other ABI slot may also have no single argument/result owner. + +The action vocabulary keeps source placement, data transfer, and native ABI +transport orthogonal. `ResultPlan.source_kind` says whether a result comes from +a `direct_return` or `hidden_output`; `CodegenAction` says how the value moves +or is owned (`DIRECT_VALUE`, `COPY_OUT`, `WRAPPER_INSTANCE`, and so on). A +hidden scalar therefore remains `DIRECT_VALUE`, while hidden strings and +ordinary arrays are `COPY_OUT`; hidden descriptor-owned objects use their +completed ownership action. `HIDDEN_OUTPUT` is not a codegen action because +hiddenness is a source location, not a transfer operation. + +Likewise, `NativeBarrierAction.PASS_ARRAY_BUFFER` means the Phase 6 data-buffer +ABI whose handoff plan carries data, rank, extents, strides, and itemsize. +`NativeBarrierAction.PASS_NATIVE_DESCRIPTOR` is reserved for the persistent +native descriptors and handles introduced in Phase 7. Neither backend may use +one action as a fallback for the other. + +`NativeBarrierAction.PASS_RAW_ADDRESS` is the third, deliberately narrower +array transport: one caller-supplied opaque address plus separately completed +pointee rank, shape, element type, and orientation facts. It does not authorize +NumPy extraction, a packed array-buffer ABI, or a native descriptor. Scalar, +fixed-string, and array raw addresses reuse this action and +`ArgumentHandoffMode.OPAQUE_ADDRESS`; their object kind then selects the named +bridge association method. Do not add datatype-specific raw-address actions. + +The binding input and bridge input may have different representations: a C +binding commonly receives `PyObject *`, produces a C scalar or address, and +the bridge then consumes a value or pointer according to its ABI slot. The plan +therefore records the producer/consumer handoff contract explicitly; it does +not assume identical types or actions. This keeps both backend plans in one +coherent editable tree while preventing them from drifting into disconnected +top-level plans. `CBindingGenerator` reads only binding views plus shared +handoff/order facts needed to create C nodes; `FortranBridgeGenerator` reads +only bridge views plus shared native-call order needed to create Fortran nodes. +Neither backend output is an input to the other backend. + +An owner may have only one active backend side when completed policy places the +behavior entirely in one backend, but that ownership must be explicit in the +plan rather than inferred during lowering. + +### Transformation Layer Ownership + +Every representation transformation has one explicit +`TransformationPlan.layer`: `BINDING` or `BRIDGE`. The record also names its +phase (`COPY_IN`, `NATIVE_MUTATION`, `COPY_OUT`, or `CLEANUP`), typed action, +source representation, target representation, and reason. These records are +subordinate to the `ArgumentTransferPlan` or `ResultPlan` that owns the value; +they are not a parallel datatype-policy hierarchy. + +Use the binding layer for transformations involving Python objects or NumPy +semantics: dtype/layout conversion, Python encoding/decoding, reference and +identity handling, copy-back into caller objects, and Python-owned temporary +cleanup. Use the bridge layer for transformations wholly between the ABI and a +native-language representation: Fortran character representation, native +descriptor/result materialization, derived native layout, or native-only +allocation and copy. + +One logical conversion and its inverse/cleanup must stay at one layer. If a +workflow genuinely needs both layers, policy completion records two distinct +transformations separated by a named intermediate ABI representation. A +backend consumes only transformations assigned to it and fails validation if +asked to lower an action owned by the other backend. Method location, datatype, +`intent`, and available local storage never select the transformation layer. +For `COPY_F`, copy-in, conditional copy-out, and cleanup are all binding-owned; +the bridge has no `COPY_F` transformation and reuses its ordinary ORDER_F +association path. + +### Editable Signature And Native Intent Boundary + +The semantic `.pyi` signature is authoritative for the Python-facing call +shape. The source converter may use native `intent` to propose the initial +generated signature, but that proposal is not backend policy. A user may +reorder visible Python arguments, keep a native output dummy as caller-supplied +storage, project it into a Python result, or introduce hidden bridge storage. +After the semantic contract is constructed or edited, completed native-call +slots must account for every required native position exactly once; stored +source `intent` must not silently override that mapping by hiding, exposing, +reordering, allocating, or projecting a Python value. + +Bridge dummies and backend-local variables use the most permissive declaration +that is compatible with the selected ABI, normally no `intent` or an internal +`intent(inout)`-equivalent writable local. The called native procedure enforces +its actual `intent(in)`, `intent(out)`, or `intent(inout)` contract. Required +interoperability attributes such as `value`, optional presence fields, standard +descriptor attributes, and true bridge-output parameters remain explicit ABI +facts; they are not permission for a backend to reconstruct the user-facing +signature from native `intent`. + +The plan includes every fact required for mechanical lowering: + +- owner path and plan-node kind; +- typed Python, native, and result actions; +- semantic datatype, datatype family, and precision/type facts; +- Python/native handoffs and bridge ABI slots; +- native-call slots in their exact order; +- result and output projection; +- ownership, transfer, destruction, mutability, writeback, release + responsibility, storage mode, nullability, and lifecycle ordering; and +- every typed lowering choice required by a supported backend. + +It contains decisions and facts, never generator method names. In particular, +there are no handler-name fields, handler records, or plan-owned handler +registries. The planner uses the completed policy actions already represented +by `PythonBarrierAction`, `NativeBarrierAction`, and `CodegenAction`; a new +typed action is added only when none of those can identify a necessary +mechanical behavior. Free-form string actions are forbidden. + +## Ownership Boundary + +Post-IR policy completion decides object kind, ownership, transfer, +destruction, mutability, writeback, nullability, output projection, release +responsibility, contract-value storage (`stack`, `heap`, or `alias`), getter +behavior, native setter assignment, Python setter exposure, ABI order, and +lifecycle order before planning starts. + +`WrapperPlanner` only projects those completed decisions and datatype facts +into readable editable records. It may traverse owners, preserve declared +order, assign stable owner paths, and wire already-decided producers to +consumers. It must not derive or replace policy from a datatype, `intent`, +decorator spelling, `is_alias`, dotted owner shape, local memory observation, +or a missing field. + +No code after planning may infer, replace, or override semantic policy. Backend +contexts may allocate temporary names and create declarations, error paths, +reference-count operations, and local cleanup statements after selecting a +typed lowering case. Those are emitted-code mechanics, not plan policy. + +`WrapperPlanner.build(module)` returns an editable `ModulePlan`. Maintainers +may edit its ordinary fields to inspect an experiment before generation. A +permanent behavior change belongs in the semantic contract and completed policy +rather than in a backend exception. + +## Generator-Owned Freezing and Validation + +Every consumer freezes the exact object it receives: + +```text +editable ModulePlan -- WrapperCodeGenerator --> frozen ModulePlan +editable backend modules -- source printers --> frozen backend modules +editable generated artifacts -- build integration --> frozen artifacts +``` + +At the start of `WrapperCodeGenerator.generate(plan)`, the generator must: + +1. recursively freeze that exact plan object; +2. run the complete binding/bridge plan-consistency validation on the final + edited plan; +3. ask each backend to preflight only its own implementation capability; and +4. recursively lower the validated plan into backend nodes before the printers + consume those nodes. + +Later mutation of the received plan raises `FrozenStageRecordError`. Backend +nodes remain editable until their printer consumes them. Generated artifacts +remain editable until `_build_rendered_wrapper_extension(...)` consumes them. + +`WrapperPlanner` does not validate its output. It mechanically projects an +editable plan, which may temporarily be inconsistent while a maintainer edits +it. `WrapperCodeGenerator` owns the private structured validation methods and +is the only validation consumer. There is no standalone validator class or +public validation operation. + +`WrapperCodeGenerator._validate_plan()` is the single plan-consistency gate. +It validates the complete binding/bridge graph after the editable plan has +been frozen and before either backend preflight or visitor runs. The gate stays +small by composing `_plan_diagnostics()` from typed private diagnostics for +namespaces, functions, arguments, results, lifecycle actions, and module +variable getter/setter action families. A cross-view invariant belongs to the +diagnostic for the lowest plan node that contains both views; for example, a +module-variable diagnostic validates Python setter exposure against its native +assignment and bridge setter role. + +`CBindingGenerator.require_supported()` and +`FortranBridgeGenerator.require_supported()` are later backend-local +capability preflights. They may reject a completed action, primitive type, +descriptor kind, or ABI combination that their own backend cannot implement, +but they do not establish whether binding and bridge views agree. Backend +visitors and `_lower_*` methods mechanically consume the decisions owned by +their view. Their exhaustive unmatched-action errors remain defensive +protection for direct backend use; the public generation path must report a +cross-view inconsistency from `_validate_plan()` first. No generator infers +consistency by reading the other backend's plan view. + +Structural validation preserves these invariants: + +- module getter actions and roles agree, and Python setter exposure agrees with + native assignment, bridge setter roles, descriptor kinds, and constant state; +- binding producer and bridge consumer roles agree; +- bridge ABI coverage, positions, and owner roles are complete; +- native-call slot coverage, exact ordering, hidden literals, and hidden + results agree, including hidden-result native and codegen actions; +- direct and hidden result producer/consumer roles agree; +- writeback, cleanup, and release actions use available source roles in their + declared order, and advertised roles exactly match their plan producers; +- positions and symbolic roles are neither duplicate nor missing; and +- external and bind-target requirements are complete. + +## Direct Recursive Lowering + +`WrapperCodeGenerator` owns two private backend visitors: + +```python +c_module, c_header = CBindingGenerator().visit(plan) +fortran_module = FortranBridgeGenerator().visit(plan) +``` + +They are private implementation organization inside direct generation, not +public stages. Both visitors recursively traverse the same plan tree and +return actual C or Fortran nodes (or tuples of actual nodes where a child needs +multiple declarations or statements). + +The recursive shape is: + +```text +ModulePlan + -> binding and bridge module contexts and backend nodes + -> NamespacePlan + -> directly owned FunctionPlan and ModuleVariablePlan records + -> binding/bridge argument transfers, result projection, lifecycle actions + -> complete backend function and namespace nodes + -> complete backend module node +``` + +An argument visitor returns the C or Fortran declarations/statements/parameters +needed for that backend. A result visitor returns the backend result nodes. +Lifecycle visitors return backend writeback, cleanup, or release nodes. Parent +visitors assemble these concrete child results directly into complete syntax +nodes. Do not introduce another wrapper-specific transport model. + +The public orchestration stays visibly direct: + +```python +class WrapperCodeGenerator: + def generate(self, plan: ModulePlan) -> RenderedGeneratedWrapperArtifacts: + plan.freeze() + self._validate_plan(plan) + self._c_generator.require_supported(plan) + self._fortran_generator.require_supported(plan) + + c_module, c_header = self._c_generator.visit(plan) + fortran_module = self._fortran_generator.visit(plan) + + c_source = self._c_printer.doprint(c_module) + c_header_source = self._c_printer.doprint(c_header) + fortran_source = self._fortran_printer.doprint(fortran_module) + return self._rendered_artifacts( + plan.owner_path, + c_source, + c_header_source, + fortran_source, + ) +``` + +The generator constructs `RenderedGeneratedWrapperArtifacts` directly from the +printed source plus artifact metadata. It does not duplicate native build plans, +compiler selection, link ordering, native-support installation, or compilation +policy; those remain in existing build/link orchestration. + +## Direct Lowering Methods + +Each backend visitor dispatches plan nodes by class through +`_visit_`. A visitor method then calls a typed +`_lower_` helper for each completed action family owned by that +backend. The helper uses an explicit, exhaustive action match and calls one +concrete `_lower__` implementation method. For example: + +```python +def _visit_ModuleVariablePlan(self, plan): + return ( + *self._lower_module_getter(plan), + *self._lower_module_setter(plan), + ) + +def _lower_module_getter(self, plan): + match plan.binding.getter_action: + case ModuleGetterAction.CONSTANT_VALUE: + return self._lower_module_getter_constant_value(plan) + case ModuleGetterAction.DIRECT_VALUE: + return self._lower_module_getter_direct_value(plan) + case ModuleGetterAction.NULLABLE_SNAPSHOT: + return self._lower_module_getter_nullable_snapshot(plan) + raise ValueError(...) +``` + +The C binding dispatches only from binding-owned actions, and the Fortran +bridge dispatches only from bridge-owned actions. In particular, native module +setter generation consumes the completed bridge assignment action rather than +the Python setter-exposure action. Post-IR policy completion records +`AssignmentMode.NONE` when no native setter is exposed and +`AssignmentMode.VALUE_COPY` for supported scalar value write-through; bridge +lowering does not reconstruct that choice from the Python setter action. +Backend support checks retain genuine ABI and capability validation; action +dispatch itself raises explicitly for every unsupported value, including an +unsupported alias assignment. + +Do not synthesize implementation method names, use `getattr` to execute +lowering, retain a fallback behavior, or store dispatcher names in the plan. +Do not create extra getter or setter plan nodes solely to gain more +`_visit_` methods. Both visitor and lowering methods return backend +syntax nodes; printers remain the only layer that renders those nodes as source +text. + +Primitive dtype spelling and converter differences live in the intentionally +scalar-specific `PrimitiveScalarTypeRegistry`; they do not duplicate control +flow methods or select semantic policy. + +Within policy, planning, support analysis, validation, and both backend +visitors, family-specific helpers stay in visibly labeled contiguous groups: +scalar helpers, string helpers, and ordinary-array helpers. Put a short section +comment above every such group so maintainers can find one datatype family +without scanning interleaved lowering methods. Generic orchestration remains +outside those groups and dispatches into them through the completed typed +actions. + +## Migration and Route Rules + +The legacy route remains the behavioral oracle until a lane has direct-plan +parity. Route selection is atomic per merged extension: a generation unit uses +either the direct wrapper-plan route or the legacy route. It never combines one +backend from one route with the other backend from the other route. + +The documented public contract is authoritative when it intentionally corrects +legacy behavior. In that case, use the legacy implementation to simplify the +mechanical ABI, conversion, ownership, and cleanup audit, improve the design +where the legacy path is unsafe or unnecessarily complex, and record every +intentional behavioral difference in focused tests. Do not preserve a known +legacy defect merely to obtain byte-for-byte or semantic parity. + +An unsupported owner may select the legacy route before planning. Once the plan +route is selected, planning, validation, lowering, printing, or compilation +failure fails the build; it must not fall back to legacy generation. + +Support reports and rollout gates keep scalar, string, and ordinary-array +input, optional, writeback, direct-result, and hidden-result lanes distinct. +Evidence for one datatype family must not make another family production +eligible accidentally. + +For each lane: + +1. replay an existing passing `tests/wrapper` case through the legacy route and + retain its generated artifacts; +2. record the relevant legacy source paths, ABI/call order, ownership and + cleanup behavior, artifact requirements, and runtime assertions; +3. complete every missing semantic decision before planning; +4. add the smallest required plan record and directly named lowering method; +5. produce the same complete artifact set through the direct route; +6. inspect differences, compile both routes, and run the existing assertions; +7. update checklist evidence only after direct-route parity is proven. + +Generated source is diagnostic evidence, not a byte-for-byte golden. Backend +temporary names and equivalent control flow may differ, but ABI, conversion, +ownership, cleanup, call order, and artifact requirements must remain proven. + +During this migration the full real-library BLAS/LAPACK wrapper corpus is +excluded locally and in CI until final cutover. General native-bundle coverage +remains active. + +## Staged Walkthrough + +`tools/wrapper_plan_staged_walkthrough.py` is the maintained hand-inspection +path. It shows only the source/contract entry, policy completion, plan creation, +a direct edit, direct generation, artifact inspection, build, and runtime use: + +```python +module = ... +complete_semantic_policies(module) + +plan = WrapperPlanner().build(module) +namespace = next(item for item in plan.namespaces if item.python_path == ()) +function = namespace.functions[0] +function.bridge.native_name = "SUB_R8" + +binding = CBindingGenerator() +bridge = FortranBridgeGenerator() +print(function.arguments[0].binding.optional_mode) +print(function.arguments[0].bridge.optional_mode) + +artifacts = WrapperCodeGenerator( + c_generator=binding, + fortran_generator=bridge, +).generate(plan) + +# inspect generated files +# build and run +``` + +It does not expose standalone validation. Printed plan inspection uses the +actual namespace, owner, and completed action records. The backend visitors +make the corresponding explicit action matches visible in their typed lowering +helpers. + +## Required Evidence + +Focused tests must prove: + +- `WrapperPlanner.build(module)` returns a directly mutable plan; +- direct edits to binding and bridge views change the relevant generated C and + Fortran source; +- `WrapperCodeGenerator.generate(plan)` freezes the exact consumed plan; +- module visitors recursively include generated function nodes; +- function visitors recursively include argument, result, and lifecycle nodes; +- directly named backend lowering methods cover every supported plan action; +- unsupported combinations fail explicitly; +- source printers freeze backend module nodes; +- generated artifacts remain editable until build consumption, which freezes + them; +- source and semantic-`.pyi` entries preserve compiled runtime parity; and +- backend lowering does not reconstruct semantic policy. + +Use package-export inspection and focused migration checks to prove removal of +obsolete internal representations; do not preserve tests whose only assertion +is that a removed API is absent. + +## Recovered Roadmap Scope + +The detailed migration queue below is retained from the original roadmap. The +obsolete Phase 0-2 emitter/fragment architecture is replaced by the simplified +direct-plan checklist later in this file; all later semantic lanes, matrix rows, +verification gates, and completion records remain explicit. + +## Existing Wrapper Suite As The Migration Queue + +`tests/wrapper` is the behavioral source and final acceptance suite for this +migration. Migrate its existing generation units one by one; do not create a +parallel wrapper suite or new native source fixtures merely to make the new +route easier to exercise. + +- Phase 0A adds a maintained migration matrix to this file covering every + Python test node under `tests/wrapper`. Each row records whether the test + generates a wrapper, the source/contract generation unit it uses, its + relevant feature lanes, and one status: + `not-applicable`, `deferred-real-library`, `legacy`, `dual-route`, or + `wrapper-plan`. +- Existing source files, contract fixtures, build helpers, runtime assertions, + failure assertions, and ABI assertions are reused as written whenever they + already cover the migrated behavior. Do not copy their behavior into a new + test with a smaller invented source. +- A new native source or contract fixture is allowed only when the audit proves + that accepted production behavior has no existing test. Record that coverage + gap and its owning semantic lane here before adding the fixture; migration + convenience is not sufficient justification. +- Whole-generation-unit routing still applies. An existing test moves to + `dual-route` only when every runtime-required feature in its module is + supported. If a nominally scalar fixture also contains results, strings, + arrays, decorators, module state, or classes, leave it on the legacy route + until those lanes are complete rather than carving out a narrower fixture. +- Dual-route parity reuses the same existing fixture and assertion function for + legacy and wrapper-plan builds through internal test orchestration. Do not + add a public route flag, duplicate the behavioral assertions, or require + byte-identical generated source. +- Once parity passes and production eligibility is widened, that existing test + moves to `wrapper-plan`. Keep deliberate legacy execution only in the + migration parity harness until final cutover. +- The final target is not merely that `tests/wrapper` passes. Every test in the + suite must be represented in the migration matrix, and every test that + generates a runtime wrapper must use the wrapper-plan route after cutover. + Tests that only inspect documentation, layout, parsing, or `.pyi` generation + may be `not-applicable` but must still pass. +- During active migration, every local and GitHub Actions pytest invocation + excludes + `tests/wrapper/fortran/real_libraries/test_real_blas_lapack.py`. Mark its BLAS + and LAPACK rows `deferred-real-library`; do not use either corpus for lane + parity or general migration verification. General native-bundle tests in + `test_stage7_native_bundles.py` remain active because they test linker/build + mechanics independently of the full BLAS/LAPACK corpora. + +### Wrapper Test Migration Matrix + +Matrix rows use pytest selector patterns. A row ending in `::*` covers every +collected test node in that Python file when all nodes share the same +generation classification. A row ending in `[*]` covers the parametrized nodes +for that test function. The structural layout test expands these selectors +against live `python3 -m pytest --collect-only -q tests/wrapper` output, so a +new wrapper test node must either match an existing row intentionally or add a +new row here before later implementation starts. + +Statuses have the meanings defined above: `legacy` still uses the current +`semantic_ir_to_codegen_ast()` route, `dual-route` runs the same generation +unit and runtime assertions through both implementations, `wrapper-plan` uses +only `WrapperPlan -> WrapperCodeGenerator`, `not-applicable` does not generate +a runtime wrapper, and `deferred-real-library` is reserved for the full BLAS +and LAPACK corpus until Phase 12. + +#### Current Wrapper Route Counts + +These are collected pytest-node counts, not matrix-row counts. The structural +layout test derives them from live `tests/wrapper` collection and fails if this +summary, the exhaustive matrix, and the test tree disagree. + +| Status | Collected nodes | +| --- | ---: | +| `wrapper-plan` | 349 | +| `dual-route` | 0 | +| `legacy` | 0 | +| `not-applicable` | 75 | +| `deferred-real-library` | 0 | + +#### Recorded Route Progression + +This history keeps phase movement visible instead of replacing the previous +snapshot with only the latest totals. Phase 2D moved all 17 dual-route nodes +and 44 legacy nodes to production plan routing, then added two parametrized +plan-route nodes. Phase 2E adds two scalar-only parity nodes, and Phase 2F adds +one isolated direct-return plus hidden-output scalar aggregation node. The +original mixed integration nodes retain their real array/string/object +blockers. + +| Proven checkpoint | `wrapper-plan` | `dual-route` | `legacy` | `not-applicable` | `deferred-real-library` | Total | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Before Phase 2D | 0 | 17 | 178 | 95 | 2 | 292 | +| Phase 2D complete | 63 | 0 | 134 | 95 | 2 | 294 | +| Phase 2E scalar isolation | 65 | 0 | 134 | 95 | 2 | 296 | +| Phase 2F scalar result aggregation | 66 | 0 | 134 | 95 | 2 | 297 | +| Phase 5A required string values | 67 | 0 | 134 | 95 | 2 | 298 | +| Phase 5B fixed string results | 69 | 0 | 134 | 95 | 2 | 300 | +| Phase 5C fixed string writeback | 70 | 0 | 134 | 95 | 2 | 301 | +| Phase 5C assumed/optional string writeback | 71 | 0 | 134 | 95 | 2 | 302 | +| Phase 5D fixed string storage/raw addresses | 72 | 0 | 134 | 95 | 2 | 303 | +| Phase 5 production route reconciliation | 76 | 0 | 130 | 95 | 2 | 303 | +| Phase 6 ordinary arrays | 78 | 5 | 130 | 95 | 2 | 310 | +| Phase 6G raw array addresses | 80 | 5 | 130 | 95 | 2 | 312 | +| Phase 6 `COPY_F` representation copy | 81 | 5 | 130 | 95 | 2 | 313 | +| Phase 7 native handles/descriptors | 88 | 5 | 129 | 96 | 2 | 320 | +| Phase 7 production route reconciliation | 94 | 5 | 123 | 95 | 2 | 319 | +| Phase 8 scalar-derived object lifetimes | 106 | 5 | 123 | 95 | 2 | 331 | +| Phase 8 complete scalar-derived actual/dummy matrix | 213 | 5 | 123 | 95 | 2 | 438 | +| Phase 8H failure, qualified-type, and typed-value closure | 222 | 5 | 123 | 95 | 2 | 447 | +| Phase 11 cross-cutting suite completion | 344 | 0 | 0 | 95 | 2 | 441 | +| Phase 12 canonical cutover | 346 | 0 | 0 | 95 | 0 | 441 | + +Migration is complete only when `legacy`, `dual-route`, and +`deferred-real-library` are all zero. At that point every runtime-generating +node must be `wrapper-plan`; `not-applicable` may remain only for tests that do +not generate a wrapper. Until then, moving a node from `legacy` to `dual-route` +records proven parity, and moving it from `dual-route` to `wrapper-plan` +records final removal of its legacy execution. + +#### Complete Route Ledger + +For a `legacy` row, the feature-lane column identifies what still blocks the +new route. For `dual-route` and `wrapper-plan` rows, it identifies the behavior +already covered by the new generator. + +| Pytest selector | Generation unit | Feature lanes / blockers | Status | +| --- | --- | --- | --- | +| `tests/wrapper/fortran/arrays/test_array_contracts.py::*` | source/generated-.pyi parity or parametrized route | ordinary arrays | `wrapper-plan` | +| `tests/wrapper/fortran/arrays/test_array_generated_pyi_contracts.py::*` | non-generating: generated semantic .pyi fixture parity | semantic .pyi generation/parsing | `not-applicable` | +| `tests/wrapper/fortran/arrays/test_array_results.py::test_array_results_follow_data_buffer_and_descriptor_handle_contracts[*]` | production plan route in source/generated-.pyi parity modes | fixed/runtime-shape ordinary array results; owned allocatable descriptor results; namespace preservation | `wrapper-plan` | +| `tests/wrapper/fortran/arrays/test_array_results.py::test_ordinary_array_results_use_canonical_plan` | canonical production output-only plan route | fixed/runtime-shape ordinary array results; ranks one through fifteen; Fortran order; zero-sized results; allocation/copy/release failure paths | `wrapper-plan` | +| `tests/wrapper/fortran/arrays/test_array_results.py::test_owned_allocatable_results_preserve_handle_state` | canonical reduced owned-result contract | allocated and zero-sized wrapper-owned `CFI_CDESC_T` function-result handles; extraction and release | `wrapper-plan` | +| `tests/wrapper/fortran/arrays/test_assumed_rank_arrays.py::test_assumed_rank_arguments_dispatch_to_runtime_rank[*]` | source/generated-.pyi parity | ordinary arrays; native-handle actuals deferred to Phase 7 | `wrapper-plan` | +| `tests/wrapper/fortran/arrays/test_assumed_rank_arrays.py::test_assumed_rank_bridge_dispatches_each_runtime_rank_argument[*]` | source/generated-.pyi parity | ordinary arrays; native-handle actuals deferred to Phase 7 | `wrapper-plan` | +| `tests/wrapper/fortran/arrays/test_assumed_rank_arrays.py::test_assumed_rank_arrays_use_explicit_plan_branches` | reduced semantic `.pyi` entry over the existing assumed-rank native unit | runtime ranks one through fifteen; mutable storage; rank validation; native-handle actuals deferred to Phase 7 | `wrapper-plan` | +| `tests/wrapper/fortran/arrays/test_multidimensional_arrays.py::test_rank2_contiguous_contract_requires_fortran_contiguous[*]` | source/generated-.pyi parity | ordinary arrays; native-handle actuals deferred to Phase 7 | `wrapper-plan` | +| `tests/wrapper/fortran/arrays/test_multidimensional_arrays.py::test_rank2_assumed_shape_accepts_fortran_ordered_strided_views[*]` | source/generated-.pyi parity | ordinary arrays; native-handle actuals deferred to Phase 7 | `wrapper-plan` | +| `tests/wrapper/fortran/arrays/test_multidimensional_arrays.py::test_rank2_assumed_shape_rejects_non_positive_strides[*]` | source/generated-.pyi parity | ordinary arrays; native-handle actuals deferred to Phase 7 | `wrapper-plan` | +| `tests/wrapper/fortran/arrays/test_multidimensional_arrays.py::test_rank2_explicit_shape_requires_fortran_contiguous[*]` | source/generated-.pyi parity | ordinary arrays; native-handle actuals deferred to Phase 7 | `wrapper-plan` | +| `tests/wrapper/fortran/arrays/test_multidimensional_arrays.py::test_rank3_contiguous_contract_requires_fortran_contiguous[*]` | source/generated-.pyi parity | ordinary arrays; native-handle actuals deferred to Phase 7 | `wrapper-plan` | +| `tests/wrapper/fortran/arrays/test_multidimensional_arrays.py::test_rank3_assumed_shape_accepts_fortran_ordered_strided_views[*]` | source/generated-.pyi parity | ordinary arrays; native-handle actuals deferred to Phase 7 | `wrapper-plan` | +| `tests/wrapper/fortran/arrays/test_multidimensional_arrays.py::test_dense_strided_and_projected_arrays_use_canonical_plan` | reduced semantic `.pyi` entry over the existing multidimensional native unit | dense/explicit extents; positive-strided views; zero-sized axes; projected output identity; native-handle actuals deferred to Phase 7 | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_pyi/test_contract_package_runtime.py::test_init_entry_uses_resolved_parent_name_from_inside_package` | direct wrapper/build route | semantic .pyi generation/parsing; build/compile/link orchestration; scalar inputs/results | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_pyi/test_contract_package_runtime.py::test_output_name_override_replaces_entry_inference` | direct wrapper/build route | semantic .pyi generation/parsing; build/compile/link orchestration; scalar inputs/results | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_pyi/test_contract_package_runtime.py::test_recursive_graph_preserves_module_and_symbol_aliases_and_ignores_support_imports` | direct wrapper/build route | semantic .pyi generation/parsing; build/compile/link orchestration; scalar inputs/results | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_pyi/test_contract_package_runtime.py::test_recursive_graph_reports_cycles_before_codegen` | non-generating: validation/failure-path assertion | semantic .pyi generation/parsing; build/compile/link orchestration | `not-applicable` | +| `tests/wrapper/fortran/build_from_pyi/test_contract_package_runtime.py::test_recursive_graph_reports_missing_relative_contract_before_native_validation` | non-generating: validation/failure-path assertion | semantic .pyi generation/parsing; build/compile/link orchestration | `not-applicable` | +| `tests/wrapper/fortran/build_from_pyi/test_contract_package_runtime.py::test_source_build_preserves_modules_and_root_externals` | direct wrapper/build route | semantic .pyi generation/parsing; build/compile/link orchestration; scalar inputs/results; module variables/state | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_pyi/test_contract_package_runtime.py::test_complete_general_source_preserves_namespaces_through_canonical_plan[*]` | canonical production plan route | Python namespace hierarchy; native import aliases; scalar inputs/results; void calls | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_pyi/test_pyi_wrapper_builds.py::test_entry_can_alias_one_module_procedure_at_the_root` | direct wrapper/build route | semantic .pyi generation/parsing; build/compile/link orchestration; scalar inputs/results | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_pyi/test_pyi_wrapper_builds.py::test_entry_rejects_colliding_wildcard_exports` | non-generating: validation/failure-path assertion | semantic .pyi generation/parsing; build/compile/link orchestration | `not-applicable` | +| `tests/wrapper/fortran/build_from_pyi/test_pyi_wrapper_builds.py::test_entry_wildcard_import_explicitly_flattens_module_leaf` | direct wrapper/build route | semantic .pyi generation/parsing; build/compile/link orchestration; scalar inputs/results; module variables/state | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_pyi/test_pyi_wrapper_builds.py::test_generated_pyi_fixture_builds_from_native_object_without_source_reparse` | direct wrapper/build route | semantic .pyi generation/parsing; build/compile/link orchestration; scalar inputs/results | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_pyi/test_pyi_wrapper_builds.py::test_generated_pyi_matches_checked_in_fixture` | direct wrapper/build route | semantic .pyi generation/parsing; build/compile/link orchestration; scalar inputs/results | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_pyi/test_pyi_wrapper_builds.py::test_mixed_entry_exposes_externals_at_root_and_modules_as_children` | direct wrapper/build route | semantic .pyi generation/parsing; build/compile/link orchestration; scalar inputs/results; module variables/state | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_pyi/test_pyi_wrapper_builds.py::test_module_leaf_can_be_the_entry_contract` | direct wrapper/build route | semantic .pyi generation/parsing; build/compile/link orchestration; scalar inputs/results; module variables/state | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_pyi/test_pyi_wrapper_builds.py::test_module_variable_runtime_contract[*]` | source/generated-.pyi parity or parametrized route | semantic .pyi generation/parsing; build/compile/link orchestration; scalar inputs/results; module variables/state | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_pyi/test_pyi_wrapper_builds.py::test_mutable_module_variable_default_initializes_native_storage` | direct wrapper/build route | semantic .pyi generation/parsing; build/compile/link orchestration; scalar inputs/results; module variables/state | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_pyi/test_pyi_wrapper_builds.py::test_one_entry_preserves_multiple_native_module_namespaces` | direct wrapper/build route | semantic .pyi generation/parsing; build/compile/link orchestration; scalar inputs/results | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_pyi/test_pyi_wrapper_builds.py::test_pyi_cli_accepts_exactly_one_entry_contract` | direct wrapper/build route | semantic .pyi generation/parsing; build/compile/link orchestration; scalar inputs/results | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_pyi/test_pyi_wrapper_builds.py::test_pyi_cli_preserves_explicit_ordered_link_items` | direct wrapper/build route | semantic .pyi generation/parsing; build/compile/link orchestration; scalar inputs/results | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_pyi/test_pyi_wrapper_builds.py::test_pyi_cli_requires_a_native_link_input` | non-generating: validation/failure-path assertion | semantic .pyi generation/parsing; build/compile/link orchestration | `not-applicable` | +| `tests/wrapper/fortran/build_from_pyi/test_pyi_wrapper_builds.py::test_pyi_makefile_manifest_and_replay_workflows` | direct wrapper/build route | semantic .pyi generation/parsing; build/compile/link orchestration; scalar inputs/results | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_pyi/test_pyi_wrapper_builds.py::test_pyi_manifest_records_pointer_descriptor_interop_requirements` | non-generating: manifest serialization unit | completed native-array build requirements and local standard-descriptor headers | `not-applicable` | +| `tests/wrapper/fortran/build_from_pyi/test_pyi_wrapper_builds.py::test_pyi_python_api_accepts_exactly_one_entry_contract` | direct wrapper/build route | semantic .pyi generation/parsing; build/compile/link orchestration; scalar inputs/results | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_pyi/test_pyi_wrapper_builds.py::test_pyi_python_api_rejects_a_missing_native_artifact` | non-generating: validation/failure-path assertion | semantic .pyi generation/parsing; build/compile/link orchestration | `not-applicable` | +| `tests/wrapper/fortran/build_from_pyi/test_pyi_wrapper_builds.py::test_pyi_python_api_rejects_invalid_address_contracts_before_codegen[*]` | non-generating: validation/failure-path assertion | semantic .pyi generation/parsing; build/compile/link orchestration | `not-applicable` | +| `tests/wrapper/fortran/build_from_pyi/test_pyi_wrapper_builds.py::test_pyi_python_api_rejects_invalid_projection_before_codegen` | non-generating: validation/failure-path assertion | semantic .pyi generation/parsing; build/compile/link orchestration | `not-applicable` | +| `tests/wrapper/fortran/build_from_pyi/test_pyi_wrapper_builds.py::test_pyi_python_api_rejects_python_suffix_as_semantic_contract` | non-generating: validation/failure-path assertion | semantic .pyi generation/parsing; build/compile/link orchestration | `not-applicable` | +| `tests/wrapper/fortran/build_from_pyi/test_pyi_wrapper_builds.py::test_reduced_entry_generates_only_reachable_module_variable_bindings` | direct wrapper/build route | semantic .pyi generation/parsing; build/compile/link orchestration; scalar inputs/results; module variables/state | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_pyi/test_pyi_wrapper_builds.py::test_scale_runtime_contract[*]` | source/generated-.pyi parity or parametrized route | semantic .pyi generation/parsing; build/compile/link orchestration; scalar inputs/results | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_pyi/test_pyi_wrapper_builds.py::test_source_named_root_discovers_and_builds_module_leaf` | direct wrapper/build route | semantic .pyi generation/parsing; build/compile/link orchestration; scalar inputs/results; module variables/state | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_source/test_build_modes.py::test_fortran_wrapper_default_module_name_does_not_collide_with_root_function` | direct wrapper/build route | build/compile/link orchestration | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_source/test_build_modes.py::test_fortran_wrapper_default_places_artifacts_in_invocation_directory` | direct wrapper/build route | build/compile/link orchestration | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_source/test_build_modes.py::test_fortran_wrapper_out_dir_separates_abi_artifact_from_cli_alias` | direct wrapper/build route | build/compile/link orchestration | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_source/test_build_modes.py::test_fortran_wrapper_out_names_importable_shared_library` | direct wrapper/build route | build/compile/link orchestration; scalar inputs/results | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_source/test_build_modes.py::test_internal_preprocessing_mode_still_builds_importable_runtime_wrapper` | direct wrapper/build route | build/compile/link orchestration; scalar inputs/results | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_source/test_build_modes.py::test_native_link_plan_serializes_interleaved_item_kinds` | direct wrapper/build route | build/compile/link orchestration | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_source/test_build_modes.py::test_source_build_result_records_structured_native_plan` | direct wrapper/build route | build/compile/link orchestration | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_source/test_build_modes.py::test_verbose_mode_prints_custom_wrapper_flags` | direct wrapper/build route | build/compile/link orchestration | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_source/test_build_modes.py::test_verbose_mode_prints_full_direct_build_commands` | direct wrapper/build route | build/compile/link orchestration | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_source/test_build_modes.py::test_wrapper_build_rejects_empty_source_list` | non-generating: validation/failure-path assertion | build/compile/link orchestration | `not-applicable` | +| `tests/wrapper/fortran/build_from_source/test_build_modes.py::test_wrapper_build_rejects_makefile_verbose_combination` | non-generating: validation/failure-path assertion | build/compile/link orchestration | `not-applicable` | +| `tests/wrapper/fortran/build_from_source/test_build_modes.py::test_wrapper_build_rejects_missing_source` | non-generating: validation/failure-path assertion | build/compile/link orchestration | `not-applicable` | +| `tests/wrapper/fortran/build_from_source/test_compiler_verbose.py::*` | direct wrapper/build route | build/compile/link orchestration | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_source/test_runtime_abi.py::*` | direct wrapper/build route | build/compile/link orchestration | `wrapper-plan` | +| `tests/wrapper/fortran/build_from_source/test_source_generated_pyi_contracts.py::*` | non-generating: generated semantic .pyi fixture parity | semantic .pyi generation/parsing | `not-applicable` | +| `tests/wrapper/fortran/callbacks/test_all_callback_shapes.py::*` | source/generated-.pyi parity or parametrized route | callbacks/trampolines | `wrapper-plan` | +| `tests/wrapper/fortran/callbacks/test_array_callbacks.py::*` | source/generated-.pyi parity or parametrized route | callbacks/trampolines; ordinary arrays | `wrapper-plan` | +| `tests/wrapper/fortran/callbacks/test_callback_generated_pyi_contracts.py::*` | non-generating: generated semantic .pyi fixture parity | semantic .pyi generation/parsing | `not-applicable` | +| `tests/wrapper/fortran/callbacks/test_derived_callbacks.py::*` | source/generated-.pyi parity or parametrized route | callbacks/trampolines; derived types/object lifetimes | `wrapper-plan` | +| `tests/wrapper/fortran/callbacks/test_scalar_callbacks.py::*` | source/generated-.pyi parity or parametrized route | callbacks/trampolines; scalar inputs/results | `wrapper-plan` | +| `tests/wrapper/fortran/derived_types/test_borrowed_finalizers.py::*` | source/generated-.pyi parity or parametrized route | derived types/object lifetimes; classes/methods/properties/overloads | `wrapper-plan` | +| `tests/wrapper/fortran/derived_types/test_constructors_and_finalizers.py::*` | source/generated-.pyi parity or parametrized route | derived types/object lifetimes; classes/methods/properties/overloads | `wrapper-plan` | +| `tests/wrapper/fortran/derived_types/test_derived_layout.py::*` | source/generated-.pyi parity or parametrized route | derived types/object lifetimes | `wrapper-plan` | +| `tests/wrapper/fortran/derived_types/test_derived_type_boundaries.py::*` | source/generated-.pyi parity or parametrized route | derived types/object lifetimes | `wrapper-plan` | +| `tests/wrapper/fortran/derived_types/test_derived_type_generated_pyi_contracts.py::*` | non-generating: generated semantic .pyi fixture parity | semantic .pyi generation/parsing | `not-applicable` | +| `tests/wrapper/fortran/derived_types/test_derived_type_methods.py::*` | source/generated-.pyi parity or parametrized route | derived types/object lifetimes; classes/methods/properties/overloads | `wrapper-plan` | +| `tests/wrapper/fortran/derived_types/test_inheritance.py::*` | source/generated-.pyi parity or parametrized route | derived types/object lifetimes; classes/methods/properties/overloads | `wrapper-plan` | +| `tests/wrapper/fortran/derived_types/test_phase8_derived_plan.py::*` | reduced passing legacy/source artifacts compared with direct typed-plan generation; plain non-target module objects intentionally use the safer member-proxy correction described in Phase 8 | scalar derived arguments/results; optional and by-value inputs; projected identity; owned/borrowed lifecycle; plain/`Aliased` module objects; scalar/string/array/nested/native-handle fields; production routing | `wrapper-plan` | +| `tests/wrapper/fortran/derived_types/test_phase9_bound_constructors.py::*` | reduced direct-plan bound-constructor runtime and artifact proof | explicit bound construction; shared method plan; allocation and owner commit | `wrapper-plan` | +| `tests/wrapper/fortran/derived_types/test_scalar_derived_actual_dummy_matrix.py::*` | complete source/generated-contract and direct-plan proof over the canonical scalar-derived matrix fixture; replaces the former isolated descriptor rejection unit; final Phase 8H cross-suite verification remains a separate closure gate | all five actual declarations from module and wrapper origins; all six dummy forms; exact action/error selection; holder, scoped-address, allocation and pointer transactions; mixed multi-argument acquisition and reverse cleanup; distinct module-origin callbacks for qualified types from separate Fortran modules | `wrapper-plan` | +| `tests/wrapper/fortran/derived_types/test_pointers.py::test_module_and_derived_pointer_handles_track_native_association[*]` | source/generated-.pyi parity or parametrized route | derived types/object lifetimes; native handles/descriptors | `wrapper-plan` | +| `tests/wrapper/fortran/derived_types/test_pointers.py::test_module_native_array_handles_use_canonical_plan` | canonical reduced module-only contract | borrowed pointer/allocatable handles; descriptor calls; strided extraction; ordinary array actuals; operation permissions | `wrapper-plan` | +| `tests/wrapper/fortran/derived_types/test_pointers.py::test_pointer_array_handles_block_on_unsupported_result_owner_policy[*]` | source/generated-.pyi parity or parametrized route | derived types/object lifetimes; native handles/descriptors | `wrapper-plan` | +| `tests/wrapper/fortran/derived_types/test_pointers.py::test_pointer_descriptor_views_preserve_slice_shape_strides_and_parent_lifetime` | direct wrapper/build route | derived types/object lifetimes; native handles/descriptors | `wrapper-plan` | +| `tests/wrapper/fortran/edit_pyi_contracts/test_native_order_contracts.py::test_editable_contract_can_use_native_order_arguments_without_native_call` | direct wrapper/build route | semantic .pyi generation/parsing; raw array addresses completed by Phase 6G; derived result remains Phase 8 | `wrapper-plan` | +| `tests/wrapper/fortran/edit_pyi_contracts/test_native_order_contracts.py::test_raw_array_addresses_use_canonical_plan` | reduced edited semantic `.pyi` entries over existing vector/matrix native routines | raw numeric addresses; visible scalar-storage extents; rank one/two; default C and explicit Fortran orientation; mutation; integer-only conversion | `wrapper-plan` | +| `tests/wrapper/fortran/edit_pyi_contracts/test_native_order_contracts.py::test_copy_f_preserves_logical_axes_through_binding_owned_temporary` | reduced edited semantic `.pyi` entries over the existing matrix native routine | explicit C-to-Fortran representation copy; native-input and inout calls; projected original identity; binding-owned copyback and cleanup | `wrapper-plan` | +| `tests/wrapper/fortran/edit_pyi_contracts/test_native_order_contracts.py::test_fixed_string_storage_and_raw_address_use_canonical_plan` | reduced edited semantic `.pyi` entry over the existing `fnative_call_examples_f90` native unit | fixed mutable rank-zero NumPy bytes storage; raw fixed-string addresses; in-place mutation; rank/dtype/itemsize/writability/type validation | `wrapper-plan` | +| `tests/wrapper/fortran/edit_pyi_contracts/test_ownership_contracts.py::*` | direct wrapper/build route | semantic .pyi generation/parsing; native handles/descriptors; derived types/object lifetimes; optional/presence/writeback | `wrapper-plan` | +| `tests/wrapper/fortran/edit_pyi_contracts/test_policy_dispatch_contracts.py::test_contradictory_ownership_contract_fails_before_bridge_generation` | non-generating: policy validation before bridge generation | semantic .pyi generation/parsing; native handles/descriptors; derived types/object lifetimes; optional/presence/writeback | `not-applicable` | +| `tests/wrapper/fortran/edit_pyi_contracts/test_policy_dispatch_contracts.py::test_immutable_array_policy_copies_in_and_returns_replacement` | direct wrapper/build route | semantic .pyi generation/parsing; native handles/descriptors; derived types/object lifetimes; optional/presence/writeback | `wrapper-plan` | +| `tests/wrapper/fortran/edit_pyi_contracts/test_policy_dispatch_contracts.py::test_immutable_scalar_string_array_and_derived_policies_return_replacements` | direct wrapper/build route | semantic .pyi generation/parsing; native handles/descriptors; derived types/object lifetimes; optional/presence/writeback | `wrapper-plan` | +| `tests/wrapper/fortran/edit_pyi_contracts/test_surface_edit_contracts.py::*` | direct wrapper/build route | semantic .pyi generation/parsing; classes/methods/properties/overloads; naming/visibility/dispatch | `wrapper-plan` | +| `tests/wrapper/fortran/edit_pyi_contracts/test_visibility_contracts.py::*` | direct wrapper/build route | semantic .pyi generation/parsing; scalar module visibility and namespace projection | `wrapper-plan` | +| `tests/wrapper/fortran/external_routines/test_external_procedures.py::test_compact_blas_like_folder_generates_one_external_entry_and_preserves_separate_objects` | direct wrapper/build route | external symbols/native linkage; ordinary arrays | `wrapper-plan` | +| `tests/wrapper/fortran/external_routines/test_external_procedures.py::test_external_bind_renames_python_export_without_changing_native_call` | direct wrapper/build route | scalar external symbol; explicit bridge interface; renamed export | `wrapper-plan` | +| `tests/wrapper/fortran/external_routines/test_external_procedures.py::test_classic_external_bridge_uses_implicit_declaration_and_no_module_use` | direct wrapper/build route | scalar external symbol; implicit external declaration | `wrapper-plan` | +| `tests/wrapper/fortran/external_routines/test_external_procedures.py::test_fixed_form_standalone_external_runtime_parity[*]` | source/generated-.pyi parity or parametrized route | scalar external symbol; explicit bridge interface | `wrapper-plan` | +| `tests/wrapper/fortran/external_routines/test_external_procedures.py::test_free_form_standalone_external_runtime_parity[*]` | source/generated-.pyi parity or parametrized route | scalar external symbol; explicit bridge interface | `wrapper-plan` | +| `tests/wrapper/fortran/external_routines/test_external_procedures.py::test_generated_external_contracts_are_non_empty_root_fragments` | direct wrapper/build route | external symbols/native linkage | `wrapper-plan` | +| `tests/wrapper/fortran/external_routines/test_external_procedures.py::test_handwritten_c_order_flat_contract_passes_rank_preserving_bridge_view` | direct wrapper/build route | external symbols/native linkage; ordinary arrays | `wrapper-plan` | +| `tests/wrapper/fortran/external_routines/test_external_procedures.py::test_module_procedure_bridge_uses_native_module_scope` | direct wrapper/build route | external symbols/native linkage | `wrapper-plan` | +| `tests/wrapper/fortran/external_routines/test_external_procedures.py::test_namespace_imported_module_rejects_external_marker_before_codegen` | non-generating: validation/failure-path assertion | external symbols/native linkage | `not-applicable` | +| `tests/wrapper/fortran/external_routines/test_external_procedures.py::test_one_source_with_several_standalone_externals_exports_each_at_root[*]` | source/generated-.pyi parity or parametrized route | scalar external symbols; explicit bridge interfaces | `wrapper-plan` | +| `tests/wrapper/fortran/external_routines/test_external_procedures.py::test_package_entry_rejects_non_external_root_declaration_before_codegen` | non-generating: validation/failure-path assertion | external symbols/native linkage | `not-applicable` | +| `tests/wrapper/fortran/function_calls/test_function_call_generated_pyi_contracts.py::*` | non-generating: generated semantic .pyi fixture parity | semantic .pyi generation/parsing | `not-applicable` | +| `tests/wrapper/fortran/function_calls/test_native_call_examples.py::test_native_call_examples_build_from_generated_pyi_and_native_shared_library` | direct wrapper/build route | native-call projections; ordinary arrays; strings; derived types/object lifetimes | `wrapper-plan` | +| `tests/wrapper/fortran/function_calls/test_native_call_examples.py::test_native_call_examples_cover_scalar_array_string_and_object_projection[*]` | source/generated-.pyi parity or parametrized route | native-call projections; ordinary arrays; strings; derived types/object lifetimes | `wrapper-plan` | +| `tests/wrapper/fortran/function_calls/test_optional_arguments.py::test_fixed_form_optional_arguments_drive_fortran_present_behavior[*]` | source/generated-.pyi parity or parametrized route | optional/presence; scalar inputs/results | `wrapper-plan` | +| `tests/wrapper/fortran/function_calls/test_optional_arguments.py::test_fixed_optional_scalar_plan_matches_all_presence_states` | canonical production plan route | optional/presence; scalar inputs/results; build/artifact integration | `wrapper-plan` | +| `tests/wrapper/fortran/function_calls/test_optional_arguments.py::test_optional_allocatable_scalar_descriptor_distinguishes_omitted_none_and_value` | canonical production plan route | optional/presence; nullable scalar descriptor; build/artifact integration | `wrapper-plan` | +| `tests/wrapper/fortran/function_calls/test_optional_arguments.py::test_optional_array_descriptors_preserve_presence_and_storage_state` | canonical reduced optional descriptor contract | omitted/`None` absence; present unallocated/unassociated and allocated/associated handle states; kind/dtype validation | `wrapper-plan` | +| `tests/wrapper/fortran/function_calls/test_optional_arguments.py::test_optional_arguments_drive_fortran_present_behavior[*]` | source/generated-.pyi parity or parametrized route | optional/presence/writeback | `wrapper-plan` | +| `tests/wrapper/fortran/function_calls/test_optional_arguments.py::test_optional_array_buffers_preserve_omission_and_identity` | reduced semantic `.pyi` entry over the existing optional native unit | omitted/`None`/present ordinary array storage; mutation; projected identity; native-handle actuals deferred to Phase 7 | `wrapper-plan` | +| `tests/wrapper/fortran/function_calls/test_scalar_writeback_plan.py::test_scalar_copy_in_out_returns_replacement` | canonical production plan route | scalar copy-in/native mutation/copy-out/cleanup; build/artifact integration | `wrapper-plan` | +| `tests/wrapper/fortran/function_calls/test_output_arguments.py::test_output_arguments_and_multiple_results_follow_python_projection_rules[*]` | source/generated-.pyi parity | mixed-type multiple-result aggregation; ordinary arrays; strings; derived types/object lifetimes; native handles/descriptors | `wrapper-plan` | +| `tests/wrapper/fortran/function_calls/test_output_arguments.py::test_hidden_ordinary_array_output_uses_canonical_plan` | canonical production output-only plan route | fixed/runtime-shape hidden ordinary array output; zero-sized output; allocation/copy failure paths | `wrapper-plan` | +| `tests/wrapper/fortran/layout_rules/test_wrapper_guide_layout.py::*` | non-generating: wrapper docs/test layout | test/docs layout | `not-applicable` | +| `tests/wrapper/fortran/module_state/test_allocatable_replacement.py::test_allocatable_inout_arrays_mutate_and_return_the_same_handle[*]` | source/generated-.pyi parity route | module variables/state; native handles/descriptors | `wrapper-plan` | +| `tests/wrapper/fortran/module_state/test_allocatable_replacement.py::test_allocatable_replacement_has_no_native_memory_errors[*]` | source/generated-.pyi parity route | module variables/state; native handles/descriptors | `wrapper-plan` | +| `tests/wrapper/fortran/module_state/test_allocatable_replacement.py::test_projected_allocatable_descriptor_preserves_same_handle_identity` | canonical reduced owned-result plus projected-descriptor contract | direct persistent descriptor mutation; allocation/reallocation/deallocation; same-handle result identity | `wrapper-plan` | +| `tests/wrapper/fortran/module_state/test_allocatable_views.py::test_allocatable_module_fields_and_results_expose_lifetime_safe_handles[*]` | source/generated-.pyi parity with one mixed generation unit | derived class/field handles and parent retention remain Phase 8/9 blockers | `wrapper-plan` | +| `tests/wrapper/fortran/module_state/test_allocatable_views.py::test_scalar_descriptor_module_variables_return_copied_optional_values[*]` | production plan route in source/generated-.pyi parity modes | rank-zero allocatable/pointer arguments, writeback, results, and copied nullable module values | `wrapper-plan` | +| `tests/wrapper/fortran/module_state/test_allocatable_views.py::test_plain_allocatable_module_array_exposes_current_live_view[*]` | production plan route after the Phase 7 contract correction | plain and `Aliased` module handles return a current live view or `None`; explicit `.copy()` is independent and a fresh extraction follows current native state | `wrapper-plan` | +| `tests/wrapper/fortran/module_state/test_common_blocks.py::*` | source/generated-.pyi parity or parametrized route | scalar calls with internal common-block storage | `wrapper-plan` | +| `tests/wrapper/fortran/module_state/test_module_state.py::test_aliased_derived_module_object_borrows_native_state[*]` | source/generated-.pyi parity or parametrized route | module variables/state; derived types/object lifetimes | `wrapper-plan` | +| `tests/wrapper/fortran/module_state/test_module_state.py::test_scalar_module_variables_use_attributes_and_parameters_have_no_native_setter[*]` | source/generated-.pyi parity or parametrized route | module variables/state | `wrapper-plan` | +| `tests/wrapper/fortran/module_state/test_module_state_generated_pyi_contracts.py::*` | non-generating: generated semantic .pyi fixture parity | semantic .pyi generation/parsing | `not-applicable` | +| `tests/wrapper/fortran/module_state/test_scalar_module_variable_plan.py::test_whole_scalar_module_variable_behavior_uses_canonical_plan` | canonical production plan route | scalar inputs/results; scalar module variables/state; build/artifact integration | `wrapper-plan` | +| `tests/wrapper/fortran/multiple_files/test_multi_source_builds.py::test_makefile_mode_reproduces_multi_source_build` | direct wrapper/build route | build/compile/link orchestration; external symbols/native linkage; module variables/state | `wrapper-plan` | +| `tests/wrapper/fortran/multiple_files/test_multi_source_builds.py::test_multi_file_modules_build_one_merged_extension` | direct wrapper/build route | scalar multi-source build/link orchestration; module variables/state | `wrapper-plan` | +| `tests/wrapper/fortran/multiple_files/test_multi_source_builds.py::test_multi_file_standalone_procedures_build_one_merged_extension` | direct wrapper/build route | scalar multi-source external symbols and link orchestration | `wrapper-plan` | +| `tests/wrapper/fortran/multiple_files/test_multi_source_builds.py::test_multi_source_generated_contract_build_matches_source_runtime_and_link_order` | direct wrapper/build route | build/compile/link orchestration; external symbols/native linkage; module variables/state; semantic .pyi generation/parsing | `wrapper-plan` | +| `tests/wrapper/fortran/multiple_files/test_multi_source_builds.py::test_multi_source_modified_entry_preserves_modules_and_adds_documented_alias` | direct wrapper/build route | build/compile/link orchestration; external symbols/native linkage; module variables/state | `wrapper-plan` | +| `tests/wrapper/fortran/multiple_files/test_multi_source_builds.py::test_multi_source_pyi_out_writes_one_flat_combined_package` | direct wrapper/build route | build/compile/link orchestration; external symbols/native linkage; module variables/state; semantic .pyi generation/parsing | `wrapper-plan` | +| `tests/wrapper/fortran/naming/test_defined_operators.py::*` | source/generated-.pyi parity or parametrized route | naming/visibility/dispatch; classes/methods/properties/overloads; operators; generic dispatch | `wrapper-plan` | +| `tests/wrapper/fortran/naming/test_generic_interfaces.py::*` | source/generated-.pyi parity or parametrized route | naming/visibility/dispatch; classes/methods/properties/overloads; generic dispatch | `wrapper-plan` | +| `tests/wrapper/fortran/naming/test_naming_generated_pyi_contracts.py::*` | non-generating: generated semantic .pyi fixture parity | semantic .pyi generation/parsing | `not-applicable` | +| `tests/wrapper/fortran/naming/test_phase9_class_overloads.py::*` | reduced direct-plan constructor and method overload runtime proof | class-owned exact predicates; constructor ownership; no speculative calls | `wrapper-plan` | +| `tests/wrapper/fortran/naming/test_visibility_naming.py::test_strict_wrapper_names_reject_python_name_fixes` | direct wrapper/build route | naming/visibility/dispatch; classes/methods/properties/overloads | `wrapper-plan` | +| `tests/wrapper/fortran/naming/test_visibility_naming.py::test_visibility_and_default_python_name_fixing_policy[*]` | source/generated-.pyi parity or parametrized route | naming/visibility/dispatch; classes/methods/properties/overloads | `wrapper-plan` | +| `tests/wrapper/fortran/real_libraries/test_real_blas_lapack.py::*` | canonical full BLAS/LAPACK wrapper generation; BLAS runs locally and both exact nodes run together in the dedicated GitHub Actions job | external symbols/native linkage; build/compile/link orchestration; broad wrapper corpus | `wrapper-plan` | +| `tests/wrapper/fortran/real_libraries/test_stage7_native_bundles.py::test_duplicate_native_definitions_report_linker_error` | direct wrapper/build route | scalar external symbols; linker failure propagation | `wrapper-plan` | +| `tests/wrapper/fortran/real_libraries/test_stage7_native_bundles.py::test_imported_contracts_resolve_from_one_archive_or_shared_library[*]` | source/generated-.pyi parity or parametrized route | external symbols/native linkage; build/compile/link orchestration | `wrapper-plan` | +| `tests/wrapper/fortran/real_libraries/test_stage7_native_bundles.py::test_incompatible_native_artifact_reports_linker_error` | non-generating: validation/failure-path assertion | external symbols/native linkage; build/compile/link orchestration | `not-applicable` | +| `tests/wrapper/fortran/real_libraries/test_stage7_native_bundles.py::test_missing_module_directory_reports_compile_error` | non-generating: validation/failure-path assertion | external symbols/native linkage; build/compile/link orchestration | `not-applicable` | +| `tests/wrapper/fortran/real_libraries/test_stage7_native_bundles.py::test_missing_symbol_reports_native_link_or_loader_error` | non-generating: validation/failure-path assertion | external symbols/native linkage; build/compile/link orchestration | `not-applicable` | +| `tests/wrapper/fortran/real_libraries/test_stage7_native_bundles.py::test_mixed_module_external_bundle_resolves_all_native_input_kinds` | direct wrapper/build route | scalar module/external symbols; ordered native inputs and library directories | `wrapper-plan` | +| `tests/wrapper/fortran/real_libraries/test_stage7_native_bundles.py::test_required_transitive_named_library_resolves_runtime_symbol` | direct wrapper/build route | scalar external symbol; transitive named library | `wrapper-plan` | +| `tests/wrapper/fortran/real_libraries/test_stage7_native_bundles.py::test_static_archive_dependency_order_resolves_transitive_library` | direct wrapper/build route | scalar external symbol; ordered archive linkage | `wrapper-plan` | +| `tests/wrapper/fortran/real_libraries/test_stage7_native_bundles.py::test_static_archive_groups_resolve_cyclic_archive_dependencies` | direct wrapper/build route | scalar external symbol; archive-group linkage | `wrapper-plan` | +| `tests/wrapper/fortran/real_libraries/test_stage7_native_bundles.py::test_unavailable_dependent_shared_library_reports_loader_error` | non-generating: validation/failure-path assertion | external symbols/native linkage; build/compile/link orchestration | `not-applicable` | +| `tests/wrapper/fortran/runtime_behavior/test_openmp_runtime.py::*` | direct wrapper/build route | runtime policies/errors/GIL; build/compile/link orchestration | `wrapper-plan` | +| `tests/wrapper/fortran/runtime_behavior/test_runtime_behavior_generated_pyi_contracts.py::*` | non-generating: generated semantic .pyi fixture parity | semantic .pyi generation/parsing | `not-applicable` | +| `tests/wrapper/fortran/runtime_behavior/test_runtime_policies.py::*` | source and edited-.pyi canonical production plan route | runtime policies/errors/GIL | `wrapper-plan` | +| `tests/wrapper/fortran/runtime_behavior/test_runtime_recursion.py::*` | source/generated-.pyi parity or parametrized route | runtime policies/errors/GIL; scalar inputs/results | `wrapper-plan` | +| `tests/wrapper/fortran/scalars/test_fortran_enums.py::test_fortran_enums_preserve_integer_runtime_surface[*]` | source/generated-.pyi parity or parametrized route | scalar inputs/results; module variables/state | `wrapper-plan` | +| `tests/wrapper/fortran/scalars/test_fortran_enums.py::test_fortran_enums_preserve_values_in_generated_pyi_contract` | direct wrapper/build route | scalar inputs/results; module variables/state | `wrapper-plan` | +| `tests/wrapper/fortran/scalars/test_scalar_boundary_plan.py::*` | scalar-only copied native routines with deliberate legacy/direct-plan parity | primitive scalar kinds; value and `Addr(Arg(i))` inputs; hidden output; copy-in/copy-out; rank-zero storage; raw `Addr(T)`; native slot reordering; direct-plus-hidden result tuple assembly | `wrapper-plan` | +| `tests/wrapper/fortran/scalars/test_scalar_generated_pyi_contracts.py::*` | non-generating: generated semantic .pyi fixture parity | semantic .pyi generation/parsing | `not-applicable` | +| `tests/wrapper/fortran/scalars/test_scalar_kinds.py::*` | source/generated-.pyi parity or parametrized route | scalar inputs/results | `wrapper-plan` | +| `tests/wrapper/fortran/scalars/test_value_and_bind_c.py::*` | source/generated-.pyi parity or parametrized route | scalar inputs/results; native-call projections | `wrapper-plan` | +| `tests/wrapper/fortran/scalars/test_verified_baseline.py::test_fmath_scalar_sources_use_canonical_wrapper_plan[*]` | canonical production plan route using the existing fixed- and free-form generation units | scalar inputs/results; native-call projections; Python namespaces; build/artifact integration | `wrapper-plan` | +| `tests/wrapper/fortran/scalars/test_verified_baseline.py::test_f90_array_wrapper_distinguishes_contiguous_and_strided_contracts[*]` | source/generated-.pyi parity or parametrized route | scalar inputs/results; ordinary arrays | `wrapper-plan` | +| `tests/wrapper/fortran/scalars/test_verified_baseline.py::test_f90_wrapper_pipeline_builds_importable_extension[*]` | source/generated-.pyi parity or parametrized route | scalar inputs/results | `wrapper-plan` | +| `tests/wrapper/fortran/scalars/test_verified_baseline.py::test_fortran_array_wrapper_pipeline_matches_fmath_results_with_contiguous_arrays[*]` | source/generated-.pyi parity or parametrized route | scalar inputs/results; ordinary arrays | `wrapper-plan` | +| `tests/wrapper/fortran/scalars/test_verified_baseline.py::test_fortran_wrapper_pipeline_builds_importable_extension[*]` | source/generated-.pyi parity or parametrized route | scalar inputs/results | `wrapper-plan` | +| `tests/wrapper/fortran/scalars/test_verified_baseline.py::test_required_array_buffers_use_canonical_wrapper_plan` | reduced semantic `.pyi` entry over the existing `fmath_arrays_f90` native unit | required rank-one dense buffers; exact dtype/rank/order/alignment/writeability; zero length; native-handle actuals deferred to Phase 7 | `wrapper-plan` | +| `tests/wrapper/fortran/strings/test_character_arguments.py::test_edited_modern_string_contract_wraps_full_axis_spelling_set` | edited semantic `.pyi` contract | strings; fixed/assumed inputs; arrays; mutable string storage | `wrapper-plan` | +| `tests/wrapper/fortran/strings/test_character_arguments.py::test_legacy_fortran_character_arguments_and_results[*]` | production plan route from source/generated-.pyi parity | fixed-form strings; fixed/assumed inputs; fixed results | `wrapper-plan` | +| `tests/wrapper/fortran/strings/test_character_arguments.py::test_modern_fortran_character_arguments_and_results[*]` | source/generated-.pyi parity | strings; fixed/assumed inputs; fixed/deferred results; arrays; writeback | `wrapper-plan` | +| `tests/wrapper/fortran/strings/test_character_arguments.py::test_deferred_allocatable_string_results_use_canonical_plan` | canonical reduced scalar descriptor result contract | runtime length; nullable copy-out; UTF-8 data; allocation failure | `wrapper-plan` | +| `tests/wrapper/fortran/strings/test_character_arguments.py::test_deferred_character_array_handles_use_canonical_plan` | canonical reduced descriptor-result and projected-descriptor contract | hidden/direct owned deferred-character arrays; runtime `S3`/`S4`/`S5` width; projected identity; nullable rank-zero result | `wrapper-plan` | +| `tests/wrapper/fortran/strings/test_character_arguments.py::test_fixed_width_character_arrays_use_canonical_plan` | reduced semantic `.pyi` entry over the existing `fstrings_f90` native unit | fixed-width `NPY_STRING` array itemsize; rank/dtype/zero-size validation; native-handle actuals deferred to Phase 7 | `wrapper-plan` | +| `tests/wrapper/fortran/strings/test_character_arguments.py::test_raw_fixed_width_character_arrays_use_canonical_plan` | reduced edited semantic `.pyi` entry over the existing `fstrings_f90` native unit | raw fixed-width character array address; literal shape; element length; integer-only conversion | `wrapper-plan` | +| `tests/wrapper/fortran/strings/test_character_arguments.py::test_required_scalar_string_inputs_use_canonical_plan` | reduced semantic `.pyi` entry over the existing `fstrings_f90` native unit | required fixed/assumed scalar string inputs; default/kind-1/`c_char`; UTF-8 length and NUL validation; scalar results | `wrapper-plan` | +| `tests/wrapper/fortran/strings/test_character_arguments.py::test_fixed_string_results_use_canonical_plan` | reduced semantic `.pyi` entry over the existing `fstrings_f90` native unit | direct fixed string results; trailing blanks; default/`c_char`; allocation failure | `wrapper-plan` | +| `tests/wrapper/fortran/strings/test_character_edge_cases.py::test_fortran_character_edge_cases_follow_copy_in_copy_out_policy[*]` | production plan route from source/generated-.pyi parity | strings; fixed/assumed input/output; optional presence; Unicode/NUL handling | `wrapper-plan` | +| `tests/wrapper/fortran/strings/test_character_edge_cases.py::test_fixed_hidden_string_output_uses_canonical_plan` | reduced semantic `.pyi` entry over the existing `fcharacter_edges_f90` native unit | fixed hidden string output; trailing blanks; allocation failure | `wrapper-plan` | +| `tests/wrapper/fortran/strings/test_character_edge_cases.py::test_fixed_string_replacement_and_identity_use_canonical_plan` | reduced edited semantic `.pyi` entry over the existing `fcharacter_edges_f90` native unit | fixed immutable replacement and discarded identity; exact length; trailing blanks; allocation failure | `wrapper-plan` | +| `tests/wrapper/fortran/strings/test_character_edge_cases.py::test_assumed_and_optional_string_replacements_use_canonical_plan` | reduced semantic `.pyi` entry over the existing `fcharacter_edges_f90` native unit | assumed-length and optional immutable replacement; empty/omitted/`None`/concrete states; NUL rejection; concrete-only allocation failure | `wrapper-plan` | +| `tests/wrapper/fortran/strings/test_string_generated_pyi_contracts.py::*` | non-generating: generated semantic .pyi fixture parity | semantic .pyi generation/parsing | `not-applicable` | + +## Incremental Protocol + +For each lane: + +1. Select and run an existing passing `tests/wrapper` generation unit through + the legacy route. Retain and inspect its complete generated artifact set. +2. Trace the current lowering, binding, bridge, node/API-model, printer, + runtime-helper, and build paths that produced those artifacts, and record the + observed behavior and existing tests that make them the migration baseline. +3. Expand this checklist with the lane's exact scope, exclusions, source-path + baseline, required backend behavior, plan fields, and validation invariants. +4. Complete every policy field required by the lane in post-IR policy + completion; do not start its planner while semantic decisions remain + scattered or implicit. +5. Implement the lane's hierarchical plan records, planner visitors, + ABI/handoff specs, generator-owned structural checks, directly named backend + lowering methods, source-printer support, and support-report coverage. +6. Implement the minimum dependency-closed backend slice in + `x2py.wrapper_codegen`: copy small suitable pieces, rewrite oversized legacy + classes as minimal equivalents, and add only the intermediate tests required + by the contract above. +7. Generate the plan from policy-completed semantic IR, invoke the directly + named binding and bridge lowering methods, assemble complete backend modules, + and print complete internal artifacts. +8. Compare the new generated artifacts with the retained legacy artifacts and + explain every material difference before compilation. +9. Compile the internal artifacts before changing production route selection. +10. Run the same eligible existing fixtures and assertions through both routes + and compare compiled runtime behavior, failure paths, native-call mapping, + and artifact requirements. +11. Extend the whole-module support predicate so a generation unit uses the + wrapper-plan route only when all its elements belong to completed lanes. + Keep the old route for generation units containing unsupported lanes. +12. Update every affected `tests/wrapper` migration-matrix row and mark the lane + complete only when the reused parity tests pass and every intentional + difference from the baseline is separately documented. + +Do not start a later lane by guessing. Each lane must define the handoff specs +and consistency checks it needs. + +## Mandatory Expansion Gate For Broad Phases + +Phases 5 through 10 are roadmap envelopes, not complete implementation +checklists. Before implementation starts on one of them, update this file and +split that phase into dependency-ordered sub-lanes. The expansion must be based +on an audit of the live semantic models, completed policies, existing +bridge/binding behavior, decorators, and focused wrapper tests. + +Each expanded sub-lane must state: + +- the exact included and excluded semantic cases; +- the completed-policy fields it consumes and any decisions that still need to + move into post-IR policy completion; +- plan records, action keys, handoff specs, native-call slots, lifecycle phases, + and required generated artifacts; +- binding and bridge handler names and which backend-local helper values they + may create; +- validation invariants across Python input/result, binding handoff, bridge + handoff, native call, writeback, cleanup, ownership, and release; +- the whole-module support-predicate change that makes the sub-lane eligible; +- existing `tests/wrapper` nodes that cover the sub-lane, their migration-matrix + status changes, and dual-route parity evidence against the legacy route; +- the exact legacy source path and consumed behavior for every isolated + primitive, whether it is copied or rewritten, its minimal dependency closure, + baseline evidence, and a reason for every behavior with no legacy source; +- dependencies on earlier lanes and the legacy behavior that can be removed + when the sub-lane is complete. + +Do not mark a broad phase complete from its current envelope items. Mark its +expanded sub-lanes individually, then close the phase only after all live cases +in its audited support matrix are either migrated or explicitly removed from +the product contract. + +## Dependency-Ordered Checklist + +### Foundation and semantic authority + +- [x] Establish the isolated `x2py.wrapper_codegen` package boundary and + visitor infrastructure. +- [x] Complete the first primitive lane in general wrapper policy before + planning, including native-call order, result projection, ownership, and + lifecycle facts. +- [x] Build editable `ModulePlan`, `FunctionPlan`, transfer, result, ABI, + native-slot, and lifecycle records from completed wrapper policy. +- [x] Refactor each cross-boundary owner into explicit binding and bridge child + plans, including module/function/result/lifecycle scope as well as arguments. +- [x] Remove plan-owned method names and handler registries; add typed + datatype-family facts required by direct lowering. +- [x] Keep structural plan validation private to `WrapperCodeGenerator`, with + no planner-time validation or standalone validator class, and verify every + listed invariant after direct plan edits. + +### Direct generator boundary + +- [x] Change `WrapperCodeGenerator.generate` to consume only `ModulePlan`, + freeze it, validate it, validate lowering support, recursively generate + backend nodes, print them, and return artifacts directly. +- [x] Implement recursive `CBindingGenerator` synthesis of complete C modules, + headers, and functions from plan nodes. +- [x] Implement recursive `FortranBridgeGenerator` synthesis of complete + Fortran modules and functions from plan nodes. +- [x] Replace plan-selected method names with directly named backend lowering + methods selected by the visible `_lower_{subject}_{action.value}` rule. +- [x] Ensure backend node printers, artifact construction, and build + consumption retain their distinct freezing boundaries. + +### Scalar parity and route evidence + +- [x] Replay the existing scalar source and semantic-`.pyi` baseline through + the direct generator and compare generated artifacts and runtime behavior. +- [x] Update the staged walkthrough to use only plan editing and the public + generator boundary. +- [x] Retire superseded internal representations, their package exports, + orchestration, validation, documentation, and focused tests. +- [x] Run focused wrapper-codegen and pipeline tests; the walkthrough for both + supported entry choices where practical; `tests/wrapper` excluding LAPACK; + documentation checks; `git diff --check`; the required static-analysis suite; + and `tools/check_wrapper_codegen_complexity.py`. + +## Phase 3 — Scalar Inout, Optional, And Descriptor-Like Scalars + +Scope: scalar copy-in/copy-out, optional arguments, present-but-null descriptor +values, and scalar allocatable/pointer descriptor boundaries. + +Phase 3 legacy replay audit: + +- `foptional_fixed.f` uses one nullable value pointer at the Bind-C ABI. + Omission and explicit `None` both pass a null pointer because both mean that + the ordinary optional dummy is absent; a concrete scalar uses call-local + storage, and the bridge branches on `c_associated(...)` before calling the + native function with or without the optional keyword. +- The existing optional allocatable-scalar contract uses two independent ABI + pointers. The value pointer is null for explicit `None`, while the presence + pointer is non-null for both `None` and a concrete value. Omission leaves both + null. The bridge therefore distinguishes absent, present-unallocated, and + present-with-value states without inferring presence from the value pointer. +- Immutable scalar replacement uses copy-in storage, native mutation of that + storage, copy-out to a new Python scalar, and scope-owned stack cleanup. The + caller's original NumPy scalar remains unchanged. The audit found no existing + runtime wrapper test for this primitive-scalar `Returns["argument", T]` + contract, so `test_scalar_writeback_plan.py` is the recorded coverage-gap + fixture for this lane. +- The first legacy replay of that coverage gap exposed a duplicate declaration + of the mutable scalar result. The legacy bridge now promotes the copy-in + temporary to the Bind-C function result and removes it from the ordinary + local-declaration set. Both routes compile, and incompatible Python values + fail with `TypeError` before the native call. Stack temporaries and local + allocatable descriptors require no explicit release action; their procedure + scope owns cleanup on normal return. + +The Phase 3 plan records optional mode, nullable value and presence handoffs, +and four ordered scalar replacement phases: `copy_in`, `native_mutation`, +`copy_out`, and `cleanup`. Generator preflight requires the complete phase set, +an existing source handoff, the correct binding/bridge owner for each phase, +and a Python result target for copy-out. Forced whole-module route selection +accepts these completed lanes after the dual-route evidence below. Automatic +production selection still remains on the legacy route under the independent +GIL parity deferral recorded for Phase 2D. + +- [x] Audit and record the legacy copy-in/out, optional presence, nullable + scalar descriptor, cleanup, and failure-path behavior for this lane. +- [x] Add or rewrite only the additional optional/descriptor nodes, API + primitives, local-state helpers, and printer cases required by this lane, + with baseline tests. +- [x] Represent copy-in, native mutation, copy-out, and cleanup as explicit + writeback phases. +- [x] Preserve the three-state optional rule: omitted argument, explicit `None`, + and present concrete value are distinct when the native ABI needs them. +- [x] Represent scalar descriptor presence tokens and nullable value handoffs in + the plan. +- [x] Validate that a writeback consumes an existing binding/bridge handoff and + writes to a Python-visible target or result slot. +- [x] Emit and print complete inout/optional/descriptor-capable modules + internally, then compile and compare both routes for all three presence + states, mutation, writeback, cleanup, ABI, and failures. +- [x] Widen whole-module route eligibility to this lane only after parity, and + complete it before moving arrays or handles to the plan path. + +## Phase 4 — Scalar Module Variables + +Scope: scalar module variables. Derived-type fields remain in Phases 8 and 9 +because their wrapper instance, owner, and property lifecycle must already be +represented before field access can use the plan route. + +Phase 4 legacy replay audit: + +- `fmodule_vars_f90.f90` establishes the ordinary scalar state contract. Its + legacy bridge emits value-returning getters and value-argument setters; + binding accessors run with the GIL held, aliases route to the same native + storage, deletion fails, and contract initializers call the native setter at + import. A `parameter` is instead copied into the Python module dictionary, so + rebinding it is local to that module object and never mutates native storage. + This whole source remains legacy because it also owns `rgb_color` and derived + module objects from later phases. +- The scalar subset of `fscalar_descriptors_f90` establishes nullable + allocatable and pointer reads. The legacy bridge returns null for absent + storage or allocates and copies one detached scalar; the binding converts the + copy, frees it, and rejects descriptor replacement. Its whole source cannot + migrate in Phase 4 because it also contains nullable snapshot-result forms + from a later lane. Allocation failure is deliberately injected with + `X2PY_WRAPPER_FAIL_ALLOC` and preserves the legacy null/`None` surface. +- No existing generation unit contained only the already completed scalar + function lanes plus every Phase 4 getter, setter, constant, descriptor, + initialization, reload, and failure behavior. The bounded + `test_scalar_module_variable_plan.py` whole-module fixture records that + coverage gap; it contains no strings, arrays, classes, or later-phase owner. + +The plan keeps only completed typed facts: Python names, getter and setter +actions, initializer or constant value, datatype family, native name/module, +native assignment, descriptor kind, and handoff roles. Both backends invoke +directly named lowering methods with matching subject/action suffixes wherever +their behavior is shared; datatype and descriptor facts stay method inputs. +The generator validates the complete frozen plan before either backend emits +anything, including binding/bridge getter agreement and the rule that a Python +write-through setter must have a compatible bridge setter role. Forced +whole-module selection now accepts `scalar-module-variables`; automatic +production selection remains independently deferred by the Phase 2D GIL gate. + +- [x] Audit and record the legacy scalar module-variable getter, setter, + rejected replacement, module initialization, and attribute-routing behavior. +- [x] Add or rewrite only the additional module/type nodes, getter/setter API + primitives, initialization nodes, and printer cases required by this lane, + with baseline tests. +- [x] Represent getter behavior, setter exposure, native setter assignment, and + rejected replacement behavior in module-variable plans. +- [x] Add binding actions for Python attribute get/set around scalar values. +- [x] Add bridge actions for scalar module-variable read/write. +- [x] Validate getter/setter pair consistency: a Python setter cannot exist + without a compatible bridge setter handoff. +- [x] Keep ordinary Python module-name rebinding semantics separate from native + module-variable storage. +- [x] Emit and print complete module-variable-capable modules internally, then + compile and compare both routes for get/set behavior, rejection paths, + initialization, cleanup, ABI, and generated artifacts. +- [x] Widen whole-module route eligibility to scalar module variables only after + that parity evidence passes. + +### Phase 3/4 whole-unit namespace correction + +The post-Phase 4 review found that scalar lowering itself matched the legacy +route, but the parity helper unwrapped a sole native child module before making +assertions. That hid a public-surface difference: the legacy route retained +Fortran modules as Python child namespaces while the plan route flattened their +members at the extension root. The support analyzer also accepted colliding +procedures from separate native modules and allowed the failure to reach the +Fortran compiler. + +This correction is part of the completed scalar foundation rather than a new +datatype lane: + +- [x] Add a concise `NamespacePlan` beneath `ModulePlan`; place functions and + variables in namespace nodes instead of flattening them into the module. +- [x] Complete Python export paths in post-IR export policy, including + namespace-local keyword and collision fixes, then mechanically group plan + owners by those paths without reconstructing namespace policy in either + backend. +- [x] Generate root, child, and nested Python modules while keeping native + module imports and generated bridge symbols unambiguous. +- [x] Support ordinary scalar subroutines with no projected result through the + existing native call plus Python `None` result path. +- [x] Reject duplicate Python exports and generated symbols before either + backend emits source. +- [x] Use one visible lowering naming rule in both backends: + `_lower_argument_`, `_lower_result_`, + `_lower_writeback_`, `_lower_module_getter_`, + and `_lower_module_setter_`. Do not store method-name strings + in the plan or hide these selections in backend dictionaries. +- [x] Remove scalar prefixes from general wrapper concepts, including the + function, argument, result, native-slot, lifecycle, node, and printer policy + surfaces. Retain scalar naming only for permanently scalar-specific ABI type + facts and actions. +- [x] Update the staged walkthrough to print the namespace tree, typed actions, + and the directly corresponding binding and bridge method names. +- [x] Compile both routes from the existing complete + `contract_mixed_module_external.f90`, `contract_import_graph.f90`, + `contract_multi_module.f90`, `contract_standalone_only.f90`, and + `contract_same_name.f90` fixtures; compare the real extension root and child + namespaces without `_sole_native_module` normalization. + +## Phase 2D — Native Call Runtime Envelope + +This is the next dependency-closed migration lane. Complete it before Phase 5 +so the already proven scalar generation units can move from temporary +`dual-route` evidence to production `wrapper-plan` routing instead of adding +more datatype lanes behind the same runtime gate. + +Scope: the binding-owned runtime envelope around an otherwise completed native +call. This phase includes default GIL release, explicit `@hold_gil`, and native +status/message projection through `@raises(...)`. Status projection is included +because the existing `fruntime_policy_f90` generation unit tests it together +with both GIL modes and whole-generation-unit routing cannot split that module. + +Excluded from this phase: + +- strings, arrays, descriptors, derived types, and callbacks, which remain in + their datatype or callback phases; +- callback re-entry and callback exception/abort behavior, which remain in + Phase 10; +- OpenMP array execution and Makefile-specific behavior, which remain blocked + by the array and cross-cutting build lanes; +- general Python exception translation that is not selected by a completed + native status policy. + +The existing legacy/runtime oracle is +`tests/wrapper/fortran/runtime_behavior/test_runtime_policies.py` in both its +source-driven and semantic-`.pyi`-driven forms. It proves that an ordinary +native pause releases the GIL, `@hold_gil` keeps it held, successful status +returns produce the declared Python result, failing status returns raise the +selected exception with the native message, and emitted C places +`Py_BEGIN_ALLOW_THREADS` / `Py_END_ALLOW_THREADS` only around eligible native +calls. `test_recursive_native_runtime_calls` is the scalar regression unit to +run after the call envelope works. Do not use the OpenMP or callback fixtures +as the first parity unit. + +The plan and generators must follow these boundaries: + +- Post-IR policy completion owns `hold_gil` and the complete native status + error decision, including status source, message source, success value, and + Python exception kind. The planner only projects those completed facts. +- Keep the runtime facts concise and function-owned. Extend the existing + binding-facing function plan rather than adding a second function plan or a + backend dispatcher table. The bridge continues to lower native call slots + and result storage mechanically; it does not decide GIL or Python exception + policy. +- Argument parsing and conversion, Python result construction, status/message + conversion, exception creation, writeback, and Python-owned cleanup always + run with the GIL held. For the default policy, release the GIL immediately + before the bridge call and reacquire it immediately after that call. For + `hold_gil=True`, emit no release region. +- Perform status evaluation and raise the selected Python exception only after + the GIL has been reacquired. Validate before emission that every status and + message projection names an existing native result slot with a compatible + completed handoff. +- Use directly named lowering methods that follow the existing visible naming + rule. Do not infer runtime policy from result types, function names, emitted + locals, or the presence of status-like native arguments. + +The completed legacy audit found one binding-owned envelope in both oracle +builds. The legacy binding parsed and converted Python inputs with the GIL +held, emitted `Py_BEGIN_ALLOW_THREADS` immediately before the bridge call and +`Py_END_ALLOW_THREADS` immediately after it by default, and omitted both +macros for `@hold_gil`. Only after reacquiring the GIL did it convert hidden +status/message outputs, compare status with `success`, construct +`RuntimeError`, suppress those policy outputs from the declared Python result, +and decref converted result objects on both the failure and success paths. A +missing or incompatible status/message name was previously rediscovered from +raw decorator dictionaries and result datatypes in `ir2ast` and the legacy C +binding; Phase 2D moved that decision to typed post-IR completion and left the +legacy route as a dispatch consumer for rollback parity. + +The direct plan route now preserves that ordering with explicit released-call +and held-call lowering methods. Its fixed native message handoff is +bridge-owned null-terminated storage that the binding converts and frees after +the GIL is reacquired. Generated symbol spelling differs from the legacy +artifacts, but the same source and edited-`.pyi` concurrency, exception, +cleanup, and runtime assertions pass. Production cutover also reused the +existing rendered-artifact build path for inferred native module include +directories, native library directories, `.pyi` manifests, verbose timing, +and scalar external explicit interfaces; no legacy retry was added. + +- [x] Audit and record the exact legacy GIL release/hold region, status/message + projection, exception construction, result suppression, cleanup, and failure + behavior from both existing runtime-policy tests. +- [x] Complete the native status error decision in post-IR policy before + planning; retain the already completed `hold_gil` fact as its single source + of truth. +- [x] Extend the concise function plan with only the binding-facing runtime + facts needed for GIL and status-error lowering, and validate all referenced + native result slots before either backend emits source. +- [x] Add direct binding lowering for the released-call and held-call envelopes + plus post-call status projection. Keep the bridge call and result-slot + lowering on their existing paths. +- [x] Replay both source and semantic-`.pyi` forms of + `test_runtime_policies.py` through legacy and wrapper-plan routes using the + same concurrency, exception, artifact, and generated-C assertions. +- [x] Run `test_recursive_native_runtime_calls` through the wrapper-plan route + as the scalar recursion regression; leave OpenMP and callbacks in their + later lanes. +- [x] After dual-route parity passes, remove the blanket Phase 2D production + deferral. Let whole-generation-unit support select `wrapper-plan` only for + units whose feature lanes are complete; do not add fallback or per-function + mixed routing. +- [x] Move the eligible scalar matrix rows from `dual-route` or `legacy` to + `wrapper-plan`, update the live route counts, and prove their default builds + no longer invoke `semantic_ir_to_codegen_ast()`. +- [x] Finish this phase only when the production `wrapper-plan` count is + nonzero and the already completed scalar baseline no longer depends on the + legacy route outside deliberate rollback diagnostics. + +## Phase 2E — Scalar Boundary Completion and Test Isolation — Complete + +Complete the scalar public boundary before stopping this migration lane. This +phase does not begin strings or arrays. It separates scalar evidence from +mixed generation units so whole-unit routing cannot hide whether one scalar +policy is implemented. + +Scope: every supported primitive scalar kind; ordinary Python scalar values; +`Addr(Arg(i))` call-local address projection; projected scalar copy-in/copy-out; +caller-owned rank-zero NumPy storage spelled `T[()]`; caller-supplied integer +raw addresses spelled `Addr(T)`; and visible or hidden scalar `in`, `out`, and +`inout` behavior. For a mixed native fixture whose declarations cannot be +safely sliced, add a small distinctly named scalar-only native test routine +that preserves the policy decision under test. + +The boundary contract remains: + +- `T` accepts a Python/NumPy scalar value. When native code only reads it, the + wrapper converts into call-local storage. When native code writes through an + address projection and the contract projects `Returns["name", T]`, the + wrapper performs copy-in, native mutation, and copy-out to a replacement + Python scalar; the caller's immutable scalar object is not mutated. +- `T[()]` accepts a rank-zero NumPy array with exactly the declared dtype. The + wrapper validates caller storage and passes its data address; native `out` or + `inout` mutation remains visible in that same array and the Python call + returns `None` unless the contract declares another result. +- `Addr(T)` accepts an integer address such as `array.ctypes.data`. The wrapper + converts it to a raw pointer and forwards that same address without copying + or owning the pointee. Mutation is therefore observed through caller-owned + storage. +- `@native_call(...)` controls only native slot order and value/address/result + projection. It does not change which Python representation (`T`, `T[()]`, or + `Addr(T)`) the declared argument accepts. +- Use one necessary-copy rule. For interoperable scalar replacement, the + binding's converted C scalar is the copy-in storage and the bridge passes + that same storage directly to the native routine; after mutation the binding + converts it once to the Python replacement. `c_f_pointer` association for + `T[()]` or `Addr(T)` is not a data copy. A bridge-local data copy is allowed + only when the native representation actually changes, such as descriptor, + string-buffer, or ownership-snapshot construction. +- Enforce that rule with a completed `BridgeDataAction` on every argument, + result, and native-call output slot. `DIRECT_TRANSFER` reuses boundary + storage, `ASSOCIATE_VIEW` may create only a non-owning native view, + `COPY_REPRESENTATION` is the sole bridge data-copy permission and requires a + non-empty policy reason, and `BLOCKED` keeps the whole generation unit off + the plan route. A non-copying action carrying a copy reason is also invalid. + New array, string, or object support must complete this fact before route + eligibility is widened. + +Excluded: simultaneous multiple-result tuple assembly; rank-positive arrays; +strings including fixed status buffers except for already completed Phase 2D +status projection; derived types; callbacks; and any compatibility fallback to +the legacy generator. + +- [x] Record scalar-only tests separately from mixed array/string/derived + generation units in the route ledger; use copied minimal native routines + when fixture declarations are coupled. +- [x] Cover every primitive scalar kind exercised by the scalar runtime suite + through the direct registry and both binding/bridge generators. +- [x] Add direct named binding and bridge lowering for rank-zero numeric/logical + storage using the completed `SCALAR_STORAGE` and `PASS_STORAGE_ADDRESS` + decisions; validate dtype, rank zero, and writability before the native call. +- [x] Add direct named binding and bridge lowering for primitive raw addresses + using the completed `RAW_ADDRESS` and `PASS_RAW_ADDRESS` decisions; accept an + integer address and forward it without copy or ownership inference. +- [x] Prove isolated scalar input, hidden output, copy-in/copy-out `inout`, + caller-storage `out`/`inout`, and raw-address `out`/`inout` behavior through + compiled legacy/direct-plan parity where applicable. +- [x] Prove scalar copy-in/copy-out reuses one binding local and does not add a + redundant bridge-local value copy. +- [x] Prove plan validation rejects an unexplained bridge copy, a copy reason + on a non-copying path, and any still-blocked bridge data action. +- [x] Prove isolated `@native_call` argument mapping, including `Addr(Arg(i))` + and hidden `Return(...)` slots, without arrays determining route selection. +- [x] Move only proven scalar-only nodes to `wrapper-plan`, update collected + route counts, and leave the original mixed integration nodes on their real + datatype blockers. + +## Phase 2F — Multiple Scalar Result Assembly — Complete + +This is result aggregation, not another scalar boundary representation. The +first isolated oracle is the `with_scalar` policy from +`test_output_arguments.py`: one direct primitive scalar function return plus +one hidden primitive scalar output, assembled into a Python tuple in declared +result order. Keep it separate from arrays, strings, derived types, and native +handles before widening the plan route. + +For source-derived contracts, an ordinary non-descriptor `intent(out)` scalar +hidden by Python result projection still selects `PASS_CALL_LOCAL_ADDRESS` +even when no edited `.pyi` `Addr(...)` spelling exists. The hidden-result +projection is itself the completed semantic fact that requires writable +call-local native storage; the binding and bridge must not rediscover that ABI +rule. Rank-zero allocatable/pointer descriptor outputs retain the distinct +Phase 7H descriptor transport and are not rewritten as ordinary addresses. + +The completed representation is an ordered `FunctionWrapperPolicy.results` +tuple and an ordered `FunctionPlan.results` tuple. Each Python-visible result +has its own `ResultPolicy` and `ResultPlan`, including its binding consumer and +`result_position`. A direct native function return has +`source_kind="direct_return"` and no native-call slot. A hidden output has +`source_kind="hidden_output"` and references the exact same mutable +`NativeCallSlotPlan` stored in `FunctionPlan.native_call_slots`. The bridge +uses the sole direct result, when present, to select its function result and +passes every hidden result through its completed output-address slot. It does +not assemble Python results. + +After the native call, the binding converts each result from its completed +source role exactly once. One result is returned directly; two or more are +assembled into a Python tuple in ascending `result_position`. Tuple allocation, +reference transfer, and failure cleanup are binding-local emission details, +not semantic policy. Before either backend emits source, validation requires +result positions to cover `0..N-1` exactly once, at most one direct result, +every hidden result to share its function native-call slot, and every +non-status native output slot to have exactly one binding result consumer. +Phase 2F does not combine these consumers with projected argument writeback; +that broader aggregation remains blocked until it receives its own completed +policy. + +- [x] Add a scalar-only copied native routine and contract for a direct return + plus hidden scalar `Return(...)` slot. +- [x] Represent every Python result as an explicit binding consumer while + preserving the bridge's direct-return and output-address ABI roles. +- [x] Validate contiguous result positions and reject unclaimed outputs before + either backend emits source. +- [x] Prove compiled legacy/direct-plan parity, then update the route counts. + +## Phase 5 — Strings + +Scope: non-descriptor scalar character values, fixed-length strings, assumed- +length call inputs, immutable replacement, mutable rank-zero byte storage, and +raw fixed-length character addresses. Character arrays remain in Phases 6 and +7; allocatable or pointer scalar character values remain in Phase 7; character +fields remain in Phases 8 and 9; character callbacks remain in Phase 10. + +The legacy wrapper is the behavioral oracle for this phase. In particular, +`CPythonBindingGenerator._convert_python_string_value_argument()` and +`_convert_python_string_storage_argument()` define Python conversion, +validation, allocation, and writeback behavior, while +`FortranToCBridgeGenerator._build_string_argument()`, +`_build_string_storage_argument()`, `_convert_raw_string_argument()`, and +`_convert_string_result()` define the bridge representation. The public +contract and observable oracle are +`docs/user/guide/fortran-wrapper.md`, `docs/user/guide/data-types.md`, +`docs/user/reference/semantic-pyi-format.md`, +`tests/wrapper/fortran/strings/test_character_arguments.py`, and +`tests/wrapper/fortran/strings/test_character_edge_cases.py`. Direct-plan +lowering may use different temporary names or an equivalent internal C ABI, +but it must preserve the legacy Python behavior, native argument order, +character payload, length, ownership, cleanup, and result projection. + +Strings use the same completed-policy and planning pipeline as the other +rank-zero scalar families: + +```text +ArgumentPolicy -> ArgumentTransferPlan -> NativeCallSlotPlan +ResultPolicy -> ResultPlan +LifecyclePolicy -> LifecycleActionPlan +``` + +Do not add a parallel string plan hierarchy or plan-owned handler names. +Numeric and logical primitive families share registry-backed lowering because +their generated structure is the same. `DatatypeFamily.STRING` dispatches to +its own directly named binding and bridge lowering methods because character +conversion and ABI structure differ. The existing `STRING_VALUE`, +`STRING_STORAGE`, `PASS_CALL_LOCAL_ADDRESS`, `PASS_STORAGE_ADDRESS`, +`PASS_RAW_ADDRESS`, and generic codegen/lifecycle actions remain authoritative; +add a new typed action only if those completed actions cannot identify a real +semantic choice. + +Every string argument plan records the completed fixed positive character +length or the absence of a fixed length. The binding-to-bridge handoff records +both the payload address and encoded payload length when the bridge needs both; +this is an ABI fact in the existing argument transfer, not a new planning +stage. A fixed `String[n]` Python value must encode to exactly `n` bytes. A +plain `String` input carries its runtime UTF-8 byte length. Embedded NUL is +rejected before the native call. The bridge may copy bytes into Fortran +character storage only when `BridgeDataAction.COPY_REPRESENTATION` and its +non-empty policy reason were completed before planning. + +The phase is split into the following dependency-ordered sub-lanes. + +### Phase 5A — Required Read-Only String Values + +Included: required rank-zero `String[n]` and `String` Python `str` inputs; +default character, kind `1`, and `c_char`; fixed-length exact encoded-byte +validation; assumed-length runtime payload size; embedded-NUL rejection; and +primitive scalar or void results already supported by earlier phases. + +Excluded: writable inputs, projected replacement, optional strings, string +results, mutable `String[n][()]` storage, raw `Addr(String[n])`, arrays, +allocatable/deferred results, fields, and callbacks. + +Completed policy must provide `ObjectKind.STRING`, +`PythonBarrierAction.STRING_VALUE`, +`NativeBarrierAction.PASS_CALL_LOCAL_ADDRESS`, +`CodegenAction.CALL_LOCAL_INPUT`, `StorageMode.STACK`, required presence, +`BridgeDataAction.COPY_REPRESENTATION`, and the reason that C UTF-8 bytes are +materialized as Fortran character storage. Planning projects those facts into +the ordinary argument/native-slot records. The C binding method +`_lower_argument_required_string_value()` validates `str`, extracts UTF-8 plus +byte length, rejects embedded NUL, and enforces a fixed length when present. +The Fortran bridge method `_lower_argument_required_string_value()` receives +the payload address and length, associates a byte view, copies it into one +backend-local character temporary, and passes that temporary in the completed +native-call position. + +Validation requires a string-value Python action, call-local-address native +action, character-buffer handoff, one matching payload-length role, required +presence, no projected result, and a justified representation copy. Whole-unit +eligibility widens only for generation units containing this lane plus already +completed scalar/result/runtime lanes. Replay uses the existing +`fstrings_f90` native object and contract package with a reduced entry that +exports only existing read-only scalar string procedures; both routes run the +same fixed/assumed-length, kind, NumPy-string-scalar, wrong-length, and embedded +NUL assertions. The mixed original string nodes remain `legacy` because their +units also contain string results, writable strings, arrays, and allocatables. + +- [x] Complete Phase 5A policy, ordinary plan projection, validation, named C + and Fortran lowering, reduced-entry dual-route runtime parity, support + predicate, and migration-ledger evidence. + +### Phase 5B — Fixed-Length String Results And Hidden Outputs + +Included: direct fixed-length scalar character results and fixed-length hidden +`intent(out)` results, including trailing blanks. The binding receives a +NUL-terminated C-owned copy, converts the full payload to a Python-owned +`str`, and releases the temporary exactly once. The bridge allocates and fills +that copy only through completed `COPY_REPRESENTATION` policy. Deferred-length +and nullable allocatable or pointer results remain in Phase 7 because their +runtime length and allocation state are descriptor lifecycle facts, not +scalar-string conversion facts. + +Both forms reuse the ordinary ordered `ResultPolicy -> ResultPlan` path and +record the fixed positive `character_length` on the result. A direct native +function result has `source_kind="direct_return"`, +`CodegenAction.COPY_OUT`, no native-call slot, and a bridge function result of +`type(c_ptr)`. The bridge first receives the native value in backend-local +`character(kind=c_char, len=n)` storage, then allocates `n + 1` bytes through +the existing `x2py_malloc` interface, copies all `n` characters, appends +`c_null_char`, and returns the pointer. A hidden output has +`source_kind="hidden_output"`, `CodegenAction.COPY_OUT`, +`NativeBarrierAction.PASS_CALL_LOCAL_ADDRESS`, and references the exact same +fixed-length `NativeCallSlotPlan` used by the function. Its existing output +slot receives native character storage, then performs the same justified +allocation and copy after the native call. + +In both cases, binding lowering checks for a null allocation, converts the +NUL-terminated UTF-8 payload with the same observable behavior as the legacy +`Py_BuildValue("s", ...)` path, frees the C allocation exactly once even when +Python conversion fails, and returns the Python-owned `str`. Phase 5B supports +exactly one Python-visible string result per function; mixed or multiple +string result aggregation remains blocked until cleanup of every unconverted +native allocation is explicitly planned. A function that combines a public +fixed string result with native status-error projection is blocked for the +same reason: the status failure path must not bypass the string allocation's +planned release. + +Validation requires a fixed positive length, `ObjectKind.STRING`, Python-owned +copy-return ownership, no Python barrier action, the source-appropriate +codegen/native action, `BridgeDataAction.COPY_REPRESENTATION` with the standard +fixed-string copy reason, and matching result/native-slot lengths for hidden +outputs. Direct results must not carry a native-call slot; hidden results must +share their function slot by identity. The C and Fortran backends dispatch +`DatatypeFamily.STRING` to `_lower_result_fixed_string()` methods instead of +the primitive scalar registry. + +Replay direct results from the existing `fstrings_f90` native object with a +reduced contract entry exporting `char_result_default`, +`char_result_c_char`, `string_result_fixed`, `string_result_padded`, and +`string_result_c_char`. Replay the hidden output from the existing +`fcharacter_edges_f90.make_out` unit through another reduced entry. Run the +same trailing-blank and returned-value assertions through legacy and direct +routes. The original mixed nodes remain `legacy` on deferred results, writable +strings, optionality, or arrays. + +- [x] Complete fixed-length direct and hidden string result policy, result-plan + length facts, allocation/failure cleanup, binding conversion, bridge copy, + validation, legacy/direct parity, support widening, and ledger updates. + +### Phase 5C — Immutable String Output And Inout Replacement + +Included: fixed and assumed-length Python `str` output/inout dummies, including +the pass-by-address mutable native call. Python strings remain immutable: the +binding creates mutable call-local storage; the bridge passes that storage to +the native dummy; a declared `Returns["name", String...]` consumer returns a +replacement string; identity form discards native mutation and returns `None`. +Fixed buffers retain their complete post-call contents and trailing blanks; +assumed-length buffers use the encoded input length. Optional omitted, +explicit-`None`, and concrete-value states are handled here after required +replacement works. + +The first Phase 5C slice is required fixed-length `String[n]` only. A projected +replacement consumes completed `ObjectKind.STRING`, Python-owned +`COPY_RETURN`, `PYTHON_REFCOUNT`, stack contract storage, +`PythonBarrierAction.STRING_VALUE`, +`NativeBarrierAction.PASS_CALL_LOCAL_ADDRESS`, +`CodegenAction.COPY_IN_OUT`, native mutation, result projection, and +`BridgeDataAction.COPY_REPRESENTATION` with the fixed-string replacement copy +reason. The ordinary argument plan records those facts plus the fixed positive +character length, and its binding and bridge views both carry the completed +codegen action. The same mutable native-call slot is referenced throughout; +there is no second output slot. + +The binding validates the input exactly as Phase 5A does, allocates one +`n + 1` byte call-local buffer through `x2py_malloc`, copies all `n` encoded +bytes, and appends NUL. Allocation failure raises `MemoryError` before native +execution. The bridge receives the mutable buffer and length, materializes +backend-local `character(kind=c_char, len=n)` storage, passes that storage to +the native dummy, then copies the complete post-call value back into the +binding buffer and restores the NUL terminator. After the call, binding +`_lower_writeback_string()` converts the replacement with the same +`Py_BuildValue("s", ...)` behavior as the legacy route and frees the call-local +buffer exactly once whether conversion succeeds or fails. + +The existing ordered lifecycle records remain authoritative: + +```text +COPY_IN (binding allocation and input copy) + -> NATIVE_MUTATION (bridge-local character call and copyback) + -> COPY_OUT (binding Python replacement conversion) + -> CLEANUP (binding call-local buffer release) +``` + +Validation requires the completed ownership/action facts, fixed result +position, one shared payload/length handoff, matching argument and native-slot +lengths/actions/copy reasons, exactly one complete lifecycle phase set, bridge +copyback ownership only for `COPY_IN_OUT`, and binding cleanup after conversion. +A replacement combined with native status-error projection stays blocked until +the status failure path also releases the mutable buffer. Multiple projected +results remain blocked by the existing single-writeback lane. + +A fixed identity contract uses the already-completed `CALL_LOCAL_INPUT` action, +the same call-local bridge character representation, no lifecycle result, and +returns `None`. Native writes affect only that temporary and are deliberately +discarded. Its binding buffer remains the borrowed read-only UTF-8 input because +the bridge never copies mutation back across the boundary. Assumed-length, +optional, mutable `String[n][()]`, and raw-address forms remain excluded from +this first slice. + +Replay both forms from the existing `fcharacter_edges_f90.fixed_inout` native +unit through a reduced edited contract that exports one projected replacement +and one identity spelling bound to the same native symbol. Run the same exact +length, trailing-blank, input-immutability, returned-value, and allocation +failure assertions through legacy and direct routes before widening the +whole-unit support predicate. + +The second Phase 5C slice keeps the same completed ownership, barrier, +representation-copy, and four-phase lifecycle records while removing the +compile-time-length restriction. For required assumed-length `String`, the +binding-recorded UTF-8 byte length is the native character length and the +replacement allocation size. A zero-byte input is valid: replacement owns a +one-byte NUL-only buffer, the bridge materializes a zero-length character +value, and binding returns the empty Python string after releasing the buffer. +Fixed strings still require the exact declared encoded length. + +Optional string values use the completed `OptionalMode.NULLABLE_VALUE`; they +do not invent a descriptor or reinterpret optionality as semantic nullability. +The binding ABI always carries the string payload pointer and runtime byte +length. Omitted and explicit `None` both send a null pointer with length zero, +so the bridge leaves the native optional dummy absent and a projected +replacement returns `None`. A concrete value is validated before native +execution, including embedded-NUL rejection and any fixed-length constraint. +Projected replacement allocates and owns the mutable `length + 1` buffer only +for that concrete value; identity form borrows the read-only payload and +discards native mutation exactly as the required identity path does. + +The bridge tests pointer association to choose the existing optional native +call branch. Only the present branch associates the payload, creates +`character(kind=c_char, len=runtime_length)` call-local storage, and invokes +the native optional dummy. Copyback is likewise guarded by pointer association, +so an absent optional never touches unassociated storage. Concrete projected +replacement restores the NUL terminator after copying all runtime-length +bytes. Binding then returns the concrete replacement and frees its allocation +exactly once, or returns `None` without calling `free` for the absent states. +Status-error combination and multiple projected replacements remain blocked +by the same explicit cleanup exclusions as the fixed required slice. + +Replay `assumed_inout` and `optional_inout` from the existing +`fcharacter_edges_f90` contract through a reduced entry module. Compare legacy +and direct routes for empty and non-empty assumed-length values, omitted, +explicit-`None`, and concrete optional states, input immutability, embedded-NUL +rejection before native execution, and allocator failure only for concrete +projected replacements. + +- [x] Complete fixed required replacement and discarded-identity policy, + writeback lifecycle, named lowering, cleanup, validation, parity, support + widening, and ledger updates. +- [x] Add assumed-length replacement and optional presence only after the fixed + required path is proven; preserve legacy empty-string and omitted/`None` + behavior and reject embedded NUL before native execution. + +### Phase 5D — Mutable Storage And Raw Fixed-Length Addresses + +Included: `String[n][()]` caller-owned rank-zero NumPy bytes storage and +`Addr(String[n])` caller-supplied integer addresses. The storage path validates +rank zero, dtype `S`, native byte order/alignment where applicable, and +writability before aliasing the caller buffer. The raw-address path does not +own or validate the pointee. Both use the declared fixed length; mutable +deferred-length scalar storage remains blocked. + +Both forms complete policy before planning and share no Phase 5C replacement +lifecycle. `String[n][()]` records `ObjectKind.STRING`, caller ownership, +`IN_PLACE`, caller destruction, alias contract/boundary storage, +`PythonBarrierAction.STRING_STORAGE`, +`NativeBarrierAction.PASS_STORAGE_ADDRESS`, `CodegenAction.IN_PLACE_ARGUMENT`, +native mutation, no result projection, and a fixed positive character length. +`Addr(String[n])` records the same caller ownership, in-place transfer, caller +destruction, mutation, and no result projection, but the contract value itself +uses stack storage while `PythonBarrierAction.RAW_ADDRESS` and +`NativeBarrierAction.PASS_RAW_ADDRESS` preserve the unsafe caller-supplied +address. The raw pointee is never adopted, released, sized, or validated by +x2py. This corrects the pre-5D raw-string decision that incorrectly retained +immutable-string call-local ownership despite the completed raw-address +barriers. + +The ordinary argument plan carries the fixed length and uses +`ArgumentHandoffMode.OPAQUE_ADDRESS` for both forms. There is one pointer ABI +field and no runtime length field: the fixed character length comes only from +the completed plan. The binding storage handler accepts exactly a rank-zero +NumPy `NPY_STRING` array whose itemsize is `n`, requires alignment and +writability, and forwards `PyArray_DATA` without allocating or copying. +The raw handler accepts an integer and uses the existing `PyLong_AsVoidPtr` +path; it deliberately does not inspect the pointee, its allocation extent, or +its lifetime. + +The native character scalar is not directly C interoperable, so both forms +record `BridgeDataAction.COPY_REPRESENTATION` with a boundary-specific reason. +The bridge associates the incoming address with exactly `n` +`character(kind=c_char)` bytes, copies them into backend-local +`character(kind=c_char, len=n)` storage, invokes the native dummy, and copies +all `n` post-call bytes back. It does not append NUL, allocate, free, infer +ownership, or create a Python result. These helper locals are emitted-code +details selected by the completed storage/raw policy. + +Optional storage/address arguments and projected returns remain blocked in +this phase. `String[()]` and `Addr(String)` are rejected by the semantic `.pyi` +contract because the bridge has no fixed extent; arrays and callback storage +remain owned by their later lanes. Validation rejects edited plans with a +missing/nonpositive length, the wrong owner/transfer/destruction/storage mode, +an inconsistent barrier or handoff, a runtime length role, an unjustified copy +reason, missing mutation, or result projection before either backend lowers. + +Replay `fixed_inout_storage` and `fixed_inout_raw` from the existing +`fnative_call_examples_f90` edited contract through a reduced entry bound to +the same `fixed_inout` native routine. Compare legacy and direct routes for +complete eight-byte mutation, rank/dtype/itemsize/writability failures, raw +integer type rejection, and lack of Python return. Keep the existing mixed +native-order test on the legacy route because its array and derived-type +neighbors belong to later phases; add the reduced replay as a separate +wrapper-plan ledger node. + +- [x] Complete mutable string-storage and raw-address policy, address handoff, + bridge association/copyback mechanics, validation, legacy/direct parity, + support widening, and ledger updates. + +### Phase 5 Completion + +Descriptor-backed scalar character values are deliberately outside this +phase. A contract such as `String | None` with +`result=Allocatable(Return(...))` carries allocation state, runtime element +length, descriptor ownership, and native release responsibility. It must enter +the direct route only through Phase 7's shared allocatable/pointer descriptor +plan; it remains a rank-zero Python `str | None` result rather than a native +array handle. Phase 5 must not add a character-only descriptor ABI or cleanup +path. + +- [x] Expand the phase under the mandatory expansion gate from the live + policies, legacy binding/bridge implementation, public string contract, and + focused wrapper tests. +- [x] Validate fixed/runtime length sources, payload/length role agreement, + result ownership, writeback consumers, and cleanup responsibility before + either backend emits source. +- [x] Keep string behavior in directly named string lowering methods while + reusing the ordinary scalar planning records and lifecycle flow. +- [x] Finish Phase 5 only when all non-descriptor scalar string sub-lanes are + proven and every affected matrix row is either `wrapper-plan` or blocked by + a later array, descriptor, field, or callback lane recorded in the ledger. + +## Phase 6 — Ordinary Arrays + +Scope: NumPy data-buffer arrays that do not require native descriptor handles. + +The ordinary-array lane borrows or copies NumPy data buffers; it never creates +or consumes a persistent native descriptor handle. `Allocatable[T[...]]`, +`Pointer[T[...]]`, rank-zero allocatable/pointer scalars, and a native handle +used as the actual value for an ordinary array dummy all remain in Phase 7. +Caller-supplied `Addr(T[...])` storage is the distinct Phase 6G follow-up and +must complete before Phase 7. +Derived-type arrays remain in Phase 8, fields in Phases 8 and 9, and callback +arrays in Phase 10. The full BLAS/LAPACK generation unit remains deferred until +final cutover even when individual ordinary-array shapes become supported. + +Whole-generation-unit rollout preserves that Phase 7 boundary. Output-only +ordinary array results and hidden outputs may select the production plan route +now. A generation unit with an ordinary array actual remains on the legacy +route, even after its NumPy-buffer path has direct-route parity, because route +selection cannot know whether a caller will pass a NumPy array or a supported +native descriptor handle. Those reduced array-actual rows remain `dual-route` +with the native-handle caller contract recorded as their sole Phase 7 blocker; +the direct route is forced only by the internal parity harness. + +The public behavior is defined by the NumPy array contract in +`docs/user/guide/fortran-wrapper.md`, the array spelling and metadata rules in +`docs/user/reference/semantic-pyi-format.md`, and the existing array wrapper +tests. The legacy binding validates exact dtype, rank, every expressible +extent, native byte order, alignment, layout/stride requirements, and +writeability for mutable storage before the native call. It does not cast, +byte-swap, repair alignment, de-alias overlapping storage, or silently copy a +rejected layout. Read-only source `intent(in)` storage may remain read-only; +edited `.pyi` array storage is writable unless a completed policy says +otherwise. Zero-sized dimensions are valid when the rest of the contract is +valid. + +Every ordinary array remains in the existing +`ArgumentPolicy -> ArgumentTransferPlan -> NativeCallSlotPlan` or +`ResultPolicy -> ResultPlan` flow. An argument embeds one editable array +handoff spec containing element family, concrete or runtime rank, declared +shape expressions, axis modes, order, contiguity, itemsize when relevant, +writeability, and the exact ABI roles for data, extents, upper bounds, strides, +runtime rank, or itemsize. Policy completion selects +`PythonBarrierAction.ARRAY_STORAGE`, +`NativeBarrierAction.PASS_ARRAY_BUFFER`, and either +`BridgeDataAction.ASSOCIATE_VIEW` for caller storage or an explicit copy action +and reason. Planning must not reconstruct any of those choices from rank, +shape spelling, or datatype. The binding and bridge dispatch only to directly +named array implementation methods selected by those completed facts. + +The binding checks the NumPy object before extracting `PyArray_DATA`, shape, +and element strides. The bridge receives only the fields named by the handoff +spec, associates the pointer with the completed element type and extents, and +constructs a stride slice only when the plan explicitly allows it. C-oriented +flat storage reverses bridge association extents only when the completed order +requires it. Backend-local pointer views and slice expressions are emitted-code +details; dtype, rank, extent, order, stride acceptance, mutation, projection, +copy, and ownership are semantic policy. + +The phase is dependency-ordered as follows. + +### Phase 6A — Required Rank-One Contiguous Buffers + +Included: required concrete-rank-one ordinary arrays with dense contiguous +axes (`T[:]`) for the existing bool, integer, real, and complex primitive +families; caller-owned borrowed/in-place storage; scalar or void neighbors and +results already supported by earlier phases; native position reordering; and +zero-length buffers. The binding requires an exact NumPy dtype, rank one, +native byte order, alignment, contiguity, and writeability only when completed +ownership says native code mutates the storage. It forwards the data address +and runtime extent. The bridge creates one typed rank-one pointer view with +`c_f_pointer` and passes that view in the completed native-call position. + +This slice records `ArgumentHandoffMode.ARRAY_BUFFER` and +`BridgeDataAction.ASSOCIATE_VIEW`; it performs no allocation, element copy, +writeback action, release, or Python result projection. Explicit/fixed extent +expressions, `Flat`, multidimensional order, strided axes, optionality, +projected output identity, array results, character arrays, assumed rank, and +native-handle actuals remain in later sub-lanes. Replay one existing +`fmath_arrays_f90` contiguous routine through a reduced semantic `.pyi` entry +and compare legacy/direct behavior for mutation, dtype, rank, alignment, +byte-order, contiguity, writeability, zero length, and native argument order. + +- [x] Complete required rank-one contiguous array policy, editable handoff + spec, validation, named C/Fortran lowering, reduced legacy/direct parity, + support widening, and ledger evidence. + +### Phase 6B — Declared Extents, Flat Storage, And Dense Rank + +Included: fixed and visible-symbol extent expressions, lower-bound-derived +extents, assumed-size `Flat`, ranks two through fifteen, `ORDER_F` and +`ORDER_C`, dense contiguous layout, and zero-sized axes. Shape expressions are +resolved against existing scalar handoff roles before backend emission; the +bridge association order follows the completed layout. Any expression that +cannot be represented by available roles remains blocked rather than being +recomputed in a backend. + +Order is an exact-storage selector, not an implicit conversion selector. +`ORDER_F` preserves logical axes over Fortran-contiguous storage. `ORDER_C` +passes the original C-contiguous address and reverses bridge extents, so native +Fortran observes the transposed storage view. Preserving the same logical axes +while accepting the opposite layout uses explicit `COPY_F` metadata, never an +inference from order. The owning `ArgumentTransferPlan` records C source order, +F native order, copy-in, conditional copy-out, original-object projection, and +temporary cleanup. The binding performs both copy directions and owns the +NumPy temporary. The bridge receives the temporary through the unchanged +ORDER_F association path and performs neither half of this representation +conversion. + +The initial `COPY_F` lane includes required, concrete-rank, dense numeric +ndarray arguments. It excludes `Flat`, assumed-rank, strided, optional and +character arrays, native descriptor arguments, and handle actuals until each +has separate policy and parity evidence. + +`Flat` is one axis marker and never collapses a multidimensional plan: +`T[:, Flat]` remains rank two in Fortran order, while +`Annotated[T[Flat, :], ORDER_C]` is its C-order orientation. The bridge reverses +only the C-order association extents. For an external assumed-size interface, +an explicit prefix such as `T[3, Flat]` may lower to `a(3, *)`; a runtime-only +prefix uses the standards-valid sequence-associated `a(*)` declaration while +the bridge retains every runtime extent and the completed logical rank. + +- [x] Complete declared-shape evaluation, flat-storage orientation, + multidimensional dense handoff, validation, parity, and ledger evidence. +- [x] Complete explicit C-to-Fortran representation copies through `COPY_F`, + including native-input and inout calls through the same binding-owned copy + lifecycle, projected original identity, temporary cleanup, direct bridge + reuse, validation, and compiled parity. Native `intent` remains owned by the + called procedure and is not duplicated in the semantic `.pyi` or bridge + temporary. + +### Phase 6C — Positive-Strided Ordinary Views + +Included: `::` axes and bounded stride-aware axes, runtime upper bounds and +element strides, Fortran-oriented positive-stride slicing, contiguous views as +a valid special case, and degenerate zero-size strides. Negative, zero on an +addressable axis, incompatible C-oriented, broadcast, and otherwise invalid +layouts fail before the native call. No copy-to-contiguous fallback is inferred. + +- [x] Complete stride roles, upper bounds, positive-stride bridge slices, + layout validation, parity, and ledger evidence. + +### Phase 6D — Output Storage And Projected Identity + +Included: ordinary `intent(out)`/`intent(inout)` caller buffers and +`Returns["name", T[...]]` projections. Native code mutates the same validated +NumPy storage; the binding returns the original Python array object with one +owned reference rather than constructing a second array or copying elements. +Read-only output storage fails before the call. Multiple projections compose +with the existing ordered result aggregation only after every projected array +identity and failure-path reference is planned. + +- [x] Complete in-place output ownership, projected identity/reference + lifecycle, multiple-result aggregation, parity, and ledger evidence. + +### Phase 6E — Ordinary Array Results And Hidden Outputs + +Included: non-allocatable direct array results and hidden output arrays whose +shape and element ownership are fully expressible without persistent native +descriptors. The plan records the producer, every runtime extent, allocation +owner, copy or transfer action, Python NumPy construction, and release on +success and every failure path. Nullable allocatable/pointer results remain in +Phase 7. + +- [x] Complete ordinary result/hidden-output allocation, shape projection, + copy ownership, cleanup, parity, and ledger evidence. + +### Phase 6F — Optional, Assumed-Rank, And Character Buffers + +Included: ordinary optional NumPy arrays, numeric assumed-rank dispatch from +one through fifteen, and fixed-width NumPy bytes character arrays with planned +itemsize. Omitted ordinary optional arrays remain distinct from present +storage. Assumed-rank plans carry a runtime-rank role and validate the supported +range before bridge dispatch. Character arrays use exact `NPY_STRING` itemsize +and remain raw fixed-width bytes; deferred descriptor-backed character values +remain in Phase 7. Fixed-shape character array direct results and hidden +outputs reuse the Phase 6E copy-result path with their itemsize included in +NumPy dtype construction and bridge byte-count calculation. + +- [x] Complete optional presence, assumed-rank dispatch, character itemsize, + validation, parity, and ledger evidence. + +### Phase 6A-F Ordinary-Buffer Completion + +- [x] Expand the phase under the mandatory expansion gate from live semantic + array contracts, legacy binding/bridge lowering, public docs, and focused + wrapper tests. +- [x] Define array handoff specs for every supported data, rank, shape, stride, + order, itemsize, writeability, result, and lifecycle role. +- [x] Validate every completed array policy and handoff role before either + backend emits source. +- [x] Finish Phases 6A-F only when every ordinary-array buffer matrix row is + migrated or remains blocked solely by an explicitly later descriptor, + derived, field, callback, or deferred-real-library lane. + +### Phase 6G — Raw Array Addresses — Complete + +Implementation status: complete. Required raw array addresses now use the +shared completed policy, `ArgumentTransferPlan`, native slot, centralized +validation, and named binding/bridge lowering paths. The dependency-closed +numeric and fixed-character runtime rows have passed compiled legacy/direct +parity and moved to `wrapper-plan`. + +Scope: required Python-visible type-level raw-address array arguments such as +`Addr(Float64[n])`. The caller supplies one Python integer address, x2py +forwards it as one opaque C address, and the bridge associates a typed native +array view using rank, shape, element type, and orientation facts completed +before `ir2ast.py`. There is no NumPy object, runtime handle, persistent native +descriptor, data copy, ownership transfer, or automatic release. + +This lane follows Phase 6 because its semantic object kind is +`ObjectKind.NUMPY_ARRAY` and its pointee layout reuses the array shape record. +It remains a distinct transport from an ordinary array buffer. The fixed +dispatch algorithm is: + +1. match `ObjectKind.NUMPY_ARRAY`; +2. match the completed Python barrier action; +3. lower `ARRAY_STORAGE` through the Phase 6A-F buffer path or `RAW_ADDRESS` + through Phase 6G; +4. require the matching native action, handoff mode, bridge data action, and + array-shape facts; and +5. fail validation rather than substituting the other transport. + +The same algorithm already separates scalar and string value, storage, and +raw-address forms. Phase 6G must extend that system; it must not add a parallel +raw-pointer planner, a datatype-based backend branch, or a special function or +module plan. + +#### Public Contract And Explicit Non-Scope + +The maintained public contract is already documented in +`docs/user/reference/semantic-pyi-format.md` and +`docs/user/guide/data-types.md`. Preserve it exactly: + +- `Addr(T[d1, ..., dr])` is depth one and has positive rank; +- the pointee dtype is primitive; +- every extent expression is resolved from literals and visible scalar + arguments or visible rank-zero scalar storage; +- the integer carries no dtype, rank, shape, order, alignment, bounds, + ownership, or lifetime metadata; +- x2py cannot prove that the supplied address actually points to compatible, + sufficiently large, live storage; and +- edited semantic `.pyi` raw-address storage is mutable caller storage unless + a completed policy explicitly says otherwise. + +The initial compiled oracle is `Addr(Float64[n])`. Before declaring the lane +complete, audit every public primitive family already accepted by semantic +policy, including bool, integer, real, complex, and fixed-width character +array pointees. Add compiled coverage for a family only when an existing native +routine can prove it without broadening the public contract. A fixed scalar +`Addr(String[n])` remains the completed Phase 5D string path; a rank-positive +`Addr(String[k][n, ...])` is an array path and must carry both the fixed element +length and the resolved array shape. + +Explicitly excluded from this lane are: + +- scalar `Addr(T)`, already completed in Phase 2E; +- fixed scalar `Addr(String[n])`, already completed in Phase 5D; +- NumPy `T[...]` storage, already completed in Phases 6A-F; +- unresolved or assumed shapes such as `Addr(Float64[:])`, assumed rank, + assumed size, and stride-marker shapes; +- optional, nullable, projected, direct-result, and hidden-output raw addresses + unless a separate public-contract audit first proves their intended Python + ownership and absence/result behavior; +- wrapped/derived pointees, pointer graphs deeper than one, and callbacks; +- `Allocatable[T[...]]`, `Pointer[T[...]]`, runtime native handles, and C + descriptors, which belong to Phase 7; and +- any implicit conversion from an ndarray or runtime handle to its address. + +#### One Action Vocabulary, Three Array Transports + +| Contract | Object kind | Python action | Native action | Handoff mode | Bridge data action | +| --- | --- | --- | --- | --- | --- | +| NumPy `T[...]` | `NUMPY_ARRAY` | `ARRAY_STORAGE` | `PASS_ARRAY_BUFFER` | `ARRAY_BUFFER` | `ASSOCIATE_VIEW` | +| Raw `Addr(T[...])` | `NUMPY_ARRAY` | `RAW_ADDRESS` | `PASS_RAW_ADDRESS` | `OPAQUE_ADDRESS` | `ASSOCIATE_VIEW` | +| Native descriptor contract | completed handle kind | completed handle action | `PASS_NATIVE_DESCRIPTOR` | Phase 7 descriptor mode | completed Phase 7 action | + +`ASSOCIATE_VIEW` means the bridge creates a typed, non-owning view; it does not +mean that the Python binding extracted a NumPy buffer. The Python and native +barrier actions remain the authoritative distinction. Do not introduce names +such as `PASS_RAW_ARRAY`, `COPY_RAW_ARRAY`, or datatype-specific address +actions. + +The completed ownership/action tuple for the required mutable public form is: + +- `OwnershipOwner.CALLER`; +- `TransferMode.IN_PLACE`; +- `DestructionPolicy.CALLER`; +- `StorageMode.STACK` for the call-local pointer carrier, not for the pointee; +- `CodegenAction.IN_PLACE_ARGUMENT`; +- `PythonBarrierAction.RAW_ADDRESS`; +- `NativeBarrierAction.PASS_RAW_ADDRESS`; +- `ArgumentHandoffMode.OPAQUE_ADDRESS`; and +- `BridgeDataAction.ASSOCIATE_VIEW` with no copy reason. + +If a retained source-derived contract can be read-only, policy may instead +complete `CALL_LOCAL` / `CALL_LOCAL_INPUT` / `NONE` destruction. Both +backends must consume that completed tuple; neither may infer mutability from +the pointee type or raw-address spelling. A raw array never has copy-in, +copy-out, projected-identity, allocation, destruction, release, or lifecycle +actions in this lane. + +#### Required Policy And Plan Shape + +Keep the feature under the existing `ArgumentTransferPlan`: + +```text +ArgumentTransferPlan + object_kind = NUMPY_ARRAY + binding.python_action = RAW_ADDRESS + bridge.native_action = PASS_RAW_ADDRESS + bridge.handoff_mode = OPAQUE_ADDRESS + bridge.data_action = ASSOCIATE_VIEW + array = ArrayHandoffPlan + native_call_slot = the same referenced NativeCallSlotPlan +``` + +Do not add `RawArrayPlan`, a second native slot, or a raw-address lifecycle +owner. Generalize the existing completed `ArrayHandoffPolicy` and +`ArrayHandoffPlan` only enough to carry raw pointee layout: + +- concrete rank and one shape expression per axis; +- one `data_role` equal to the binding/bridge/native-slot address role; +- `extent_reference_roles` naming the existing visible scalar handoff roles + used by each shape expression; +- the completed orientation used for native pointer association; +- fixed character element length/itemsize when the pointee family is string; + and +- no binding-extracted runtime rank, extent, upper-bound, stride, or itemsize + ABI roles. + +For a raw address, the shape record describes the pointee view; it does not +describe fields packed by the binding. A visible `n` used by +`Addr(Float64[n])` already has its own `ArgumentTransferPlan` and native-call +slot. Reference that role rather than passing a duplicate array extent. A +literal extent requires no extra ABI field. The bridge resolves the shape +expression from those planned native role names. + +Post-IR policy completion must explicitly select multidimensional orientation +before planning. Preserve the current legacy interpretation, including its +default orientation, only after capturing a rank-two artifact/runtime oracle. +Do not leave `ir2ast.py`, a codegen-model `order` default, or the bridge's local +shape reversal to make that decision. + +#### Completed Direct-Plan Seams + +The implementation split completed array policy by Python barrier action, +selected `OPAQUE_ADDRESS` and `ASSOCIATE_VIEW` before lowering, projected raw +pointee layout into the shared array record, omitted packed NumPy-buffer roles, +and added named raw-address checks and association methods to both backends. +Ordinary-array buffer checks remain unchanged and fail closed; neither backend +substitutes one transport for another. + +#### Dependency-Ordered Implementation Slices + +##### Phase 6G1 — Complete Raw Array Policy + +- [x] Make the `NUMPY_ARRAY` boundary validator dispatch on + `PythonBarrierAction` and add a named raw-address branch with the exact + ownership/action tuple above. +- [x] Complete raw pointee rank, shape expressions and their visible-scalar + dependencies, primitive family, fixed character element length, and + orientation before `ir2ast.py`. +- [x] Complete `OPAQUE_ADDRESS` and `ASSOCIATE_VIEW` from the action pair; do + not infer either in a backend. +- [x] Keep unresolved dimensions, unsupported pointee families, optionality, + projection, nullability, and deeper pointer graphs blocked with owner-path + diagnostics. +- [x] Freeze current behavior for zero/negative extent expressions, zero or + negative integer addresses, and integer overflow against the public docs and + legacy conversion before changing any rule. If a rule changes, change it in + policy and public docs, not in one backend. + +Audit result: resolved zero and negative extent expressions remain accepted +without a positivity check; integer zero becomes a null pointer without a +conversion error; negative integers follow `PyLong_AsVoidPtr`; and pointer-size +overflow raises `OverflowError`. Public documentation now states that these are +unsafe caller responsibilities, and tests prove the conversion guard and +generated shape without dereferencing an invalid address. + +##### Phase 6G2 — Project And Validate The Shared Plan + +- [x] Populate the existing `ArgumentTransferPlan.array` and its shared + `NativeCallSlotPlan.array` with one identical raw pointee layout record. +- [x] Reuse the scalar/string address handoff role and + `ArgumentHandoffMode.OPAQUE_ADDRESS`; add no raw-array ABI action. +- [x] Resolve every shape symbol to an existing visible scalar role and reject + unavailable, cyclic, hidden, non-scalar, or result-only dependencies before + lowering. +- [x] Split central array diagnostics by the completed Python action so buffer + validation still requires packed extent/layout roles while raw validation + forbids them. +- [x] Add editable-plan tests that independently corrupt object kind, Python + action, native action, handoff mode, bridge data action, rank, shape, + reference roles, element family, character length, orientation, and native + slot identity. + +##### Phase 6G3 — Reuse Binding Raw-Address Extraction + +- [x] Reuse `_lower_argument_required_raw_address()` for the Python integer + check and `PyLong_AsVoidPtr` conversion. Scalar, string, and array raw + addresses should share this extraction code. +- [x] Emit one `void *` handoff value and no `PyArray_*`, dtype, rank, shape, + layout, writeability, or itemsize checks. +- [x] Keep object-kind-specific logic out of the conversion method; array + shape affects only validation, the bridge view, and native call. +- [x] Preserve the existing conversion rule under which integer zero produces + a null pointer without itself raising a Python conversion error. Prove that + rule without dereferencing the null pointer; runtime tests must never call + native code with an invalid test address. + +##### Phase 6G4 — Add Named Raw Array Bridge Association + +- [x] Add directly named raw-array declaration and association methods in the + array method group. Dispatch to them only for + `NUMPY_ARRAY` / `RAW_ADDRESS` / `PASS_RAW_ADDRESS` / + `OPAQUE_ADDRESS` / `ASSOCIATE_VIEW`. +- [x] Declare one `type(c_ptr), value` bridge parameter and one backend-local + typed pointer view. The local view is an emitted-code helper, not a new plan + owner. +- [x] Associate the view with `c_f_pointer` using only the planned shape and + orientation, then pass that view in the existing native-call slot position. +- [x] Preserve fixed character element length when the pointee is a character + array. Do not pass a runtime itemsize unless a future public contract + explicitly requires one. +- [x] Emit no copy, writeback, allocation, release, descriptor, or NumPy + mechanics. + +##### Phase 6G5 — Prove The Route Before Widening It + +- [x] Retain semantic conversion coverage in + `tests/semantics/conversion/pyi/test_calls_and_projections.py` for round-trip, + visible extent sources, primitive pointees, and rejection of unresolved or + wrapped forms. +- [x] Add focused completed-policy tests for every authoritative action and + blocker, plus `array-raw-address-inputs` support classification. +- [x] Add `tests/wrapper_codegen/test_phase6g_raw_array_addresses.py` for plan + shape, edits, validation, C nodes, Fortran nodes, native order, and the + absence of buffer/descriptor/lifecycle nodes. +- [x] Extract `fill_vector_raw` from + `test_editable_contract_can_use_native_order_arguments_without_native_call` + into a reduced legacy/direct-plan parity test. Cover mutation through a valid + `raw_vector.ctypes.data`, ndarray rejection, wrong Python types, a visible + rank-zero scalar extent, and the established native argument order. +- [x] Prove raw-array native argument reordering in the direct-plan generated + call test. The legacy AST route retains only a projection marker and is not + an oracle for reordered projection-slot lowering. +- [x] Add literal and arithmetic extent-role cases. Add a rank-two runtime + parity case before freezing default/explicit orientation. Add a fixed-width + character-array case if the public family audit retains that contract. +- [x] Keep the broad native-order test `legacy` until its derived-type owner is + migrated; only the reduced raw-array row may move to `wrapper-plan` here. +- [x] Run the focused policy/plan/backend tests, the relevant wrapper test, + documentation checks, wrapper-codegen complexity checker, and required + static-analysis suite before changing route support. + +#### Phase 6G Exit Gate + +- [x] Expand raw array addresses as the explicit next lane using the public + contract, completed semantic policy, legacy binding/bridge primitives, and + the existing compiled `Addr(Float64[n])` oracle. +- [x] Complete Phases 6G1 through 6G5 without changing the public raw-address + contract or introducing a parallel action vocabulary. +- [x] Prove that one maintainer algorithm—object kind, Python action, native + action, handoff mode, data action, then typed shape facts—covers ordinary and + raw arrays without backend inference. +- [x] Move only dependency-closed raw-array test rows after generated-artifact + comparison and compiled legacy/direct parity pass. +- [x] Begin Phase 7 only after this exit gate is complete. Phase 7 must consume + the established distinction among array buffers, raw addresses, and native + descriptors rather than revisiting it. + +## Phase 7 — Native Array Handles And Descriptors + +Implementation status: reopened for the view-only `to_numpy()` contract +correction. The previously completed direct Phase 7A-H slices remain evidence +for unaffected descriptor handoffs, but Phase 7 is not closed again until +plain and `Aliased` module-array handles both return a current live view or +`None` without an implicit copy and the final verification gate is rerun. +Every field, pointer-result, callback, and deferred-real-library exclusion +remains on its later blocker. + +Scope: migrate the existing native descriptor and runtime-handle contract into +the wrapper-plan path without redefining that public contract. The maintained +`native-array-handle-checklist.md` remains the feature-level behavioral oracle; +this section owns only its migration into completed wrapper policy, +`ArgumentTransferPlan`, `ResultPlan`, `ModuleVariablePlan`, subordinate native +slots and lifecycle actions, direct C/Fortran lowering, and production route +selection. + +The shared descriptor family includes: + +- rank-positive `Allocatable[T[...]]` and `Pointer[T[...]]` handle arguments; +- optional-absent array handles, where omission or `None` means the native + optional dummy is absent; +- projected writable descriptors whose mutation must remain attached to the + same caller handle; +- wrapper-owned allocatable array results and hidden outputs; +- borrowed module allocatable and pointer handles plus their generated operation + tables; +- native handles passed as actual values to ordinary `T[...]` dummies without + an implicit `.to_numpy()` call; +- build requirements for standard C descriptors; and +- the remaining rank-zero allocatable/pointer result cases, including nullable + deferred-length scalar character values, which return copied Python values + rather than native-array handle objects. + +Allocatable and Pointer remain separate public contract types but share one +plan and lowering structure. Descriptor kind selects only the operations that +genuinely differ: allocation state versus association state, allowed +shape-changing operations, target lifetime, extraction policy, and release. +Do not create independent allocatable and pointer planner hierarchies. + +For every rank-positive module handle, `to_numpy()` has one public result: +`None` for an unallocated/unassociated native object and a live NumPy view of +the current allocation/target otherwise. Plain and `Aliased` allocatable +module variables use the same behavior. `Aliased` remains semantic metadata +but never selects a detached copy. Users call `.copy()` explicitly for +independent storage; an old live view may become stale after native +deallocation, reallocation, nullification, or reassociation, and a fresh +`to_numpy()` call must inspect current native state. + +### Phase 7 Boundary And Explicit Non-Scope + +The following four boundaries must remain distinct: + +| Python contract | Planned Python input | Native transport | Owner phase | +| --- | --- | --- | --- | +| `T[...]` with a NumPy array | validated NumPy storage | `PASS_ARRAY_BUFFER` | Phase 6 | +| `T[...]` with an allocated/associated native handle actual | validated handle array-data facet | `PASS_ARRAY_BUFFER` | Phase 7A | +| `Allocatable[T[...]]` / `Pointer[T[...]]` | matching runtime handle object | `PASS_NATIVE_DESCRIPTOR` | Phase 7B onward | +| `Addr(T[n, ...])` | caller-supplied integer address | `PASS_RAW_ADDRESS` | Phase 6G prerequisite, not Phase 7 | + +`Addr(Float64[n])` is a supported public semantic `.pyi` contract when +every extent is a literal or an expression over visible scalar arguments or +rank-zero scalar storage. It accepts an integer such as `array.ctypes.data` and +forwards that address without ownership, dtype, alignment, lifetime, or bounds +validation. Parsing, policy completion, printing, and both compiled wrapper +routes support it through the completed +`RAW_ADDRESS` / `PASS_RAW_ADDRESS` selector pair. Do not misclassify this raw +pointer as a NumPy buffer, native handle, or C descriptor while maintaining +Phase 7. + +Other exclusions and dependencies are: + +- ordinary NumPy-only buffer extraction, shape, stride, output identity, and + copy-result behavior already completed in Phase 6; +- caller-supplied raw array addresses, completed separately by the Phase 6G + entry dependency; +- derived-type field attachment, class construction, parent-wrapper creation, + and property orchestration, which require Phases 8 and 9 even though the + shared native-handle plan must already be reusable by those later owners; +- scalar derived module-variable member access and argument compatibility, + which belong to Phase 8. Phase 7 descriptor machinery remains limited to + array handles; scalar derived module allocatables use the exact local + move-out/move-back route specified in Phase 8H and do not consume Phase 7 CFI + descriptor machinery. A failed scalar-object call handoff must not become a + module-access blocker; +- pointer results without completed stable owner storage and target lifetime; +- callback descriptor arguments or results, which remain in Phase 10; +- compiler-private descriptor layout inspection or copying; +- any implicit `.to_numpy()` conversion when a native handle is passed to an + ordinary array dummy; and +- the deferred BLAS/LAPACK generation unit until final cutover. + +### Existing Semantic Authority And Legacy Oracle + +Do not redesign the public feature while migrating it. Reuse these completed +sources of truth: + +- `x2py/semantics/native_array_handles.py` defines + `NativeArrayHandlePolicy`, `ArrayInteropPolicy`, handle facts, descriptor + kinds, and completed build requirements. +- `x2py/semantics/policy_completion.py` completes handle kind, origin, owner, + owner retention, descriptor ownership, getter/setter behavior, output + projection, release, target lifetime, destruction, extraction, interop, + nullability, storage mode, operations, and blockers before `ir2ast.py`. +- `x2py/runtime/handles.py` owns the reusable runtime protocol, including + `_native_array_actual_argument_for_binding_positional`, + `_native_array_descriptor_argument_for_binding_positional`, and + `_native_array_descriptor_handoff_for_binding_positional`. Direct lowering + must call these helpers rather than duplicate their Python validation. +- `x2py/codegen/bindings/c_to_python.py` is the legacy binding oracle. Its + `_ARRAY_INTEROP_POLICY_DISPATCHER`, `_NATIVE_ARRAY_HANDLE_DISPATCHER`, + descriptor-argument handlers, owned-result handlers, operation wrappers, and + descriptor reader define the currently passing C behavior. +- `x2py/codegen/bridges/fortran_to_c.py` is the legacy bridge oracle. Its + corresponding dispatchers, descriptor-argument handlers, module/field + operation generators, and owned-allocatable result helpers define the + currently passing Fortran behavior. +- `x2py/pipeline/build.py` already derives native-array build requirements from + completed semantic policy and records them in manifests. The wrapper plan + must carry and emit the matching artifact requirements without rediscovering + them from generated source text. + +The legacy generators are behavioral oracles, not dependencies of +`x2py/wrapper_codegen`. Reuse the runtime helpers and completed semantic +records directly. Rewrite the smallest equivalent node/lowering methods in the +direct generators; do not import legacy binding/bridge generator methods or +legacy codegen-model nodes into the wrapper-plan package. + +### Completed Direct-Plan Shape + +Wrapper policy now carries the completed native-handle and array-actual facts. +`ArgumentTransferPlan`, `ResultPlan`, and `ModuleVariablePlan` distinguish a +NumPy data-buffer transfer, a normal array dummy receiving a handle actual, and +a descriptor-handle transfer. Central validation fails closed when any typed +handoff, operation, role, ownership fact, or required header is inconsistent; +neither backend infers policy from datatype or `descriptor_boundary`. + +### Required Plan Shape + +Keep all descriptor-specific state subordinate to the existing datatype- +varying owners: + +```text +ArgumentTransferPlan + array: ArrayHandoffPlan | None + native_array_actual: NativeArrayActualPlan | None + native_array_handle: NativeArrayHandlePlan | None + handoff: NativeDescriptorHandoffPlan + native_call_slot: NativeCallSlotPlan + +ResultPlan + native_array_handle: NativeArrayHandlePlan | None + native_call_slot: NativeCallSlotPlan | None + +ModuleVariablePlan + native_array_handle: NativeArrayHandlePlan | None + +FunctionPlan + native_call_slots: shared ordered references + lifecycle actions: ordered handle materialization/release references +``` + +`NativeCallSlotPlan` and `LifecycleActionPlan` are not competing top-level +semantic owners. A native slot is the argument/result ABI facet shared by its +owning transfer plan, while lifecycle records are function-wide ordering +indexes back to argument/result roles. Descriptor ownership, release, and +operation policy stay under `ArgumentTransferPlan`, `ResultPlan`, or +`ModuleVariablePlan`. Backend-local CFI storage, copy buffers, and failure +cleanup remain inside the named lowerer selected by those plans. + +`NativeArrayActualPlan` is used only when an ordinary `T[...]` argument permits +a runtime native handle as another source for the existing array-buffer ABI. It +records the explicitly accepted Python source kinds and the shared dtype, rank, +shape, layout, writeability, native-byte-order, alignment, and ABI-role checks. +It never carries descriptor ownership or extraction policy. + +`NativeArrayHandlePlan` is one editable projection of the completed handle +policy. It must contain, using typed values rather than free-form backend +method names: + +- descriptor kind and handle kind; +- origin, owner, owner-retention mode, descriptor ownership, and borrowed state; +- element datatype family, dtype, rank, declared shape, order, and character + element length when applicable; +- getter behavior, Python setter exposure, and native setter assignment; +- output projection and same-handle identity requirements; +- release responsibility, target lifetime, destroy behavior, and storage mode; +- `.to_numpy()` extraction action and allowed generated operations; +- descriptor-interop requirement and required headers; +- nullability and optional-absent-handle behavior; and +- one `NativeDescriptorHandoffPlan` with its ABI form and symbolic roles. + +`NativeDescriptorHandoffPlan` must distinguish these typed ABI forms: + +- `FACT_PACKED_CALL_LOCAL`: a non-projected descriptor argument supplies + validated standard descriptor facts; the binding passes those fields and the + bridge establishes call-local standard C descriptor storage. +- `DIRECT_STANDARD_DESCRIPTOR`: a projected writable handle passes its + persistent standard-descriptor pointer so allocation, deallocation, + reassociation, and shape changes remain attached to that handle. +- `OWNED_RESULT_STORAGE`: an allocatable result is materialized into persistent + wrapper-owned CFI storage and later destroyed by the runtime handle. + +The handoff records the descriptor-pointer role when present, `base_addr`, +`elem_len`, runtime rank, per-axis lower-bound/extent/stride-multiplier roles, +an optional presence role, owner-storage role, and generated-operation roles. +The `NativeCallSlotPlan` and its owning argument or hidden result must reference +the same mutable handoff record; do not duplicate descriptor facts that a +maintainer would need to edit twice. + +Convert the current string-valued completed policy selectors into typed plan +enums or validate and translate them exactly once while building wrapper +policy. Backends must not match raw strings such as `argument_descriptor`, +`projected_handle`, or `pointer_c_descriptor` to choose behavior. + +### Consistent Action Vocabulary + +Reuse the existing orthogonal actions: + +| Case | `ObjectKind` | Python action | Native action | `CodegenAction` | Bridge data action | +| --- | --- | --- | --- | --- | --- | +| Ordinary array with ndarray or handle actual | `NUMPY_ARRAY` | `ARRAY_STORAGE`, with explicitly planned accepted sources | `PASS_ARRAY_BUFFER` | existing Phase 6 input/in-place action | `ASSOCIATE_VIEW` | +| Read-only descriptor handle argument | `NUMPY_ARRAY` | `WRAPPER_INSTANCE` | `PASS_NATIVE_DESCRIPTOR` | `CALL_LOCAL_INPUT` | `ASSOCIATE_VIEW` | +| Writable projected descriptor handle | `NUMPY_ARRAY` | `WRAPPER_INSTANCE` | `PASS_NATIVE_DESCRIPTOR` | `IN_PLACE_ARGUMENT` | `DIRECT_TRANSFER` | +| Owned allocatable handle result | `NUMPY_ARRAY` | `NONE` | `NONE` or hidden `PASS_NATIVE_DESCRIPTOR` | `WRAPPER_INSTANCE` | `COPY_REPRESENTATION` with an ownership-transfer reason | +| Borrowed module handle getter | `NUMPY_ARRAY` | module getter action `NATIVE_ARRAY_HANDLE` | operation-specific | `BORROWED_VIEW` | completed per operation | + +Using `WRAPPER_INSTANCE` for the Python handle is consistent with the existing +action axis: the binding validates and consumes a generated runtime wrapper +object, while `ObjectKind.NUMPY_ARRAY` still identifies its array semantic +family. Add a new Python action only if a proven backend operation cannot be +expressed by this existing pair. Add `ArgumentHandoffMode.NATIVE_DESCRIPTOR` +because descriptor tuples are a genuinely different binding-to-bridge ABI from +`ARRAY_BUFFER`; do not overload the Phase 6 mode. + +Keep rank-zero descriptor values on the scalar or string object-kind route. +Their result action creates a Python scalar/string or `None`, not +`WRAPPER_INSTANCE`, and they must not carry `NativeArrayHandlePlan`. + +### Cross-Backend Validation Invariants + +Before either backend emits source, `_validate_plan()` must reject every one of +these inconsistencies: + +- a descriptor plan whose completed `ObjectKind` is not `NUMPY_ARRAY`; +- `PASS_ARRAY_BUFFER` carrying descriptor ownership or CFI roles; +- `PASS_NATIVE_DESCRIPTOR` carrying ordinary data-buffer handoff roles without + a descriptor handoff; +- a disagreement among handle policy, interop ABI, descriptor kind, handle + kind, argument/result plan, and native-call slot; +- a required handle accepting `None`; +- an optional absent handle without a presence role, or a required handle with + one; +- collapsing optional absence into present-unallocated/present-unassociated + state: an absent handle has null fields and a null presence token, whereas a + present handle may have null `base_addr` but must have a non-null presence + token; +- fact-packed handoff for a projected writable descriptor, or direct persistent + descriptor handoff for a policy that does not permit descriptor mutation; +- direct descriptor handoff without a typed + `_NativeArrayDescriptorHandoff`-compatible runtime operation; +- descriptor dtype, rank, shape, element length, or per-axis field counts that + disagree with the declared handle data facet; +- pointer reassociation, allocation, deallocation, or resize without completed + `PointerPolicy` permission; +- a pointer result without stable owner storage and target lifetime; +- an owned result without wrapper ownership, heap/alias boundary storage, + destroy behavior, owner retention, or a failure-path release action; +- a borrowed module/field handle that claims to destroy native owner storage; +- descriptor-view extraction without its completed C-descriptor build + requirement; +- a C-descriptor header requirement on a generation unit whose completed plans + do not need that interop; and +- any semantic helper temporary represented by a fabricated + `OwnershipDecision`. Call-local CFI variables, decoded-dimension locals, + pointer views, status locals, and operation tables are backend-local emitted + storage inside the already selected method. + +### Phase 7A — Ordinary Array Dummies Accepting Native Handle Actuals + +Included: concrete-rank numeric `T[...]` arguments already supported by Phase 6 +when the runtime value is either a valid ndarray, an allocated allocatable +handle, or an associated pointer handle. The handle path validates the same +dtype, rank, shape, layout, writeability, byte-order, and alignment contract, +then calls the handle's internal `array_actual` operation and packs the existing +Phase 6 pointer/extent/stride ABI. It never calls `.to_numpy()` and never passes +the allocatable/pointer descriptor to the ordinary native dummy. + +Initially excluded: optional, assumed-rank, character, and unsupported +noncontiguous handle actuals. Audit each against live runtime-helper behavior +before widening this sub-lane; a rejected form must remain an explicit blocker, +not silently fall back to `.to_numpy()` or a raw address. + +Those exclusions remain visible as the uncompleted +`array-handle-actuals-excluded` rollout lane. Their direct Phase 6 ndarray +lowerers remain testable with a forced wrapper-plan route, but automatic +production selection stays on the legacy route until each corresponding handle +source has parity evidence. + +Legacy oracle: `CPythonBindingGenerator._native_array_actual_argument_body`, +the normal-array runtime helpers in `x2py/runtime/handles.py`, and the existing +Phase 6 bridge array-buffer lowering. Reuse the runtime helpers and bridge ABI; +rewrite only the minimal direct binding call and source-kind branch. + +Plan and lowering requirements: + +- [x] Add `NativeArrayActualPlan` or equivalent accepted-source facts beneath + the existing ordinary `ArgumentTransferPlan`; keep + `PASS_ARRAY_BUFFER`, `ArgumentHandoffMode.ARRAY_BUFFER`, and + `ArrayHandoffPlan` unchanged. +- [x] Make the C binding's named ordinary-array input method call + `_native_array_actual_argument_for_binding_positional` with only planned + validation flags and ABI-field selections. +- [x] Keep the Fortran bridge on the exact Phase 6 array-buffer method; it must + not know whether Python supplied an ndarray or a handle. +- [x] Validate that handle actuals are allocated/associated, have a non-null + data address, and satisfy the same declared contract as ndarray inputs; + preserve allocated/associated zero-length arrays. +- [x] Add the `array-native-handle-actuals` support lane and remove the current + production gate on ordinary array actuals only after reduced compiled parity + proves both runtime source kinds and all rejection paths. +- [x] Reuse the normal-array calls in + `test_module_and_derived_pointer_handles_track_native_association` and + allocatable handle fixtures as the legacy baseline, but extract a class-free, + dependency-closed parity contract so Phase 8 does not determine this lane's + route. + +### Phase 7B — Required Read-Only Descriptor Handle Arguments + +Included: required, non-projected `Allocatable[T[...]]` and +`Pointer[T[...]]` arguments. The Python binding accepts only the matching +runtime handle class. A present unallocated allocatable or unassociated pointer +is still a present descriptor argument and may carry a null `base_addr`. + +The binding uses the existing descriptor runtime helper to obtain validated +standard descriptor facts. The bridge establishes rank-specific call-local CFI +storage from `base_addr`, `elem_len`, rank, and dimension records, then passes +the native allocatable or pointer dummy. This association is an emitted-code +view, not a semantic data copy. + +Legacy oracle: + +- binding `_bind_allocatable_descriptor_argument`, + `_bind_pointer_descriptor_argument`, and + `_bind_fact_packed_native_array_descriptor_argument`; +- bridge `_bridge_allocatable_descriptor_argument`, + `_bridge_pointer_descriptor_argument`, and + `_bridge_native_array_descriptor_argument`; and +- runtime `_native_array_descriptor_argument_for_binding_positional`. + +- [x] Carry the completed `NativeArrayHandlePolicy` and descriptor + `ArrayInteropPolicy` into `ArgumentPolicy`, `ArgumentTransferPlan`, and its + shared native slot. +- [x] Add `ArgumentHandoffMode.NATIVE_DESCRIPTOR` and a + `FACT_PACKED_CALL_LOCAL` descriptor handoff with exact symbolic roles. +- [x] Add directly named C and Fortran descriptor-input methods grouped under + the native-array-handle family; backend-local tuple items and CFI locals may + be created only inside those selected methods. +- [x] Validate matching handle class, descriptor kind, dtype, rank, declared + shape, and element length before the call. Reject ndarray inputs. +- [x] Add separate `allocatable-descriptor-inputs` and + `pointer-descriptor-inputs` support lanes after reduced descriptor-argument + parity passes. +- [x] Replay the descriptor calls in + `test_pointer_descriptor_views_preserve_slice_shape_strides_and_parent_lifetime` + and the allocatable descriptor fixtures through minimal class-free contracts; + retain the mixed original nodes as legacy until all their later owners migrate. + +### Phase 7C — Optional Absent Descriptor Handles + +Included: `Allocatable[T[...]] | None = ...` and +`Pointer[T[...]] | None = ...` callable arguments. Omission and explicit +`None` both mean native `present(...)` is false. A present handle remains +present even when its descriptor has absent allocation/association state. + +This is a two-level handle-presence contract, not the Phase 3 scalar descriptor +three-state value contract. Do not reuse the value pointer as the presence +token. The runtime helper already produces null fact fields plus null presence +for absence, and a distinct non-null token for every present handle. + +- [x] Project `optional_absent`, `nullable`, presence mode, and the dedicated + presence role from completed handle policy without inspecting the Python + object in planning or bridge code. +- [x] Generate both required and optional fact-packed descriptor calls through + the Phase 7B methods, adding only the planned presence ABI field and native + branch. +- [x] Validate required-versus-optional annotation, field count, presence role, + and the distinction between absent handle and present null `base_addr`. +- [x] Add `optional-native-array-handles` route coverage only after compiled + tests exercise omission, explicit `None`, present allocated/associated, + present unallocated/unassociated, wrong handle kind, and wrong dtype/rank. +- [x] Treat the lack of one isolated compiled optional array-handle fixture as + a coverage gap: create a reduced semantic `.pyi` entry over an existing + native optional descriptor routine instead of inventing behavior from the + runtime-only tests. + +### Phase 7D — Writable And Projected Descriptor Handles + +Included: descriptor arguments whose allocation, deallocation, resize, +reassociation, or nullification must remain visible through the same Python +handle, plus a matching projected result that returns that identical handle. +Allocatable mutation follows completed ownership. Writable pointer descriptor +mutation requires explicit `PointerPolicy` permissions and target-lifetime +facts. + +Fact-packed call-local descriptors are forbidden here because native mutation +would be discarded at return. The binding must request the handle's typed +persistent standard-descriptor pointer and the bridge must pass it directly. +Returning the projection increments/transfers the existing Python reference; it +does not construct a replacement handle or call `.to_numpy()`. + +The direct handoff requires generated persistent standard-descriptor storage. +Wrapper-owned result handles provide it. Borrowed module handles expose current +descriptor facts for read-only calls, but they are not accepted for projected +writable mutation because a reconstructed call-local descriptor would lose the +native descriptor update. + +Legacy oracle: + +- binding `_bind_direct_native_array_descriptor_argument` and + `_bind_projected_native_array_handle_result`; +- bridge descriptor argument dispatch with completed output projection; and +- runtime `_native_array_descriptor_handoff_for_binding_positional`. + +- [x] Add `DIRECT_STANDARD_DESCRIPTOR` handoff and same-handle result identity + to the owning `ArgumentTransferPlan`, shared native slot, `ResultPlan` or + lifecycle consumer, and function-wide result order. +- [x] Reuse `CodegenAction.IN_PLACE_ARGUMENT` and `DIRECT_TRANSFER`; do not add + a descriptor-copy action for same-handle mutation. +- [x] Plan success and failure reference handling so a projected handle is + returned exactly once and borrowed caller storage is never destroyed. +- [x] Validate operation permissions, descriptor ownership, target lifetime, + direct handoff type, result identity, and optional presence before emission. +- [x] Add `projected-native-array-handles` support only after + `test_allocatable_inout_arrays_mutate_and_return_the_same_handle` has a + reduced legacy/direct-plan parity replay covering allocation, reallocation, + deallocation, identity, wrong input types, and native-memory checks. +- [x] Keep writable pointer reassociation blocked unless the completed policy + proves every required permission and lifetime fact; never downgrade it to a + read-only fact-packed call. + +### Phase 7E — Owned Allocatable Results And Hidden Outputs + +Included: rank-positive allocatable direct function results and hidden output +descriptors whose completed policy selects `owned_result_descriptor`. A valid +allocatable function result is allocated when returned; an unallocated +nonpointer function result is a nonconforming native procedure and the wrapper +does not compensate for it. An allocatable output dummy may validly remain +unallocated and still returns a present `AllocatableArray` handle whose state +lives inside that handle. Pointer handle results remain blocked until stable +owner storage and target lifetime are explicit. + +For a numeric direct allocatable function result, the bridge assigns the native +function expression once into a procedure-local allocatable and then uses +`move_alloc` to transfer that allocation into the allocatable `intent(out)` +dummy backed by persistent wrapper-owned `CFI_CDESC_T(rank)` storage. The move +does not copy the array payload. Do not insert a collector helper, an +`allocated(...)` guard, or a second intrinsic assignment. The native function +must return an allocated, defined result; an unallocated result is a +nonconforming native procedure and remains the user's responsibility rather +than a wrapper fallback. Other procedure-local storage remains permitted only +when representation conversion genuinely requires it, such as +deferred-character byte materialization. The binding constructs the complete +generated operation table and Python handle only after owner storage is valid. +Ownership transfers to the handle exactly once; every earlier failure path +releases the persistent allocation and any genuinely required bridge-local +allocation. + +Character-element handles carry runtime `elem_len` and declared element-length +policy in the same descriptor record. Because a deferred character width is +unknown until the native result exists, the bridge first copies the bytes and +the binding then establishes and allocates persistent CFI storage with that +runtime width. This is a named lowering method under the same result handle +plan, not a separate string-result ownership hierarchy. + +Legacy oracle: + +- binding `_bind_owned_allocatable_result_handle`, owned-result operation + builders, `_bind_materialized_native_array_handle_result`, and destroy body; +- bridge `_bridge_owned_allocatable_result_handle` plus allocatable result + helper/copy logic; and +- the runtime handle factory and exactly-once `close()`/finalizer protocol. + +- [x] Attach one `NativeArrayHandlePlan` with `OWNED_RESULT_STORAGE` to direct + and hidden `ResultPlan` owners; hidden outputs share their exact descriptor + native slot. +- [x] Use `CodegenAction.WRAPPER_INSTANCE` and an explained + `COPY_REPRESENTATION` only for materialization into persistent owner storage; + source hiddenness remains `source_kind`, not a codegen action. +- [x] Record owner storage, materialization, handle construction, ownership + transfer, destroy behavior, and release responsibility under the result's + typed handle plan. Keep backend-local CFI allocation/copy/free nodes inside + the selected result lowerer rather than fabricating lifecycle policy records. +- [x] Require generated `shape`, `array_actual`, `descriptor`, extraction/state, + allowed mutation, and `destroy` operations before publishing the handle. +- [x] Validate CFI rank, dtype, element length, allocated state, owner + retention, release responsibility, destroy behavior, and all success/failure + paths before emission. +- [x] Add `owned-allocatable-results` and + `owned-allocatable-hidden-outputs` support lanes after reduced parity from + `test_array_results_follow_data_buffer_and_descriptor_handle_contracts`, + `test_output_arguments_and_multiple_results_follow_python_projection_rules`, + and `test_allocatable_module_fields_and_results_expose_lifetime_safe_handles`. +- [x] Keep pointer result tests on their explicit policy blocker; do not make + their matrix rows `wrapper-plan` merely because allocatable results pass. + +### Phase 7F — Borrowed Module Handles And Generated Operations + +Included: rank-positive allocatable and pointer module variables exposed as one +stable borrowed handle object at module initialization. Repeated attribute reads +return the same handle. Replacement assignment is rejected. The generated +operation table accesses current native state and includes only operations +allowed by completed policy. + +Allocatable operations include allocation state, shape, array actual, +descriptor handoff, extraction, deallocation, and resize where allowed. Pointer +operations include association state, shape, array actual, descriptor handoff, +nullification, extraction, and policy-gated allocation/deallocation/resize. +Borrowed module handles retain the Python module and never destroy native-owned +descriptor storage. + +Deferred-character handles also expose runtime `element_length`. Shape-only +`allocate` and `resize` operations are omitted because they cannot state the +new character width; native procedures that declare the width remain the +authoritative mutation path. + +Legacy oracle: the bridge's `_native_array_module_handle`, +`_native_array_module_handle_operations`, and operation-specific module methods; +the binding's `_bind_borrowed_native_array_module_handle`, operation wrappers, +and handle creation; and the current runtime handle factory. + +- [x] Add a native-handle getter action and one `NativeArrayHandlePlan` beneath + `ModuleVariablePlan`; keep Python attribute exposure and native operation + generation in its binding and bridge child views. +- [x] Plan operation roles and export names explicitly while leaving operation + call locals backend-local. Do not store generated method names in the plan. +- [x] Validate stable handle identity, module owner retention, rejected + replacement, descriptor kind, operation completeness, and borrowed/no-destroy + lifecycle. +- [x] Add `allocatable-module-handles` and `pointer-module-handles` support + lanes only after module-only reduced parity covers state changes, zero-length + state, extraction policy, operation permissions, stale-view behavior, and + module lifetime. +- [x] Use `test_allocatable_module_fields_and_results_expose_lifetime_safe_handles`, + `test_plain_allocatable_module_array_exposes_current_live_view`, + and the module portion of + `test_module_and_derived_pointer_handles_track_native_association` as legacy + oracles. Split out field/class assertions, which remain Phase 8/9 work. + +#### Phase 7F Contract Correction — View-Only Module Extraction + +The checked Phase 7F items above record the original migration slice; they do +not close this changed public contract. Complete this correction before Phase +8 implementation. + +- [x] Update public docs, maintainer docs, generated/checked semantic `.pyi` + evidence, and wrapper coverage rows to specify current live view or `None`, + explicit `.copy()`, and the unsupported stale-view window. +- [x] Complete plain and `Aliased` allocatable module arrays as native-owned + borrowed handles with the same extraction result. Keep addressability, + descriptor mechanism, owner retention, mutability, nullability, storage, + operation permissions, and release responsibility as separate completed + facts. +- [x] Remove `read_only_detached_copy` and extraction-only `copy_only` policy, + plan, runtime, binding, and bridge dispatch. Preserve only typed live-view + mechanisms such as contiguous or standard-descriptor views; unsupported + extraction fails instead of copying. +- [x] For a plain allocatable module array, add the completed standard- + descriptor module-state mechanism needed to inspect the current allocation + on each extraction. Keep it beneath `ModuleVariablePlan`; do not retain a + descriptor or data address as if it were permanently current. +- [x] Keep binding/bridge ownership explicit: the bridge exposes current native + descriptor facts without NumPy knowledge, and the binding validates + dtype/rank/shape/strides and creates the NumPy view with its handle owner as + the base. Native-handle argument handoff must not call `to_numpy()`. +- [x] Replace the obsolete read-only-copy test with source/generated-`.pyi` + parity covering plain and `Aliased` live mutation, allocated/unallocated and + associated/unassociated state, fresh extraction after state changes, + explicit-copy independence, stale-view documentation, parent/owned-result + retention, and contiguous/strided pointer views. +- [x] Rerun focused policy/plan/backend/runtime tests, documentation checks, + the wrapper suite excluding LAPACK, the wrapper-codegen complexity checker, + and the required static-analysis suite before closing Phase 7 again. + +### Phase 7G — Pointer Descriptor Extraction And Build Requirements + +Included: pointer `descriptor_view`, `contiguous_view`, and explicitly +unsupported extraction actions already selected by completed policy; standard +descriptor decoding; positive and negative strides; and local build/header +requirements. A `copy_only` `to_numpy()` action is obsolete and must not reach +the corrected plan. + +Descriptor views, the corrected plain allocatable module-state path, and +persistent allocatable owner storage require standard C descriptor support. +Generated code may read `CFI_cdesc_t` through `ISO_Fortran_binding.h` when the +completed plan requests it. It must never guess or expose a compiler-private +descriptor layout. Unsupported toolchains fail readiness/build with the +completed owner path and requirement. + +- [x] Carry typed extraction and descriptor-interop actions plus required + headers into handle/module/result plans and rendered artifact metadata. +- [x] Reuse the runtime descriptor-view helper for shape, stride, buffer-window, + dtype, rank, and null-address validation; direct C lowering only decodes the + standard descriptor fields into its expected mapping. +- [x] Add directly named C descriptor-reader and operation-wrapper methods; + decoded dimension objects and mapping temporaries remain binding-local. +- [x] Validate that build requirements equal the union of completed plans, + appear in replayable manifests, and do not leak into wrappers that need only + ordinary buffers or non-CFI borrowed allocatable handles. +- [x] Replay + `test_pointer_descriptor_views_preserve_slice_shape_strides_and_parent_lifetime` + and `test_pyi_manifest_records_pointer_descriptor_interop_requirements`, plus + focused `tests/runtime/handles`, before enabling + `pointer-descriptor-extraction`. +- [x] Preserve the explicit readiness failure when required C descriptor + support is unavailable; no contiguous-copy fallback may be inferred in the + backend. + +### Phase 7H — Remaining Rank-Zero Descriptor Results And Strings + +Phase 3 already owns ordinary and optional scalar descriptor inputs, including +omitted/present-null/present-value state. Phase 4 already owns nullable scalar +descriptor module reads as copied Python snapshots. Do not rebuild those paths +or turn rank-zero descriptors into runtime handle objects. + +Included here: direct scalar descriptor function results, hidden scalar +descriptor outputs, projected scalar descriptor readback, and allocatable or +pointer scalar character results with runtime/deferred length. The Python result +is `T | None` or `String | None`; absent allocation/association returns `None`. +An allocated/associated value is copied exactly once before the call-local or +native descriptor is released. Deferred-length strings use runtime element +length and preserve the existing encoding/byte contract. + +- [x] Add a subordinate scalar-descriptor handoff/result record to the existing + scalar or string `ArgumentTransferPlan`/`ResultPlan`; do not attach + `NativeArrayHandlePlan` or use `ObjectKind.NUMPY_ARRAY` for rank zero. +- [x] Complete result source, descriptor kind, presence, runtime element length, + copy action/reason, release owner, and failure cleanup in wrapper policy before + planning. +- [x] Reuse existing Phase 3 presence records and typed lifecycle ordering; + extend named scalar/string result lowering only for the descriptor producer + and copy/release steps. +- [x] Validate direct versus hidden descriptor source, nullable result spelling, + result ordering, runtime string length, null state, copy count, and cleanup on + conversion/status failure. +- [x] Add isolated legacy/direct parity for numeric allocatable and pointer + results and for `string_result_deferred` from + `test_modern_fortran_character_arguments_and_results`, including an absent + result and non-ASCII encoded data. +- [x] Keep pointer array results blocked even after pointer scalar values pass; + copied scalar readback does not prove array target lifetime. + +### Derived Fields Remain A Recorded Later Dependency + +The shared handle plan must be capable of recording +`borrowed_field_descriptor`, `owner_retention=parent_wrapper`, field operation +roles, and parent-owned destruction behavior. Do not add field/class traversal +or route eligibility in Phase 7. `BindCNativeArrayHandleProperty`, field +operation generation, and the field portions of allocatable/pointer tests remain +legacy oracles for Phases 8 and 9, where the owning wrapper instance and property +lifecycle exist in the plan. + +This boundary prevents Phase 7 from either duplicating future `FieldPlan` +ownership or falsely marking mixed module-and-field generation units supported. + +### Legacy Primitive Inventory And Rewrite Rule + +| Primitive | Legacy source | Direct-plan treatment | +| --- | --- | --- | +| Normal array handle actual | binding `_native_array_actual_argument_body`; runtime normal-array helpers | reuse runtime helper and Phase 6 bridge ABI; rewrite minimal binding nodes | +| Required/optional descriptor argument | binding descriptor argument helpers; bridge descriptor handlers | rewrite named direct methods around shared runtime packer and planned CFI roles | +| Projected writable descriptor | binding direct descriptor handler; bridge descriptor projection | rewrite direct pointer handoff and identity lifecycle; no fact-packed fallback | +| Owned allocatable result | binding owned-result/operation helpers; bridge allocatable result helper | rewrite minimal CFI owner-storage and result lifecycle nodes; assign once locally and transfer the allocation into the CFI-backed output dummy with `move_alloc` | +| Borrowed module handle | binding handle creation/operation wrappers; bridge module operations | rewrite under `ModuleVariablePlan`; reuse runtime factory | +| Pointer descriptor view | binding descriptor reader; runtime view helper | reuse runtime view helper; rewrite only standard-descriptor decoding nodes | +| Scalar descriptor result | legacy scalar descriptor/result conversion | extend existing scalar/string plan route; do not create an array handle | +| Build requirement | semantic `native_array_handle_build_requirements`; build manifest | reuse completed requirements and carry them through rendered artifacts | + +For every primitive, first retain generated legacy C/Fortran/header artifacts +from the cited passing wrapper test. Explain each material direct-plan artifact +difference before compilation. Copy a small legacy method only when it already +matches the direct node API and complexity limit; otherwise rewrite the minimal +equivalent. Do not copy legacy dispatcher classes, scope mutation machinery, or +datatype/policy inference. + +### Route And Test Migration Matrix For Phase 7 + +Mixed rows retain their later derived/field owners. The dependency-closed +Phase 7 rows were split, proved through both routes, and then recorded as +`wrapper-plan` in the complete ledger. + +| Existing node or group | Current role/status | Phase 7 owner and target | +| --- | --- | --- | +| `arrays/test_array_results.py::test_array_results_follow_data_buffer_and_descriptor_handle_contracts[*]` | ordinary/allocatable result generation unit; `wrapper-plan` | Phase 6 ordinary and Phase 7E allocatable results now share the production plan route | +| `build_from_pyi/test_pyi_wrapper_builds.py::test_pyi_manifest_records_pointer_descriptor_interop_requirements` | non-generating manifest policy; `not-applicable` | Phase 7G plan/header union is covered by direct generated-artifact tests | +| `derived_types/test_pointers.py::test_module_and_derived_pointer_handles_track_native_association[*]` | module, normal-array actual, and field mix; `legacy` | split Phase 7A/7F module subsets; field subset remains Phase 8/9 | +| `derived_types/test_pointers.py::test_pointer_descriptor_views_preserve_slice_shape_strides_and_parent_lifetime` | module/field descriptor views; `legacy` | Phase 7B/7G module subset; field owner remains Phase 8/9 | +| `derived_types/test_pointers.py::test_pointer_array_handles_block_on_unsupported_result_owner_policy[*]` | explicit supported blocker; `legacy` | remain blocker until owner/lifetime policy changes; never auto-promote | +| `edit_pyi_contracts/test_ownership_contracts.py::*` | module, field, result lifetime mix; `legacy` | Phase 7E/7F subsets; field/finalizer owners remain Phase 8/9 | +| `function_calls/test_optional_arguments.py::test_optional_allocatable_scalar_descriptor_distinguishes_omitted_none_and_value` | scalar baseline; `wrapper-plan` | reuse Phase 3 behavior; no status change | +| `function_calls/test_output_arguments.py::test_output_arguments_and_multiple_results_follow_python_projection_rules[*]` | mixed scalar/array/string/derived/allocatable outputs; `legacy` | Phase 7E reduced allocatable result; retain mixed row | +| `module_state/test_allocatable_replacement.py::*` | projected same-handle descriptor mutation plus a derived factory generation unit; `legacy` | Phase 7D reduced parity is `wrapper-plan`; the broad factory/class unit remains Phase 8/9 | +| `module_state/test_allocatable_views.py::test_allocatable_module_fields_and_results_expose_lifetime_safe_handles[*]` | module, result, and derived-field mix; `legacy` | field/class owner retention remains Phase 8/9 | +| `module_state/test_allocatable_views.py::test_scalar_descriptor_module_variables_return_copied_optional_values[*]` | scalar descriptor arguments/results/module state; `wrapper-plan` | source conversion records descriptor kind and argument/return reference before completed Phase 7H policy | +| `module_state/test_allocatable_views.py::test_plain_allocatable_module_array_exposes_current_live_view[*]` | corrected source/generated-`.pyi` production-plan evidence | proves Phase 7F plain/`Aliased` current live-view or `None` parity, native mutation, explicit-copy independence, and fresh extraction after state changes | +| `strings/test_character_arguments.py::test_modern_fortran_character_arguments_and_results[*]` | fixed strings plus deferred allocatable result; `legacy` | Phase 7H reduced deferred-result parity; retain mixed row as needed | +| `edit_pyi_contracts/test_native_order_contracts.py::test_editable_contract_can_use_native_order_arguments_without_native_call` | includes raw `Addr(Float64[n])` plus a derived result; `legacy` | raw-array subset is the completed Phase 6G prerequisite; derived subset remains Phase 8 | + +| Completed sub-lane | Dependency-closed compiled evidence | +| --- | --- | +| Phase 7A, 7B, 7F, and 7G | `derived_types/test_pointers.py::test_module_native_array_handles_use_canonical_plan` | +| Phase 7C | `function_calls/test_optional_arguments.py::test_optional_array_descriptors_preserve_presence_and_storage_state` | +| Phase 7D | `module_state/test_allocatable_replacement.py::test_projected_allocatable_descriptor_preserves_same_handle_identity` | +| Phase 7E numeric | `arrays/test_array_results.py::test_owned_allocatable_results_preserve_handle_state` | +| Phase 7E deferred character | `strings/test_character_arguments.py::test_deferred_character_array_handles_use_canonical_plan` | +| Phase 7H numeric | `scalars/test_scalar_boundary_plan.py::test_scalar_descriptor_results_copy_values_or_none_through_wrapper_plan_route` | +| Phase 7H deferred scalar character | `strings/test_character_arguments.py::test_deferred_allocatable_string_results_use_canonical_plan` and the nullable case in the deferred-character handle test | +| Phase 7H source/default projection | `module_state/test_allocatable_views.py::test_scalar_descriptor_module_variables_return_copied_optional_values[*]` | + +Required focused intermediate coverage includes: + +- completed semantic handle/interop policy projection tests; +- editable plan tests for every descriptor kind, handoff form, operation set, + ownership, optional presence state, and lifecycle edit; +- C and Fortran preflight rejection of mismatched or incomplete descriptor + plans; +- generated artifact assertions for standard descriptor fields, optional + presence, operation functions, owner storage, destroy paths, and local header + requirements; +- runtime helper tests under `tests/runtime/handles` without duplicating their + validation in wrapper-codegen tests; and +- compiled legacy/direct parity for each reduced sub-lane before any migration + ledger or production-route change. + +### Phase 7 Completion + +- [x] Expand Phase 7 under the mandatory gate using the live completed policy, + runtime handle implementation, legacy backends, build integration, public + contract, and focused wrapper tests. +- [x] Complete the shared typed handle, array-actual, and descriptor-handoff + plan records without adding a parallel top-level plan hierarchy. +- [x] Complete every missing semantic selector before `ir2ast.py`; remove + bridge-created `ArrayInteropPolicy` and fabricated semantic ownership choices. +- [x] Finish Phases 7A through 7H individually with legacy artifact capture, + direct lowering, validation, compiled parity, route evidence, and matrix + updates. +- [x] Preserve the completed Phase 6G raw-address boundary while keeping every + derived-field, pointer-result, callback, and deferred-real-library exclusion + on its explicit later blocker. +- [x] Complete the Phase 7F view-only correction for plain and `Aliased` + allocatable module arrays and remove every implicit-copy extraction path. +- [x] Rerun focused policy/plan/backend tests, relevant runtime-handle tests, + documentation checks, the wrapper suite excluding LAPACK, the wrapper-codegen + complexity checker, and the required static-analysis suite after the + correction. +- [x] Close Phase 7 only when every live non-field native-handle/descriptor case + is migrated or explicitly removed from the product contract, and no backend + infers descriptor policy or silently substitutes a data buffer, raw pointer, + `.to_numpy()` extraction, or copy fallback. + +Historical pre-correction evidence: 538 focused semantic/policy/plan/backend/ +runtime tests, 1,133 documentation and layout tests, and all 318 wrapper tests +outside the deferred BLAS/LAPACK file passed. The wrapper-codegen complexity +check, Ruff, formatting, Bandit, Vulture, whitespace, and explicit-base Radon +policy also passed. This evidence remains valid for unaffected sub-lanes but is +not closure evidence for the changed view-only extraction contract. Record a +new success signal after the Phase 7F correction. + +Post-correction closure evidence (2026-07-14): 214 focused runtime-handle, +policy, readiness, lowering, legacy-dispatch, and Phase 7 direct-plan tests; +199 complete `tests/wrapper_codegen` tests; 1,123 documentation tests; 317 +wrapper tests outside the shared real-library parameter plus the BLAS-only +parameter; and zero locally executed LAPACK tests all passed. The wrapper +complexity checker, Ruff lint/format, Bandit, Vulture, whitespace, and the +explicit-`origin/main` Radon policy passed. The required `--base-ref auto` +Radon invocation could not resolve a CI base SHA locally; the explicit base +rerun passed. Advisory full Radon complexity and maintainability reports were +also produced. + +Phase 7 was re-verified again with the final Phase 8 closure run on +2026-07-15: the 704-test semantic/policy/plan/backend regression batch, all 79 +runtime-handle tests, 1,123 documentation tests, and all 326 wrapper tests +outside LAPACK passed. No LAPACK test was run locally. + +## Phase 8 — Derived Types And Object Lifetimes + +Expansion status: complete. Implementation proceeds only after the Phase 7 +view-only correction is re-verified. + +Implementation status: reopened for the complete rank-zero scalar-derived +actual/dummy compatibility matrix in Phase 8H. The previous Phase 8A-I +evidence remains authoritative for unaffected fields and lifecycle paths, but +the old module-allocatable rejection, nonreassociating pointer-only path, +interoperable-only value restriction, and incomplete module-object call routes +are superseded. Phase 8 must not close again until direct, scoped-address, +wrapper-holder, module-transaction, pointer-input, and typed-value actions are +implemented without a fallback and re-verified with multi-argument calls. + +Scope: migrate scalar derived-type storage, arguments, results, borrowed +objects, and field handoffs into the wrapper-plan route. Phase 8 owns the +opaque native-instance substrate and the typed transfers that use it. Phase 9 +owns public constructors, methods, overloads, inheritance, and general +class-surface orchestration built on that substrate. Phase 8 owns public field +descriptors and their typed getters/setters because every live object origin, +including plain module proxies, needs the same readable and writable member +surface. + +Do not begin implementation while a Phase 7 native-array-handle correction is +open. In particular, Phase 8 must consume the final view-only `to_numpy()` +contract: an array-handle extraction is a live view or `None`, and an +independent array is obtained with an explicit `.copy()`. + +`Snapshot[T]` is no longer an active public contract. Plain and `Aliased` +rank-zero derived module variables both expose the normal live generated object +surface. Their lowering mechanisms remain distinct: `Aliased` proves a direct +address-backed borrow, while a plain declaration requires typed module-specific +bridge access and must not fabricate a native address. `Aliased` remains an +addressability and aliasing fact for raw-address legality, pointer association, +C-pointer policy, and direct derived-object handoff; it does not select +array-handle `to_numpy()` behavior. + +### Phase 8 Boundary And Explicit Non-Scope + +The first implementation slice is rank-zero, non-polymorphic derived values. +The runtime wrapper is opaque: the binding carries a native address, ownership +state, and an optional retained Python owner, while the bridge performs typed +native association, assignment, allocation, and destruction. The binding must +not depend on component offsets or reproduce native aggregate layout. + +The following surfaces are in Phase 8: + +- required and optional scalar derived arguments; +- visible `out` and `inout` wrappers whose identity remains caller-visible; +- hidden output and direct-function-result values materialized as + wrapper-owned instances; +- native `value` arguments for an exact rank-zero monomorphic derived type, + using a Fortran bridge-owned typed value copy rather than C-side layout + inference; the native type need not be `bind(C)` when the bridge imports its + exact definition; +- derived `parameter` and other explicit constant-value origins materialized + through the existing wrapper-owned immutable-value path, never as a fallback + for an ordinary mutable module object; +- plain rank-zero native module objects exposed as live module-backed proxies + through typed bridge operations; +- `Aliased` rank-zero native module objects exposed as live borrowed wrappers; +- borrowed nested component wrappers, their public field descriptors, and the + owner-retention facts required by those descriptors; +- Phase 7 allocatable/pointer field-handle plans attached to a derived owner; +- exact destruction, finalization, cleanup, and failure ownership for each of + those origins. + +The following remain outside Phase 8: + +- public default/keyword constructors, explicit `@bind(...)` constructors, + `tp_init`, methods, static methods, overload dispatch, Python inheritance, + and ordinary type-bound surface assembly; these remain Phase 9. Public field + descriptors, getters, and setters are Phase 8 and are not a Phase 9 blocker. + A generated semantic `.pyi` field constructor is therefore a whole-unit + Phase 9 blocker; only an opaque contract that suppresses default construction + may use the direct Phase 8 object route; +- scalar polymorphic dispatch and inheritance even where the legacy route + supports them; Phase 9 owns the class relationship needed to validate the + accepted runtime type set; +- callback-derived arguments and results, adapter procedures, and trampoline + ownership; these remain Phase 10; +- arrays of derived values, whose element layout, construction, destruction, + copy, and partial-failure behavior remain explicit readiness blockers; +- polymorphic results, mutable polymorphic arguments, `class(*)`, abstract + instantiation, deferred bindings, and allocatable/pointer polymorphic + scalars; +- polymorphic descriptor-backed scalars. Wrapper-owned allocatable and pointer + holders plus scalar derived module ordinary/`TARGET`/allocatable/pointer + variables are supported only by their explicit Phase 8H matrix rows. A + pointer holder owns its association container, never its target by default; + target retention and native release responsibility are completed separately + before lowering. Module allocation and pointer transactions use shared typed + holder addresses in interoperable callbacks, never CFI or a compiler-private + descriptor; +- any other derived origin that cannot use one of the explicit matrix rows. It + remains blocked rather than being silently turned into an address-backed + borrow or detached object; +- C-side aggregate casts, `ctypes` layout promises, compiler-private descriptor + inspection, or direct component offsets; +- ownership of targets reachable through pointer components. A containing + derived wrapper does not own such a target without completed pointer policy; + and +- detached whole-object snapshot classes or recursive member-copy graphs. They + are removed rather than retained as a compatibility path. + +### Public Representation And Lifetime Matrix + +Complete this matrix in post-IR policy before adding planner or backend code. +The rows are distinct origins, not datatype guesses made during lowering. + +| Surface | Python representation | Native handoff/storage | Owner and release | +| --- | --- | --- | --- | +| required `in` argument | existing wrapper instance | pass its opaque wrapper address and associate a typed native view for the call | wrapper remains owned by its existing Python object; call destroys nothing | +| required visible `inout` or caller-supplied `out` | same wrapper instance | pass the same address for native mutation | caller-visible wrapper retains identity; its normal wrapper finalizer remains the sole destroyer | +| optional argument, omitted or `None` | no wrapper instance | explicit absence token/branch; no fabricated native object | no allocation or cleanup | +| optional argument, present | validated wrapper instance | same typed address handoff as the required case | existing wrapper owner remains responsible | +| hidden output | new opaque wrapper object | allocate persistent wrapper-owned native storage before the call and pass its address | wrapper deallocator invokes native-aware destruction exactly once | +| direct function result | new opaque wrapper object | move or copy the native result before its temporary expires into persistent wrapper-owned storage | wrapper deallocator invokes native-aware destruction exactly once | +| constructor-created instance | Phase 9 only | Phase 9 must allocate through the same persistent wrapper-owned storage and native-aware destructor established here | explicitly blocked until Phase 9 class construction orchestration; no Phase 8 fallback constructor | +| native `value` input | existing wrapper instance | exact Fortran bridge passes the typed pointee to the native by-value slot; C never lays out or copies the aggregate | call-local native copy only; wrapper ownership is unchanged | +| plain rank-zero module variable | normal live generated object | module-specific typed getter/setter operations plus a synchronous scoped-address consumer when the object is passed to another procedure | native module owns storage; wrapper retains the module and never destroys storage; a temporary target/address cannot escape its consumer scope | +| `Aliased` or explicit `TARGET` rank-zero module variable | live borrowed wrapper | use `C_LOC` as the sole whole-object handoff; reconstruct the exact typed bridge view without copying | native module owns storage; wrapper never destroys it and rejects replacement | +| derived `parameter` or other explicit constant-value origin | wrapper-owned value copy with an immutable module binding | materialize the native value into persistent wrapper-owned storage | wrapper destroys only its materialized copy; no native module setter; normal writable fields modify only that independent copy | +| nested derived field | live borrowed child wrapper | address/alias of the component through the parent wrapper | child retains parent; child never destroys component storage | +| allocatable scalar derived module origin | nullable live module-backed proxy carrying its runtime origin | scoped-address consumer for payload-only calls; for an allocatable dummy, module-specific interoperable operations move between the module variable and a bridge-local shared typed holder addressed by `C_PTR` | native module owns storage before and after a transaction; successful move-out has exactly one reverse-order move-back; no descriptor crosses C | +| pointer scalar derived module origin | nullable live module-backed proxy carrying its runtime origin | current-target address for payload-only calls; for a pointer dummy, a bridge-local shared typed pointer holder receives the initial association and its address is passed to the module-specific restore operation | native module owns the pointer variable and, by default, its target; final association is restored exactly once after a normally returning native call | +| wrapper-owned allocatable scalar derived result | nullable live generated wrapper backed by one persistent typed allocatable holder per native type | result is moved into `holder%value`; ordinary, target, allocatable, allocatable-target, pointer-input, and value dummies use the explicit compatible matrix actions | each Python wrapper owns one target-capable holder and destroys it exactly once; allocation-state writeback preserves wrapper identity | +| wrapper-owned pointer scalar derived result | nullable live generated wrapper backed by one persistent typed pointer holder per native type | holder component stores current association and is passed directly to a compatible pointer dummy; payload-only calls use its associated target | wrapper owns and destroys only the holder; target ownership stays native unless completed policy retains a known wrapper/module target; destruction nullifies the component and never deallocates an unowned target | +| detached whole-object snapshot | removed | no recursive copy graph or snapshot helper is generated | no compatibility parser, lowering, or fallback; read the live object through normal fields instead | + +`Aliased` remains a public, language-neutral addressability/aliasing fact and +must survive parsing, semantic IR, and printing. For derived module objects it +distinguishes direct-address lowering from module-proxy lowering, not live +versus copied public behavior. It must never be reused to select live versus +copied native-array-handle extraction. + +### Existing Semantic Authority And Legacy Oracle + +Use the current implementation as an oracle, not as permission to preserve its +architecture: + +- `x2py/semantics/ownership.py` already names `DERIVED_TYPE`, + `PASS_WRAPPER_ADDRESS`, `WRAPPER_INSTANCE`, and `BORROWED_VIEW`, and contains + the current argument/result/module/field owner defaults. Remove the obsolete + derived whole-object snapshot action without disturbing ordinary result + copies, scalar descriptor value copies, or explicit non-object uses of + `snapshot_copy` transfer policy. +- `x2py/semantics/policy_completion.py` is the only allowed owner of origin, + ownership, transfer, destruction, mutability, nullability, projection, + release, storage, getter/setter, owner-retention, module-object handoff, and + field decisions. It must complete module-proxy policy for plain module + objects and direct-address borrowed policy for `Aliased` module objects. +- `x2py/semantics/wrapper_policy.py` must gain a derived-specific policy branch. + Derived values must not continue through primitive-scalar blockers, + primitive result checks, or primitive bridge data-action selection. +- `x2py/semantics/ir2ast.py` and the legacy generators remain the generated + artifact oracle. Direct lowering must not call `semantic_ir_to_codegen_ast()` + or reconstruct legacy codegen variables. +- `x2py/codegen/bindings/c_to_python.py` contains the existing wrapper-instance + conversion, checked casts, owned/borrowed result construction, owner + retention, and allocator/destructor helpers. Remove recursive snapshot + construction rather than migrating it into the direct route. +- `x2py/codegen/bridges/fortran_to_c.py` contains the existing typed wrapper + address conversion, native result materialization, borrowed field/module + access, native-aware destruction, and typed component getters/setters. Reuse + those live member-access mechanics as the artifact oracle while moving every + decision into typed plans. + +Capture complete legacy artifacts before each direct slice. Preserve observable +runtime behavior while replacing backend inference with completed typed plans. +Do not copy the broad legacy generator control flow into `wrapper_codegen`. + +The existing wrapper tests decompose as follows: + +| Existing test or generation unit | Phase 8 oracle | Required split or later owner | +| --- | --- | --- | +| `function_calls/test_native_call_examples.py::test_native_call_examples_cover_scalar_array_string_and_object_projection[*]` | hidden derived result selected by `Return(...)` | add a reduced object-result entry; retain the mixed unit until every included lane is direct | +| `function_calls/test_output_arguments.py::test_output_arguments_and_multiple_results_follow_python_projection_rules[*]` | hidden derived output and mixed result aggregation | add a reduced derived-output entry; retain the broad unit until its complete tuple is direct | +| `edit_pyi_contracts/test_policy_dispatch_contracts.py::test_immutable_scalar_string_array_and_derived_policies_return_replacements` | edited projected derived replacement | isolate `make_point` as Phase 8 evidence; retain the mixed policy unit until whole-unit eligibility follows | +| `derived_types/test_derived_type_boundaries.py::test_scalar_derived_types_cross_procedure_boundaries[*]` | required input, in-place mutation, hidden/direct result, nested borrowed component | reduce first to result-created opaque objects passed back to `point_sum`/`move_point`; field descriptors and nested borrowing are Phase 8, while construction remains Phase 9 | +| `function_calls/test_optional_arguments.py::test_optional_arguments_drive_fortran_present_behavior[*]` | optional derived input and exact type/absence behavior | complete the optional transfer in Phase 8; the existing constructor-dependent broad runtime unit remains Phase 9 until it can route whole | +| `module_state/test_module_state.py::test_aliased_derived_module_object_borrows_native_state[*]` | native-owned borrowed module object and replacement rejection | use as the direct-address oracle; add a reduced plain-module proxy case with the same live field behavior; methods remain Phase 9 | +| `derived_types/test_borrowed_finalizers.py::test_borrowed_child_wrapper_never_finalizes_native_component[*]` | parent retention and exactly-once owner finalization | Phase 8 owns storage/lifetime plans and public field descriptors; constructor/method orchestration remains Phase 9 | +| `module_state/test_allocatable_views.py::test_allocatable_module_fields_and_results_expose_lifetime_safe_handles[*]` | Phase 7 field handle attached to a derived owner | reuse the existing `NativeArrayHandlePlan` and expose its public property in Phase 8 | +| `derived_types/test_pointers.py::test_pointer_descriptor_views_preserve_slice_shape_strides_and_parent_lifetime` | pointer field handle and parent lifetime | reuse Phase 7 descriptor extraction; do not move pointer target ownership into Phase 8 | +| `derived_types/test_derived_layout.py::test_bind_c_derived_types_use_accessors_and_fortran_value_copy[*]` | opaque `bind(C)` wrapper, field accessors, and typed native `value` copy | Phase 8 owns the handoff and field properties; constructor orchestration remains Phase 9 | +| former `module_state/contracts/fmodule_derived_snapshot_f90/` snapshot fixture | obsolete detached-object behavior | remove the `Snapshot[box]` fixture and snapshot-only runtime assertions; reuse the native unit only for reduced live module-proxy evidence where applicable | +| `derived_types/test_constructors_and_finalizers.py::*`, `derived_types/test_derived_type_methods.py::*`, and `derived_types/test_inheritance.py::*` | owned-instance finalizer and type facts may inform Phase 8 | production migration remains Phase 9 because the observable unit is constructor/method/property/inheritance owned | +| `callbacks/test_derived_callbacks.py::*` | none | remain Phase 10 even after ordinary derived transfers are complete | + +The plain non-target module-object row has one recorded intentional correction: +the legacy whole-object getter attempts `c_loc` on storage without the required +addressability property and therefore has no passing whole-object artifact. +Phase 8 uses the real source declaration, the passing legacy typed component +getters/setters, and the passing `Aliased` direct-address behavior as its +mechanical oracles, then improves the plain path to a typed member proxy. The +compiled Phase 8 evidence asserts that this proxy never emits a fabricated +whole-object `c_loc` while its reads and writes remain live. + +### Mandatory Phase 8 Migration Algorithm + +Apply this same sequence to every dependency-closed sub-lane. A checked item in +a later step cannot compensate for an incomplete earlier step. + +1. Capture the complete generated artifacts and runtime assertions from one + real passing legacy/source case. Record which constructor/method or + callback assertions remain outside the reduced unit. +2. Complete object kind, origin, ownership, transfer, destruction, mutability, + nullability, projection, storage, release, owner retention, getter/setter, + native assignment, and any module-object mechanism before `ir2ast.py`. +3. Project those facts mechanically into `ArgumentTransferPlan`, `ResultPlan`, + or `ModuleVariablePlan`, with native slots and lifecycle actions remaining + subordinate references. +4. Validate type identity, roles, actions, owners, storage, result positions, + releases, and cross-backend handoffs before either backend emits source. +5. Lower through small named binding and bridge methods selected by typed + object kind and action. Backend-local temporaries remain implementation + details inside the already selected method. +6. Compare binding, bridge, header, and build artifacts with the captured + oracle and explain every intentional difference before compiling. +7. Add focused policy, plan-edit, validation, printer, backend, runtime, + documentation, and source/generated-`.pyi` parity tests. +8. Promote the reduced generation unit only after compiled legacy/direct parity + passes; otherwise retain one exact blocker without a fallback route. + +The maintainer trace is therefore always: + +```text +completed semantic facts + -> typed argument/result/module-variable plan + -> subordinate native slots and lifecycle actions + -> validation + -> binding and bridge lowering + -> generated artifacts + -> compiled runtime evidence +``` + +### Plan Shape And Stable Action Vocabulary + +Do not create a second function plan, a second result hierarchy, or a rendered +derived-plan layer. Extend the existing plan tree as follows: + +- add one explicit derived datatype family or equivalent non-primitive marker + so a derived semantic type never indexes the primitive scalar dtype maps; +- add a concise namespace-owned derived-type definition record containing + canonical semantic/native identity, native scope, Python exports, opaque + runtime type symbol, allocation role, destruction/finalization role, and the + minimal field identities needed by later field plans; +- add one `DerivedHandoffPlan`-style facet, analogous to `ArrayHandoffPlan`, to + `ArgumentTransferPlan`, `ResultPlan`, `ModuleVariablePlan`, and the owning + `NativeCallSlotPlan` only where that transfer needs it; +- give a derived `ModuleVariablePlan` one typed module-object access facet that + records the completed direct-address or opaque-callback mechanism, its + context/address roles, compiler capability, and module-lifetime owner. This + mechanism is subordinate to the module-variable policy and must not change + its public borrowed-wrapper facts; +- keep native slot order, symbolic roles, and ABI positions subordinate to the + owning argument or result transfer; +- represent result destruction, failed-construction cleanup, parent retention, + through transfer-owned `LifecycleActionPlan` records in function-wide + execution order; +- add field-handoff records beneath the owning derived-type definition. Do not + put their ownership decisions into a backend registry; and +- keep `FunctionPlan`, `ModulePlan`, namespace assembly, result ordering, GIL + envelope, and status-error behavior stable. + +Reuse the existing action vocabulary: + +- Python boundary: `WRAPPER_INSTANCE` for accepted live wrapper objects and + `NONE` for native-produced results; +- native boundary: `PASS_WRAPPER_ADDRESS` for opaque live objects and `NONE` + when the bridge itself owns result production; module-backed proxies use a + distinct typed module-origin handoff rather than a fabricated address; +- transfer/codegen: `CALL_LOCAL_INPUT`, `IN_PLACE_ARGUMENT`, + `IDENTITY_OUTPUT`, `WRAPPER_INSTANCE`, and `BORROWED_VIEW` according to the + completed matrix row; +- bridge data: `DIRECT_TRANSFER`, `ASSOCIATE_VIEW`, or + `COPY_REPRESENTATION`, with a completed copy reason only when a real native + representation copy occurs; and +- lifecycle: existing ordered copy-in/native-mutation/copy-out/cleanup phases, + extended only with a genuinely missing release phase/action rather than a + derived-only parallel lifecycle system. + +If one of these actions cannot express a required operation, document the +missing semantic distinction before adding exactly one typed action. Do not use +method-name strings, datatype conditionals, `intent`, `is_alias`, dotted-name +shape, or local temporary existence as hidden dispatch. + +### Binding, Bridge, And Validation Ownership + +| Layer | Owns | Must not own | +| --- | --- | --- | +| post-IR policy | origin, dynamic/static type allowance, owner, transfer, destruction, mutability, projection, nullability, storage, owner retention, getter/setter behavior, and blockers | emitted local names or source syntax | +| wrapper planner | mechanical projection into derived type/handoff facets, native roles, ordered results, and lifecycle indexes | new ownership or lifetime decisions | +| binding lowering | exact Python type checks, opaque wrapper address extraction, Python wrapper allocation, retained-owner references, result aggregation, and Python reference cleanup | native component layout, native assignment, or native finalization semantics | +| bridge lowering | typed association from opaque addresses, exact Fortran-owned `value` calls, native instance allocation/assignment, module/component access, and native-aware destruction/finalization | Python classes, C aggregate layout, reference counting, detached-copy fallback, or ownership inference | +| plan validation | matching type identity, roles, actions, owners, releases, result positions, and cross-backend handoffs before emission | fallback selection | + +Validation must reject at least: + +- a derived transfer without canonical type identity or an exported runtime + wrapper type; +- a wrapper-address slot whose binding and bridge roles or ABI positions differ; +- a primitive scalar action or datatype family applied to `DERIVED_TYPE`; +- a wrapper-owned result without persistent storage, allocator, destroy action, + or failure cleanup; +- a borrowed wrapper with a destroy action, or without its required native + module/parent owner retention; +- a call-local argument that schedules destruction of the caller's wrapper; +- a visible in-place argument projected as a replacement without completed + policy; +- a hidden output or direct result whose native temporary can escape by + address; +- a plain module proxy without complete typed member-path operations, or an + `Aliased` live module borrow without a completed direct-address handoff; +- binding and bridge module-object access roles that disagree; +- an obsolete `Snapshot` contract, recursive detached-copy action, or backend + fallback that manufactures a detached object; +- a derived array or unsupported polymorphic form entering scalar-derived + lowering; and +- any backend request to infer a class, owner, addressability, or release from + semantic datatype or `intent`. + +### Phase 8A — Contract, Origin, And Post-IR Policy Completion + +Complete the semantic contract before defining direct plan records. + +- [x] Inventory every live scalar derived origin from source and semantic + `.pyi`: constructor-created storage, wrapper-owned result, caller-supplied + argument, native module object, and nested component. +- [x] Introduce one typed completed origin/retention representation shared by + class-instance, argument, result, module-variable, and field policy. + Do not encode origins as ad hoc reason strings. +- [x] Keep generated and edited `.pyi` type identity stable across module + namespaces, imported derived types, renamed Python exports, and same-name + types from different native scopes. +- [x] Complete required, optional, visible `out`, visible `inout`, hidden + output, and direct-result ownership without treating `intent` as the final + Python signature. The editable signature and `@native_call(...)` projection + decide visibility and order; policy only ensures the native call is valid. +- [x] Complete wrapper-owned result storage and destruction, borrowed + module/field owner retention, native setter rejection, result projection, + and failure cleanup before `ir2ast.py`. +- [x] Preserve `Aliased` parsing, printing, and source-derived metadata. Use it + for a live derived-module borrow and direct-address legality, but never as a + native-array extraction mode. +- [x] Complete a plain ordinary module object as `owner=NATIVE`, + `transfer=BORROWED_VIEW`, native-owner destruction, module lifetime, module + owner retention, typed member-path access, and replacement rejection. Do not + claim or require a whole-object native address. +- [x] Complete an `Aliased` module object as `owner=NATIVE`, + `transfer=BORROWED_VIEW`, native-owner destruction, alias storage, module + owner retention, direct address acquisition, and replacement rejection. +- [x] Remove the obsolete public `Snapshot` keyword from `x2py.contracts`, + parser, printer, generated `.pyi`, semantic IR, policy actions, legacy + generators, documentation, and fixtures. Do not remove unrelated explicit + copy-result or scalar descriptor value-copy policy. +- [x] Complete finite typed member-path traversal for plain module proxies. + Memoize derived type identities so recursive graphs do not expand forever; + require explicit pointer/allocatable association, ownership, and stale-child + policy at recursive descriptor-backed edges. +- [x] Remove the obsolete wrapper-owned pointer-result blocker. Complete a + persistent typed pointer-holder origin whose wrapper owns the holder but not + its target, then keep only arrays of derived values and unsupported + polymorphic forms on exact readiness blockers. Supported scalar module + allocatable/`TARGET`/pointer origins use only their explicit Phase 8H + actions. +- [x] Add focused parser, printer, source-conversion, ownership, accessor, + policy-completion, and readiness tests for every active matrix row and + blocker. Assert the deliberate module-proxy versus direct-address mechanism + distinction, their shared live public behavior, and that neither changes a + contained native handle's view-only extraction. + +### Phase 8B — Derived Plan Records And Preflight Validation + +- [x] Add the minimal namespace-owned opaque derived-type definition record and + derived handoff facets described above. Keep all per-call decisions in + `ArgumentTransferPlan`, `ResultPlan`, or `ModuleVariablePlan`. +- [x] Add an explicit derived datatype-family/type-reference representation so + documentation, roles, native slots, lifecycle records, and printers never + fall through primitive scalar maps. +- [x] Project class instance/self policies, native type identity, wrapper type + symbol, native scope, allocator/destroy roles, and finalizer requirements + mechanically from completed semantic policy. +- [x] Project optional presence, input/in-place/output action, native call + position, ownership, storage, owner retention, and result position into the + existing transfer records. +- [x] Share the exact `DerivedHandoffPlan` object with its owning + `NativeCallSlotPlan` where the array/handle lanes already share subordinate + facets; do not duplicate editable state. +- [x] Add recursive validation for the namespace type definitions, arguments, + results, module variables, module-object access facets, field facets, and + lifecycle indexes. +- [x] Make plan edits observable: changing a derived owner, action, type + identity, retained owner, or release must either change both backend + artifacts consistently or fail `_validate_plan()` before source emission. +- [x] Extend support analysis with precise derived lanes and blockers. Do not + remove the blanket class-owner blocker until the minimal opaque type surface + is direct and every remaining Phase 9 dependency is reported separately. +- [x] Add normal-print plan tests and direct generator preflight tests under + `tests/wrapper_codegen/test_phase8_derived_types.py`. + +### Phase 8C — Minimal Opaque Wrapper Storage And Lifecycle + +This sub-lane creates the runtime substrate needed to return and pass opaque +objects. It does not implement public construction, fields, or methods by +itself; Phase 8F/H add the public field surface on this substrate. + +- [x] Emit one minimal runtime wrapper type per exported semantic derived type, + with an opaque native address, an owned/borrowed state, and an optional + retained Python owner. Keep the public constructor unavailable until Phase 9. +- [x] Generate bridge allocation and destruction helpers from completed type + policy. Native-aware destruction owns allocatable components and supported + finalization; the binding must not free native storage directly. +- [x] Ensure owned allocation, initialization, and result conversion failures + run native destruction and Python cleanup exactly once. +- [x] Ensure borrowed wrappers never run native destruction, including when + their retained owner is released through cyclic or delayed garbage + collection. +- [x] Register the minimal type in the correct exported namespace so result and + module-variable materialization use the same class identity in source and + generated-`.pyi` builds. +- [x] Keep wrapper struct/type declaration, allocation, owner retention, and + destruction methods grouped under a derived-type comment in the binding; + keep native allocate/associate/destroy helpers grouped likewise in the + bridge. +- [x] Add source-printer and artifact tests for owned, borrowed, failed + allocation, failed conversion, and exactly-once native destruction paths. + +### Phase 8D — Wrapper-Owned Hidden Outputs And Function Results + +- [x] Plan hidden `Return(...)` outputs and direct derived function results as + `WRAPPER_INSTANCE` results with persistent wrapper-owned native storage. +- [x] For hidden output, allocate the result wrapper before the native call and + pass its native address at the declared native slot. On failure, destroy it + before returning the Python error. +- [x] For a function result, move or copy the returned native value into + persistent wrapper-owned storage before the native temporary expires. Never + retain an address into a bridge local. +- [x] Preserve result order and mixed-result aggregation through the existing + `ResultPlan` and lifecycle sequence; do not special-case a derived result in + function/module orchestration. +- [x] Reuse the same result type object and destructor for direct results, + hidden outputs, and edited `Returns[...]` projections. +- [x] Add reduced legacy/direct compiled parity over the existing + `make_point` cases in `test_native_call_examples.py`, + `test_output_arguments.py`, and `test_derived_type_boundaries.py`, inspecting + result storage, slot order, allocation failure, and cleanup artifacts. +- [x] Promote only those reduced generation units after both source and + generated-`.pyi` routes return the correct opaque wrapper and finalization is + proved. Field-based assertions remain on Phase 8F/H until their typed member + operations are complete. + +### Phase 8E — Required, Optional, In-Place, And Caller-Supplied Outputs + +- [x] Accept only the exact completed wrapper type for a concrete derived + argument. Subclass acceptance belongs to completed Phase 9 polymorphic + policy, not normal Python `isinstance` convenience. +- [x] Extract the opaque native address in the binding and pass it through the + single planned role. The bridge associates the matching typed native pointer + and calls the native procedure without copying for ordinary reference + arguments. +- [x] Preserve the same Python wrapper identity for visible `inout` and + caller-supplied `out` arguments. Return it only when the edited projection + requests that sole result; otherwise return `None`. Keep a mixed direct or + hidden result plus visible derived writeback on an exact policy blocker until + general mixed result/writeback aggregation is completed; do not let the + direct route select it and then drop the wrapper identity. +- [x] Represent optional omission and explicit `None` as native absence. A + present wrapper follows the same typed handoff as a required input; no empty + wrapper or call-local default object may be fabricated. +- [x] Keep native slot order independent of normalized Python argument order + and preserve user edits to argument visibility and projection. +- [x] Keep an immutable visible derived replacement on its existing exact + blocker because no passing legacy contract defines its native copy and + finalization semantics. Existing hidden/direct derived outputs use the owned + result path completed in Phase 8D; do not mutate an immutable input or invent + a generic object copy merely to remove the blocker. +- [x] Add focused type-error, optional-presence, wrong-wrapper-class, + in-place-identity, caller-supplied-output, projection, and cleanup tests. +- [x] Add reduced compiled parity that creates a `point` through the Phase 8D + result path, passes it to `point_sum`, mutates it through `move_point`, and + observes the new value through another native call without requiring a + constructor; the follow-on Phase 8F evidence also observes public fields. + +### Phase 8F — Module Objects, Components, And Field Owners + +- [x] Plan every eligible plain rank-zero derived module variable as a + native-owned live module proxy with rejected replacement; plan every + supported `Aliased` equivalent as a native-owned direct-address borrowed + wrapper. Both retain the module and have no destroy action. +- [x] Preserve `Aliased` in generated semantic `.pyi` only when supplied by the + native/source contract. Prove its module-proxy-versus-direct-address lowering + meaning while separately proving that both are live and that it does not + affect any contained native handle's view-only extraction. +- [x] Repeated `Aliased` module reads may create separate Python wrappers, but + every wrapper must refer to the same native object and never claim ownership; + repeated plain reads may create separate proxies, but every proxy must + delegate to the same current native module object. +- [x] Plan a nested derived component as a borrowed wrapper whose retained + owner is the containing wrapper. Releasing the parent name must not destroy + the parent while a child wrapper remains live. +- [x] Ensure a borrowed child never invokes its own native finalizer; releasing + the final child/owner reference triggers the containing owned instance's + destruction exactly once. +- [x] Reuse the Phase 7 `NativeArrayHandlePlan` for allocatable/pointer fields, + changing only origin=`derived_field`, owner retention=`parent_wrapper`, and + the completed field operation roles. Do not create a derived-only handle. +- [x] Plan scalar, string, ordinary-array, nested-derived, and native-handle + field getter/setter handoffs beneath the owning type for both address-backed + and module-backed objects. Use typed bridge procedures rather than C layout + offsets. Phase 8 emits both the typed low-level operations and public property + descriptors, including setter exposure completed by semantic policy. +- [x] Traverse nested value components by finite member paths and type identity. + Memoize recursive type definitions; recursive pointer/allocatable edges use + their completed association and owner policy instead of unbounded flattening. +- [x] Preserve pointer-field target ownership and stale-view rules from + completed pointer policy; parent retention does not make the parent own an + external pointer target. +- [x] Add direct plan/backend lifetime tests, then reduced compiled evidence + for the distinct plain proxy and `Aliased` direct-address origins, plus the + borrowed-finalizer, allocatable-field, and pointer-field fixtures, without + promoting constructor or method surfaces that remain Phase 9. + +### Phase 8G — Exact Native `value` Copies And Opaque Layout + +- [x] Preserve `bind(C)`/`sequence`/ordinary derived-type facts and native + `value` transport through generated `Value(Arg(i))`, post-IR policy, and the + derived handoff plan. Do not store this per-call ABI choice on the annotated + Python type. +- [x] For every supported exact rank-zero monomorphic native `value` argument, + keep Python on the opaque wrapper contract. The Fortran bridge imports the + exact native type, reads the typed pointee, and performs the typed call. The + binding and C boundary never cast, lay out, or byte-copy the aggregate. +- [x] Remove the obsolete requirement that the native type itself be + interoperable. Ordinary, `sequence`, and `bind(C)` exact derived types use + the same Fortran-owned typed-value action; polymorphic or unresolved native + types remain exact blockers for type-identity reasons, not layout guesses. +- [x] Keep ordinary reference arguments and all component access on generated + bridge helpers even when a type is `bind(C)`; interoperability does not turn + fields into a public binary-layout promise. +- [x] Replace the obsolete unsupported-aggregate-layout assertions with + policy, plan, artifact, and compiled tests for ordinary, `sequence`, and + `bind(C)` exact typed value calls. Field-property assertions are Phase 8 + evidence; retain only constructor-dependent assertions in + `test_derived_layout.py` for Phase 9 production promotion. + +### Phase 8H — Direct-Address And Module-Proxy Object Access + +This sub-lane supplies the distinct lowering mechanisms for the two completed +module-object origins in Phase 8A/8F: direct address acquisition for an +`Aliased` live borrow, and typed live member access for a plain module proxy. + +- [x] Add one typed module-object access facet beneath `ModuleVariablePlan`. + Record `DIRECT_ADDRESS` or `MODULE_PROXY`, the native object type, member-path + operations, owner/release behavior, and failure behavior. Do not encode a + backend method name. +- [x] Use the direct path only when completed source/semantic facts make the + native address legal. The bridge exposes the opaque address mechanically; + the binding constructs the borrowed wrapper and retains its module owner. +- [x] For a plain module object, use typed per-field bridge getters/setters and + operations selected by the completed member graph. The binding constructs a + module-retaining proxy with no native destroy action; every read observes + current module state and every permitted write updates it. +- [x] Keep the initial direct-address and module-proxy paths rank-zero, + nonallocatable, nonpointer, noncoindexed, and nonpolymorphic; the explicit + descriptor-backed correction below adds only its named storage origins and + call actions. Record exact blockers for + unsupported type parameters, dynamic types, unresolved recursive pointer + ownership, or any member without a complete live operation. Do not switch + mechanisms as a fallback. +- [x] Validate direct-address roles or module-proxy member-operation coverage, + exported wrapper type identity, owner/release behavior, and + replacement rejection before either backend emits source. +- [x] Prove the `Aliased` address/lifetime premise and plain proxy live + read/write behavior in focused compiled source/generated-`.pyi` tests. +- [x] Remove the `Snapshot` contract name, metadata, recursive copy policy, + generated helper classes, documentation, and snapshot-only fixtures. Do not + retain a compatibility parser, printer, alias, or backend fallback. +- [x] Preserve ordinary result materialization, explicit constant-value + materialization, scalar descriptor value copies, ordinary array copy + results, and any unrelated active transfer action. Their copy semantics are + separate from removed whole-object snapshot behavior. +- [x] Replace the former plain-module snapshot fixture with source/generated- + `.pyi` parity and runtime evidence for live scalar, string, ordinary-array, + allocatable/pointer-handle, and nested-derived member paths, including + recursive-edge blockers and parent/module retention. + +#### Phase 8H Contract Correction — Complete Scalar-Derived Call Matrix + +This correction replaces every earlier isolated module-allocatable, stable +pointer-target, direct-address-only, and interoperable-value proposal with one +complete compatibility matrix. It covers exact rank-zero, monomorphic +`type(item)` objects. Phase 9 still owns `class(item)`, inheritance, and dynamic +dispatch; arrays of derived values remain outside this matrix. + +The actual declaration and its runtime origin are independent axes. The five +actual declaration forms are ordinary, `TARGET`, `ALLOCATABLE`, +`ALLOCATABLE,TARGET`, and `POINTER`; each can be module-owned or represented by +wrapper-owned storage where such storage is meaningful. The six native dummy +forms are: + +| Key | Exact native dummy | +| --- | --- | +| `O` | `type(item) :: arg` | +| `T` | `type(item), target :: arg` | +| `A` | `type(item), allocatable :: arg` | +| `AT` | `type(item), allocatable, target :: arg` | +| `P` | `type(item), pointer :: arg` | +| `V` | `type(item), value :: arg` | + +`OPTIONAL`, rank, and qualified type identity remain separate facts. Source +`INTENT` may propose the initial Python projection, but it is not a completed +matrix selector. For the `P` column, `Pointer(Arg(i))` without a matching +projected return selects a call-local pointer input adapter and discards native +reassociation. A matching `Returns[...]` selects association writeback and +therefore requires persistent pointer storage. x2py never selects between these +paths from native `INTENT`. + +Use these completed action names. Parenthesized state requirements are runtime +preconditions, not alternative fallback actions: + +| Action | Meaning | +| --- | --- | +| `DIRECT_REFERENCE` | wrapper-owned or direct module address; reconstruct the exact typed object and pass it by reference | +| `SCOPED_REFERENCE` | originating module synchronously invokes a generic address consumer; the native call completes before the temporary target scope returns | +| `HOLDER_REFERENCE` | reconstruct a persistent typed holder and pass its component directly | +| `MODULE_ADDRESS` | originating module returns `C_LOC` for an explicit durable target | +| `ALLOCATABLE_HOLDER` | pass a persistent wrapper-owned allocatable holder component directly, including unallocated state | +| `MODULE_ALLOCATABLE_TRANSACTION` | move between the module variable and a bridge-local shared typed transaction holder through interoperable holder-address operations | +| `POINTEE_REFERENCE` | pass the current target of a pointer holder or module pointer to a nonpointer dummy | +| `POINTER_HOLDER` | pass a persistent wrapper-owned pointer holder component directly so association writeback updates the same holder | +| `MODULE_POINTER_TRANSACTION` | initialize one bridge-local typed pointer holder from the current target and restore its final association through an interoperable holder-address operation | +| `POINTER_INPUT_ADAPTER` | expose a payload through a call-local pointer carrier because the Python contract does not project pointer association writeback | +| `TYPED_VALUE_COPY` | the exact Fortran bridge passes the typed object into the native `VALUE` slot; C never copies aggregate bytes | +| `INCOMPATIBLE` | language-level storage mismatch; raise the specified `TypeError` and never enter native code | + +`[allocated]` means an allocated value is required. `[associated]` means an +associated pointer target is required. `A`, `AT`, and `P` descriptor calls +accept unallocated or disassociated state where the table does not carry one of +those preconditions. + +| Actual declaration | Origin | `O` | `T` | `A` | `AT` | `P` | `V` | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `type(item) :: var` | non-module | `DIRECT_REFERENCE` | `DIRECT_REFERENCE` with call-scoped target | `INCOMPATIBLE` | `INCOMPATIBLE` | `POINTER_INPUT_ADAPTER` | `TYPED_VALUE_COPY` from direct reference | +| `type(item) :: var` | module proxy | `SCOPED_REFERENCE` | `SCOPED_REFERENCE` with call-scoped target | `INCOMPATIBLE` | `INCOMPATIBLE` | scoped `POINTER_INPUT_ADAPTER` | scoped `TYPED_VALUE_COPY` | +| `type(item), target :: var` | non-module | `DIRECT_REFERENCE` | `DIRECT_REFERENCE` with owner target lifetime | `INCOMPATIBLE` | `INCOMPATIBLE` | `POINTER_INPUT_ADAPTER` | direct `TYPED_VALUE_COPY` | +| `type(item), target :: var` | module | `MODULE_ADDRESS` | `MODULE_ADDRESS` with module target lifetime | `INCOMPATIBLE` | `INCOMPATIBLE` | `POINTER_INPUT_ADAPTER` | module-address `TYPED_VALUE_COPY` | +| `type(item), allocatable :: var` | non-module holder | `HOLDER_REFERENCE [allocated]` | `HOLDER_REFERENCE [allocated]` with holder target lifetime | `ALLOCATABLE_HOLDER` | `ALLOCATABLE_HOLDER` | holder `POINTER_INPUT_ADAPTER [allocated]` | holder `TYPED_VALUE_COPY [allocated]` | +| `type(item), allocatable :: var` | module | `SCOPED_REFERENCE [allocated]` | `SCOPED_REFERENCE [allocated]` with call-scoped target | `MODULE_ALLOCATABLE_TRANSACTION` | `MODULE_ALLOCATABLE_TRANSACTION` with call target | scoped `POINTER_INPUT_ADAPTER [allocated]` | scoped `TYPED_VALUE_COPY [allocated]` | +| `type(item), allocatable, target :: var` | non-module holder | `HOLDER_REFERENCE [allocated]` | `HOLDER_REFERENCE [allocated]` with holder target lifetime | `ALLOCATABLE_HOLDER` | `ALLOCATABLE_HOLDER` | holder `POINTER_INPUT_ADAPTER [allocated]` | holder `TYPED_VALUE_COPY [allocated]` | +| `type(item), allocatable, target :: var` | module | `MODULE_ADDRESS [allocated]` | `MODULE_ADDRESS` with module target lifetime | `MODULE_ALLOCATABLE_TRANSACTION` preserving target | `MODULE_ALLOCATABLE_TRANSACTION` preserving target | `POINTER_INPUT_ADAPTER [allocated]` | module-address `TYPED_VALUE_COPY [allocated]` | +| `type(item), pointer :: var` | non-module holder | `POINTEE_REFERENCE [associated]` | `POINTEE_REFERENCE [associated]` with retained target owner | `INCOMPATIBLE` | `INCOMPATIBLE` | `POINTER_HOLDER` | pointee `TYPED_VALUE_COPY [associated]` | +| `type(item), pointer :: var` | module | module `POINTEE_REFERENCE [associated]` | module `POINTEE_REFERENCE [associated]` with native target owner | `INCOMPATIBLE` | `INCOMPATIBLE` | `MODULE_POINTER_TRANSACTION` | module-pointee `TYPED_VALUE_COPY [associated]` | + +An `Aliased` ordinary module object follows `MODULE_ADDRESS` instead of +`SCOPED_REFERENCE`, but its original target-lifetime fact still controls whether +a native pointer may outlive the call. This does not change `Aliased` array-view +semantics. + +The matrix is exhaustive for this Phase 8 scope. Every cell becomes either one +completed action or one deliberate language-level error before lowering. No +backend may infer a different action from datatype, `intent`, module shape, +address presence, or local memory checks. + +The table's `P` entries show the non-projecting input-adapter form. When the +Python contract projects pointer association writeback, replace every +nonpointer `P` cell with `INCOMPATIBLE`; the two pointer-storage rows retain +`POINTER_HOLDER` and `MODULE_POINTER_TRANSACTION`. + +##### Shared Holder And Callback ABI + +Define these support types once per qualified native derived type and import +the same definitions in every producer, origin operation, and consumer: + +```fortran +type :: item_allocatable_holder + type(item), allocatable :: value +end type + +type :: item_pointer_holder + type(item), pointer :: value => null() +end type +``` + +A persistent wrapper-owned holder is allocated through a Fortran pointer and +its opaque holder address is stored by the Python wrapper. Its nonpointer +allocatable component is a targetable subobject of the persistent holder +target, so the same carrier supports both `A` and `AT`; do not invent a second +allocatable-target holder. + +Module allocation and pointer transactions use bridge-local holder objects +declared `TARGET`. The module-specific operations are interoperable +`BIND(C)` procedures taking only `type(C_PTR), value :: holder_address` plus +interoperable status/context values. Each operation reconstructs the exact +shared holder with `C_F_POINTER` and performs `MOVE_ALLOC` or pointer assignment +entirely in Fortran. The binding transports a typed function pointer and an +opaque holder address; no allocatable or pointer descriptor crosses C. + +The old proposal to pass `type(item), allocatable` or `type(item), pointer` +directly through a runtime C callback is removed as noninteroperable. The old +proposal to avoid a transaction holder for module allocation/pointer restore is +also removed. A bridge-local transaction holder is the portable carrier; it is +not a persistent replacement for the originating module variable. + +For a module allocatable transaction, the bridge performs the equivalent of: + +```fortran +type(item_allocatable_holder), target :: transaction + +status = move_out(c_loc(transaction)) +if (status == X2PY_STATUS_OK) then + call native_procedure(transaction%value) + restore_status = move_back(c_loc(transaction)) +end if +``` + +`move_out` executes `move_alloc(module_value, transaction%value)` and +`move_back` executes `move_alloc(transaction%value, module_value)`. A successful +move-out makes the module variable unavailable until restoration. When the +module actual has `TARGET`, both destinations preserve pointer association; +when it lacks `TARGET`, aliases created through a temporary target have only +call lifetime. + +For a module pointer transaction, the bridge initializes +`transaction%value` from the current `C_LOC`/`C_NULL_PTR`, passes that component +to the native pointer dummy, and invokes `restore_pointer(c_loc(transaction))`. +The origin reconstructs the pointer holder and executes +`module_pointer => transaction%value`. The final nullification, +reassociation, allocation, or deallocation is therefore visible in the module +pointer. + +Operation tables use typed C function-pointer fields; do not round-trip a +function pointer through `void *`. The proxy retains its originating extension +until every active scoped call or transaction has unwound. + +##### Pointer Target Ownership + +A pointer holder owns the holder and association variable, not its target. +Default scalar-derived pointer target ownership is native: holder destruction +nullifies the component and deallocates only the holder. It must never +deallocate an unowned target. When final association matches a known module, +parent, or wrapper-owned target, retain that owner in completed policy; an +otherwise durable native target retains the originating extension and remains +the native program's release responsibility. Native code that returns a pointer +to an expired local target violates the contract rather than creating an x2py +fallback. + +This completed owner/release rule removes the old wrapper-owned pointer-result +blocker. Reassociation is supported, but it never silently transfers target +ownership to Python. Public documentation must warn that a native pointer saved +through a wrapper-owned target remains valid only while the wrapper and target +allocation remain alive. + +##### Multiple Scalar-Derived Arguments + +Do not generate `2**N` native call branches. Build one call context with one +slot per native argument and an ordered acquisition program: + +1. validate every Python wrapper, exact qualified type, storage capability, + allocation/association precondition, optional presence, and pointer-target + owner before entering any native origin operation; +2. retain all Python/module owners and acquire module transaction guards in a + deterministic total order; +3. deduplicate repeated actual identities so one module allocation or pointer + is checked out once and its holder/address can feed multiple native slots; +4. move out module allocatables in deterministic order, rolling back already + moved values in reverse order if a later acquisition fails; +5. initialize module pointer transaction holders; +6. enter all `SCOPED_REFERENCE` producers as a nested continuation chain, + storing each address in the context; and +7. invoke the native procedure exactly once after every slot is ready, then + unwind scoped producers, pointer restorations, allocation restorations, + guards, and retained owners in reverse order. + +If the same actual appears in multiple slots and any corresponding dummy may +define it while another slot references or defines it, reject the call before +checkout unless completed `INTENT` facts prove the aliasing legal. Read-only +duplicates share one acquisition. Never move the same module allocatable twice +or restore the same module pointer through independent locals. + +The generic scoped-address consumer ABI remains +`consumer(object_address, context) -> status`. The context carries all earlier +addresses, holders, ordinary arguments, result slots, and the first error. A +consumer never retains `object_address`; multiple module variables are handled +by nesting producers, not by generating one origin-module cross product per +native procedure. + +##### Error And Cleanup Contract + +Use one status protocol across scoped consumers and module transaction +operations. Do not raise a Python exception, `longjmp`, or unwind C++ through a +Fortran frame. Record status and any Python exception data in the call context, +return normally through every producer, complete cleanup, and only then raise +in the binding. + +- wrong qualified wrapper type, an incompatible matrix cell, or a known + reassociable pointer dummy receiving nonpointer storage raises `TypeError` + before native entry; +- a required ordinary/target/value/pointer-input actual whose allocatable is + unallocated or pointer is disassociated raises `ValueError` before native + entry; +- `A`, `AT`, and `P` descriptor calls preserve valid unallocated or + disassociated state and do not reinterpret it as optional omission; +- only an omitted Python argument or explicit `None` for an optional contract + selects native absence; a present empty handle never becomes omitted by + accident; +- an active recursive/concurrent transaction raises `RuntimeError` before the + affected origin changes state; +- every successful move-out has exactly one attempted move-back on every + normally returning path, and every native module-pointer call has exactly one + attempted association restore; +- cleanup continues in reverse order after the first restoration failure so + independent origins are not stranded; the first failure is reported with + later cleanup failures attached as context; +- a failed restoration leaves its origin guard poisoned instead of advertising + a usable proxy, and raises `RuntimeError` after all other cleanup attempts; +- conversion, result allocation, and Python-object creation that can fail are + completed before checkout where possible; failures after native return still + restore every transaction before propagating; and +- process termination, `ERROR STOP`, signals, or invalid native pointers are + not recoverable wrapper exceptions. The documentation must state that this + cleanup guarantee covers paths that return through the generated ABI. + +The per-origin guard must be thread-safe, or the binding must prove that the +GIL remains held for the complete transaction and that no callback re-entry is +possible. An unsynchronized Fortran `logical` is not a sufficient concurrency +guard. Internal synchronous address consumers are Phase 8 bridge machinery; +they do not expose the public callback semantics deferred to Phase 10. + +##### Implementation And Proof Checklist + +- [x] Preserve actual declaration attributes, module/non-module origin, + `TARGET` lifetime, allocatable/pointer state, exact type identity, and + pointer-dummy `INTENT` authority through parsing, semantic IR, and edited or + generated `.pyi` round trips. +- [x] Replace the former category/action-only contract with completed facets + capable of representing all six dummy forms and every action in the matrix. + `DerivedDummyCategory` remains the completed declared-form label and + `DerivedCallAction` remains the completed selected-action label; neither is + allowed to stand in for the lifetime, access, failure, cleanup, target-owner, + or release facets. The complete record includes `ALLOCATABLE,TARGET`, typed + value, target lifetime, pointer-input + validation, transaction cleanup, and target owner/release. Remove + `RUNTIME_POINTER_TARGET`, the module-allocatable incompatibility, and all old + fallback/rejection actions they made obsolete. +- [x] Complete every matrix decision in post-IR policy before `ir2ast.py`. + Binding and bridge generation only dispatch named actions; neither backend + inspects datatype, `intent`, module shape, address presence, or allocation + state to select a different mechanism. +- [x] Generate one shared allocatable holder and pointer holder per qualified + native type, with persistent create/destroy helpers and bridge-local + transaction use. Prove source/generated-`.pyi` bundles import the identical + holder definition and reject ABI/type mismatch before reconstruction. +- [x] Generate scoped-address producer operations for plain module objects and + non-`TARGET` allocated module allocatables, direct address operations for + durable module targets, move-out/move-back holder-address operations for + module allocatables, and current-target/restore holder-address operations for + module pointers. +- [x] Implement the ordered multi-argument acquisition/unwind program, + deduplicated origin identity, legal read-only aliasing, reverse rollback, + poisoned restoration failures, and a single final native invocation. +- [x] Implement the exact Python error mapping and optional/empty-state rules + above. Add injected failures before first acquisition, after one of several + acquisitions, during scoped nesting, after native return, and during each + cleanup category. +- [x] Remove the interoperable-`bind(C)` restriction from typed derived + `VALUE` calls. The Fortran bridge must perform the exact typed call without a + C aggregate cast, byte copy, layout promise, or detached-object fallback. +- [x] Support wrapper-owned pointer results with a persistent pointer holder, + native target ownership by default, explicit known-owner retention, direct + association writeback, and holder-only destruction. Remove the old blanket + target-ownership blocker rather than retaining it as a compatibility path. +- [x] Update public and maintainer documentation to teach the five actual + declarations, six dummy forms, complete matrix, direct versus scoped + address acquisition, holder and module transactions, `INTENT(IN)` pointer + exception, target lifetime, native pointer-target ownership, multi-argument + nesting, errors, and cleanup. Examples must show more than one scalar-derived + argument and link back to one canonical explanation instead of repeating + incomplete fragments. +- [x] Add one comprehensive native fixture at + `tests/data/fortran/wrapper/fscalar_derived_actual_dummy_matrix_f90.f90`, its + reduced source/generated contract under + `tests/wrapper/fortran/derived_types/contracts/fscalar_derived_actual_dummy_matrix_phase8/`, + focused policy/plan/artifact tests in + `tests/wrapper_codegen/test_phase8_scalar_derived_actual_dummy_matrix.py`, and + compiled tests in + `tests/wrapper/fortran/derived_types/test_scalar_derived_actual_dummy_matrix.py`. + Replace the earlier proposed separate module-allocatable and + module-target/pointer fixtures; do not retain tests that assert their old + rejection paths. +- [x] Make that native fixture a complete Fortran module containing all five + module actual declarations, wrapper-owned ordinary/allocatable/pointer + producers, all six dummy forms, both pointer `INTENT(IN)` and reassociable + pointer procedures, two qualified native types with the same short name, + optional arguments, injected operation failures, and state-reset helpers. +- [x] Parameterize policy/plan tests over every matrix cell. Every legal cell + must select its one completed action; every incompatible cell must assert its + exact pre-native `TypeError`; allocated/unallocated and + associated/disassociated states must assert their exact `ValueError`, valid + descriptor call, or optional-absence behavior. +- [x] Compiled tests must exercise mixed calls containing several + scalar-derived arguments. Include at least: multiple nested scoped module + objects; two module allocatable transactions plus a module pointer + transaction; mixed direct, holder, scoped, allocatable, pointer, target, and + value slots in one native call; repeated read-only actual identity; rejected + writable duplicate identity; failure after the first of several checkouts; + reverse restoration; native deallocation/reallocation; pointer + nullification/reassociation/allocation/deallocation; and owner retention. + Phase 8 cannot close if the new compiled procedures test only one derived + argument at a time. +- [x] Run the portable ABI fixture with the supported GNU toolchain and every + available secondary compiler in the development environment. The proof must + cover scoped `C_FUNPTR` consumers, `C_PTR` transaction holders, + `C_F_PROCPOINTER`, holder targetability, target-preserving `MOVE_ALLOC`, and + the accepted-`INTENT(IN)`/rejected-reassociable pointer distinction. + +### Phase 8I — Production Routing, Regression, And Completion + +- [x] Add separate support-report lanes for derived inputs, optional derived + inputs, in-place derived arguments, wrapper-owned derived results, plain + module proxies, `Aliased` borrowed module objects, borrowed field owners, + and the exact typed-value slice. +- [x] Replace the isolated scalar-derived descriptor routes with one + dependency-closed actual/dummy-matrix lane only after every unchecked Phase + 8H row passes. It must cover direct and scoped references, target adapters, + allocatable and pointer holders, module allocation and association + transactions, exact typed values, and multi-argument acquisition/unwind. + No old call-incompatible, nonreassociating-only, or interoperability-only + compatibility route may remain selectable. +- [x] Add one deliberate legacy/direct parity node for every dependency-closed + Phase 8 lane and append it to the production rollout evidence only after its + generated artifacts and runtime behavior match. +- [x] Update the migration matrix row for each reduced unit. Keep broad units + containing constructors, methods, inheritance, or callbacks on + their explicit Phase 9/10 blockers until whole-generation-unit support is + complete. +- [x] Treat source and generated-`.pyi` default field constructors as Phase 9 + class-surface blockers. Do not select the Phase 8 route merely because the + generated constructor was consumed into origin metadata rather than retained + as a semantic method; reduced opaque Phase 8 contracts must explicitly + suppress construction. +- [x] Prove an eligible opaque-derived generation unit selects the production + wrapper-plan route and no longer invokes `semantic_ir_to_codegen_ast()`. +- [x] Keep direct plan edits meaningful across both backends and preserve the + global no-fallback rule when a derived type, owner, release, or field action + is incomplete. +- [x] In every relevant planner, validator, binding generator, and bridge + generator, keep scalar, string, ordinary-array/native-handle, and + derived-type lowering methods in consistent groups with one short comment + above each group. Preserve typed object-kind/action matching; grouping must + not introduce datatype inference or a second dispatcher. +- [x] Run focused parser/printer, ownership/policy/readiness, plan/validation, + binding/bridge/printer, and runtime tests; relevant source/generated-`.pyi` + wrapper parity; and regressions for scalar, string, array, and Phase 7 handle + lanes. +- [x] Run the wrapper suite excluding the deferred LAPACK coverage, the wrapper + codegen complexity checker, documentation checks, whitespace check, and the + required static-analysis suite before closing implementation. +- [x] Run the comprehensive Phase 8 scalar-derived actual/dummy matrix policy, + artifact, and multi-argument compiled tests; all retained holder and Phase + 7/8 regressions; the wrapper suite excluding LAPACK; documentation and + whitespace checks; the wrapper complexity checker; and the required static + suite after the replacement route is implemented. +- [x] Close Phase 8 only when every supported rank-zero non-polymorphic derived + input/result/module transfer is direct, both plain module-proxy and `Aliased` + address-backed module-object paths are direct, live member operations and + recursive-edge policy are validated before emission, and + every remaining class-surface/callback/derived-array case has an exact Phase + 9/10 or unsupported-policy blocker. + +### Phase 8 Implementation Evidence + +- Post-IR origin, identity, handoff, ownership, field, lifecycle, and exact + blocker evidence lives in + `tests/wrapper_codegen/test_phase8_derived_types.py`, with supporting parser, + printer, source-conversion, ownership, and readiness suites named in + `tests/wrapper/CHECKLIST_COVERAGE.md`. +- Public-field validation is split into named completed-policy, descriptor, + typed object-kind, and setter checks so no single semantic-policy routine + becomes a second backend-style dispatcher. +- Compiled legacy/source and direct-plan evidence lives in + `tests/wrapper/fortran/derived_types/test_phase8_derived_plan.py`. It covers + required, optional, in-place, caller-supplied output, ordinary and `bind(C)` + typed native `value`, direct/hidden owned result, module-proxy, + direct-address module object, constant value, field, owner-retention, + allocation/cleanup artifact, and exactly-once finalization behavior. +- The former isolated scalar-derived descriptor evidence in + `tests/wrapper_codegen/test_phase8_scalar_derived_descriptors.py`, + `tests/wrapper/fortran/derived_types/test_scalar_derived_descriptor_plan.py`, + and `tests/data/fortran/wrapper/fscalar_derived_descriptors_f90.f90` is + superseded by the comprehensive policy/artifact and compiled matrix files + named in Phase 8H. They cover all 60 declaration/dummy cells, empty states, + qualified same-short-name identities, `sequence` typed values, holder and + module transactions, multi-origin unwind, pointer target ownership, injected + cleanup failures, and the exact retained incompatibilities. No obsolete + module-allocatable rejection, stable-pointer-only, or wrapper-pointer-result + blocker remains as negative compatibility coverage. +- `x2py/pipeline/build.py` registers the dependency-closed Phase 8 support + lanes and their passing production evidence. The automatic-route test + replaces `semantic_ir_to_codegen_ast()` with a failure sentinel and proves an + eligible opaque-derived unit never invokes it. +- Constructors, methods, properties beyond the completed field descriptors, + inheritance, and polymorphic class orchestration remain Phase 9. Public + callbacks remain Phase 10. Arrays of derived values, non-scalar holder member + operations, recursive value edges without completed descriptor policy, + unresolved imported types without an exact runtime definition, immutable visible + derived replacement, and mixed native result plus visible-writeback envelopes + carry exact unsupported-policy blockers instead of selecting a fallback. + Internal synchronous scoped-address consumers and module transactions are + Phase 8 implementation machinery, not deferred public callbacks. + +Historical Phase 8 closure evidence before the module-allocatable and +module-pointer restore redesigns (2026-07-15): all 39 focused Phase 8 +plan/compiled tests, the 711-test +cross-stage regression batch, all 79 runtime-handle tests, 1,133 documentation +and layout tests, and all 329 wrapper tests outside the deferred full +BLAS/LAPACK file passed. The wrapper complexity checker, Ruff lint/format, +Bandit, Vulture, whitespace, and explicit-`origin/main` Radon policy passed; +the advisory full Radon complexity and maintainability reports were also +produced. The required `--base-ref auto` Radon invocation could not resolve +CI-only base-SHA variables locally, and the explicit-base rerun passed. No +LAPACK test was run locally. This evidence does not close the reopened Phase 8H +rows. + +Final Phase 8H/I closure evidence (2026-07-15): the focused Phase 8 plus route- +ledger batch passed 180 tests; the affected cross-stage semantic, lowering, +runtime-handle, planner, and backend batch passed 611 tests; and the complete +wrapper suite outside the deferred combined BLAS/LAPACK file passed 445 tests. +Documentation checks passed 1,123 tests and whitespace validation passed. The +GNU toolchain compiled and ran the complete matrix suite; Intel `ifx` 2026.1.0 +compiled, linked, and ran the same generated ABI for `sequence` typed values, +mixed six-form input, module allocatable/pointer transactions, target-preserving +`MOVE_ALLOC`, and the accepted-input/rejected-reassociable pointer distinction. +The wrapper complexity checker, Ruff lint/format, Bandit, Vulture, explicit- +`origin/main` Radon policy, and advisory Radon complexity/maintainability runs +passed. The CI-only `--base-ref auto` Radon lookup was unavailable locally, so +the required explicit-base rerun was used. No LAPACK test was run locally. + +### Phase 8 Expansion Gate + +- [x] Inventory the live semantic contract, post-IR ownership policy, active + snapshot paths to remove, legacy binding/bridge paths, plan-route blockers, + public docs, checked `.pyi` fixtures, and real wrapper tests. +- [x] Separate origin, owned/borrowed lifetime, module address acquisition, + input/result/field/module-state use, destruction, owner retention, and + recursive member-path access into dependency-ordered Phase 8A-I sub-lanes. +- [x] Record the strict Phase 8/9/10 boundaries and identify reduced existing + native units that can prove opaque transfers without first migrating public + constructors, methods, inheritance, or callbacks. Public field descriptors + are part of Phase 8. +- [x] Begin Phase 8 implementation only from Phase 8A and keep every later + sub-lane blocked on its declared dependencies. + +## Phase 9 — Classes, Constructors, And Methods + +Expansion status: complete. Implementation status: complete. The direct class +path is covered by policy, plan-edit, artifact, compiled runtime, production +routing, and broad non-LAPACK wrapper-suite evidence below. + +Scope: generated Python class objects, namespace registration, default and +keyword constructors, explicit constructor bindings, constructor overloads, +instance and static methods, type-bound dispatch, method overloads, finalizer +attachment, inheritance, and the first supported scalar polymorphic input +dispatch. Phase 9 assembles those public class surfaces on the opaque storage, +field descriptors, handoffs, and lifetime rules completed in Phase 8. + +### Phase 9 Boundary And Explicit Non-Scope + +Phase 9 may compose completed Phase 8 records but must not revisit them. +Constructor and method policy may select how an instance is created or passed; +it may not change object origin, storage kind, field access, owner retention, +release, nullability, native setter assignment, or destruction. A class plan +references the namespace-owned `DerivedTypePlan` and its field plans rather +than copying or rendering them. + +The following surfaces are in Phase 9: + +- one generated Python type object for each public supported semantic class, + with stable native identity and explicit Python base identity; +- an explicitly present or deliberately absent public constructor surface; +- generated default/keyword field initialization for eligible public scalar + fields, including omitted-keyword preservation of native defaults; +- direct `@bind("native_name")` constructors and explicit constructor overload + candidates linked to concrete native procedures; +- passed-object type-bound instance methods, non-type-bound methods attached to + the class by the semantic contract, and supported `@staticmethod` methods; +- class-owned overload sets with exact candidate signatures and deterministic + runtime selection; +- owned-instance finalization through the Phase 8 destroy/release path and + borrowed-instance non-destruction; +- Python inheritance for supported Fortran extension types; and +- scalar, input-only polymorphic calls whose accepted runtime class set and + concrete native dispatch targets are fully enumerated before lowering. + +The following remain outside Phase 9: + +- callbacks, adapters, trampolines, and callable lifetime; these remain Phase + 10 even when a callback argument/result is a derived object; +- module-level generic/operator migration units that do not require a class + surface; those remain in Phase 11, although they may reuse the same overload + candidate and runtime-match vocabulary; +- arrays of derived or polymorphic values, elemental class dispatch, and + partial construction/destruction of array elements; +- polymorphic results, mutable polymorphic dummies, allocatable/pointer + polymorphic scalars, unlimited polymorphism, abstract instantiation, + deferred-binding execution, and runtime extension types not enumerated in + the semantic module; +- any unresolved Phase 8 storage blocker merely because a constructor or + method happens to use that type; Phase 9 reuses the completed allocatable and + pointer holders and must not invent a second storage path; +- generic constructor selection whose candidates are indistinguishable at the + Python boundary; and +- compatibility aliases, synthesized legacy entrypoints, string-built backend + method names, or a fallback from an incomplete class plan to legacy class + lowering. + +### Phase 9 Existing Oracle And Inventory + +The legacy route plus existing source/generated-`.pyi` runtime assertions are +the behavioral oracle. Capture complete binding, bridge, header, and runtime +evidence before each reduced direct-plan slice. Correct unsafe behavior only +when the documented contract says so; do not preserve legacy architecture. + +| Existing unit | Phase 9 behavior to preserve | Required reduced slice | +| --- | --- | --- | +| `derived_types/test_constructors_and_finalizers.py::test_fortran_default_constructor_keywords_and_finalization[*]` | default construction, keyword-only scalar fields, native defaults, invalid-call cleanup, and exactly-once finalization | default/keyword constructor plus owned destroy path | +| `derived_types/test_derived_type_methods.py::test_modern_fortran_derived_type_exposes_class_and_type_bound_methods[*]` | instance methods, explicit binding names, scalar arguments/results, class static factory, and Phase 7 handle fields | split `vector` methods from `vector_store` handle methods and static factory | +| `derived_types/test_borrowed_finalizers.py::test_borrowed_child_wrapper_never_finalizes_native_component[*]` | borrowed child retains parent; only the owned parent finalizes | class assembly over the completed Phase 8 borrowed-field owner path | +| `derived_types/test_derived_layout.py::test_bind_c_derived_types_use_accessors_and_fortran_value_copy[*]` | default class creation and methods coexist with opaque field access and typed native value copy | class surface only; Phase 8 retains layout and handoff ownership | +| `derived_types/test_inheritance.py::test_fortran_extension_types_generate_python_inheritance[*]` | Python subclass relationships, inherited field/method access, overridden methods, unbound base calls, and scalar polymorphic input dispatch | base/extension class graph first, polymorphic call second | +| `tests/semantics/conversion/pyi/test_classes_and_overloads.py` | generated versus bound constructors, removed constructors, direct constructor targets, explicit overload links, type-bound root targets, and invalid metadata diagnostics | semantic-policy fixtures before planner/backend work | +| `edit_pyi_contracts/test_surface_edit_contracts.py` | edited contracts can remove constructors/methods/candidates and add explicit bindings without resurrecting source declarations | absence/export validation and source/generated/edited parity | +| `naming/test_defined_operators.py` and `naming/test_generic_interfaces.py` | exact candidate matching and Python export naming | reuse candidate-match vocabulary; broad module/generic units remain Phase 11 | + +Inventory these legacy owners without importing them into the direct package: + +- `x2py/semantics/ir2ast.py` currently interprets constructor overloads, + passed-object positions, type-bound names, polymorphic variants, and class + insertion. Each semantic decision found there must move into post-IR class + policy before direct lowering. +- `x2py/codegen/bindings/c_to_python.py` currently assembles type objects, + constructors, methods, overloads, properties, inheritance, module exports, + and finalizers. Reuse emitted behavior as the oracle, not its broad control + flow or method-name synthesis. +- `x2py/codegen/bridges/fortran_to_c.py` currently supplies typed constructor + allocation, passed-object association, method calls, overload interfaces, + and finalization helpers. Direct bridge generation must consume completed + class/method actions and reuse Phase 8 native storage helpers. +- Generated semantic `.pyi` class declarations are a public contract. A + consumed default constructor still counts as a constructor surface and must + remain a whole-unit Phase 9 route requirement. + +### Phase 9 Plan Shape And Action Vocabulary + +Extend the existing namespace plan; do not introduce a rendered-class layer or +a second function plan. + +- Add one namespace-owned `ClassSurfacePlan` (name illustrative, not + prescriptive) that references exactly one `DerivedTypePlan`, its Python + exports, optional base-class identity, constructor plan, ordered methods, + ordered overload sets, type-object slots, and module-registration action. +- Add a `ConstructorPlan` with an explicit kind: `ABSENT`, + `DEFAULT_FIELDS`, `BOUND_PROCEDURE`, or `OVERLOAD_SET`. Record allocation + action, accepted Python parameters, native target/call slots, initialized + fields, omitted-field behavior, cleanup action, and success transition. +- Reuse `FunctionPlan` for each concrete method or constructor target. Add only + a class-call facet recording method kind, passed-object position, self + storage requirement, result attachment, public descriptor flags, and the + owning class identity. +- Add an `OverloadSetPlan` containing public export, overload kind, ordered + concrete candidate references, typed runtime predicates, ambiguity result, + no-match diagnostic, and selected native target. Candidate predicates use + exact dtype/rank/derived-class facts already completed by argument plans. +- Add an `InheritancePlan` containing canonical base identity, storage + compatibility, inherited/overridden method ownership, Python base type + symbol, and module initialization dependency order. +- Add a `PolymorphicDispatchPlan` only for supported scalar input calls. It + enumerates accepted concrete class identities and a concrete `FunctionPlan` + variant for each; it must not rediscover subclasses from runtime object names. +- Keep destructor selection on the referenced Phase 8 derived handoff/release + plan. Phase 9 records only which class slot invokes that existing action and + which constructor failure edges need cleanup. + +Stable semantic action names must describe behavior, not backend function +names. At minimum distinguish: + +- class registration: `CREATE_TYPE`, `SET_BASE`, `READY_TYPE`, `EXPORT_TYPE`; +- construction: `OMIT`, `ALLOCATE_DEFAULT`, `ALLOCATE_AND_ASSIGN_FIELDS`, + `CALL_BOUND_CONSTRUCTOR`, `DISPATCH_CONSTRUCTOR`, `REJECT_CONSTRUCTION`; +- method binding: `INSTANCE`, `STATIC`, and explicit unsupported class-method + policy until a real class-method contract exists; +- passed-object handoff: `WRAPPER_ADDRESS`, `BORROWED_ADDRESS`, or the exact + completed Phase 8 storage action; +- overload selection: `MATCH_EXACT`, `SELECT_CANDIDATE`, `NO_MATCH`, + `AMBIGUOUS`; and +- construction lifecycle: `ALLOCATE`, `INITIALIZE`, `COMMIT_OWNER`, + `CLEANUP_UNCOMMITTED`, `DESTROY_OWNED`. + +### Mandatory Phase 9 Migration Algorithm + +For every dependency-closed sub-lane: + +1. Capture one passing source/generated-`.pyi` legacy unit and its complete + class, binding, bridge, header, and runtime assertions. +2. Complete class export, constructor kind, method kind, passed-object policy, + overload candidates, inheritance, polymorphic accepted set, allocation, + commit, cleanup, and destruction before `ir2ast.py`. +3. Project those facts into the existing namespace, derived-type, function, + lifecycle, and native-slot plans plus the smallest class-specific facets. +4. Validate the complete class graph and every cross-backend symbolic role + before either backend emits source. +5. Lower through small named methods selected only by typed actions. Backend + local temporaries may implement a selected action but cannot choose policy. +6. Compare generated artifacts with the oracle and record intentional + differences before compiling. +7. Add focused policy, plan-edit, validation, printer, binding, bridge, + source/generated-`.pyi`, edited-contract, and compiled runtime tests. +8. Promote production routing only after the reduced unit passes direct-plan + runtime parity and no class-surface fallback remains. + +### Phase 9A — Semantic Class-Surface Completion + +- [x] Add completed post-IR policy records for public class identity, exports, + constructor kind, method kind, passed-object position, overload ownership, + base identity, type-object registration, and construction permissions. +- [x] Preserve explicit absence: an edited `.pyi` that removes `__init__`, a + method, or an overload candidate must produce an absent plan entry and cannot + resurrect source behavior. +- [x] Move any class-surface inference still in `ir2ast.py` into policy + completion. Readiness must report the owner path and exact missing decision. +- [x] Add policy tests for generated, bound, removed, overloaded, inherited, + abstract, and invalid class surfaces before planner changes. + +### Phase 9B — Typed Class Plan And Validation + +- [x] Add the namespace-owned class, constructor, method-call, overload, and + inheritance plan facets described above, each referencing existing Phase 8 + type/field/lifetime plans rather than copying them. +- [x] Project Python/native names and export aliases once. Do not synthesize + backend method names from strings or recover native targets by scanning + emitted functions. +- [x] Validate unique class/type identity, base-before-derived order, one + constructor kind, method ownership, passed-object position, native call-slot + agreement, field-plan identity, lifecycle roles, and module export symbols. +- [x] Add direct plan-edit tests proving invalid constructor, method, base, + overload, or lifecycle references fail before emission in both backends. + +### Phase 9C — Class Creation And Module Registration + +- [x] Emit one Python type object per supported class, attach the completed + Phase 8 field descriptors, set the validated base type, ready the type, and + export every completed Python name in dependency order. +- [x] Keep opaque instance storage identical to Phase 8 wrapper storage. Class + assembly must not add C aggregate layout, component offsets, or a second + native owner field. +- [x] Attach the Phase 8 destruction action only to owning classes; borrowed + proxies and nested objects retain owners and never gain independent destroy + slots. +- [x] Add artifact and compiled reduced tests for an opaque constructible class, + an intentionally nonconstructible class, a borrowed child, and exact module + export identity. + +### Phase 9D — Default And Keyword Field Constructors + +- [x] Build constructor parameters only from fields explicitly eligible in + completed constructor policy. Preserve keyword-only behavior and native + default component initialization for omitted fields. +- [x] Allocate the Phase 8 persistent native instance first, apply validated + field assignments through existing field setter actions, then commit wrapper + ownership only after every step succeeds. +- [x] On parse, conversion, allocation, or field-assignment failure, clean up + the uncommitted native instance exactly once. Failed `tp_init` must not leak, + double-finalize, or expose a partially initialized wrapper. +- [x] Replay `fconstructors_f90` for default, partial, complete, positional, + unknown-keyword, native-default, and finalization-count assertions through + both source and generated-`.pyi` contracts. + +### Phase 9E — Explicit And Overloaded Constructors + +- [x] Represent direct `@bind("native_name")` construction as one constructor + action linked to a concrete function plan. It replaces, rather than wraps or + falls back to, the generated field constructor. +- [x] Represent constructor overloads as an explicit constructor-owned overload + set. Do not combine `@overload` and `@native_call`, and do not reinterpret a + normal method overload as `tp_init`. +- [x] Complete allocation-before-call versus native-produced-instance policy, + result attachment, owner commit, failure cleanup, and exactly-once release for + every candidate before lowering. +- [x] Reject indistinguishable candidates, missing targets, incompatible self + types, mixed constructor kinds, or ambiguous edited declarations during + policy/readiness or plan validation, never from candidate trial calls. +- [x] Add isolated semantic and compiled fixtures for direct bound construction, + two distinguishable constructor candidates, no-match, ambiguity, target + failure cleanup, and source/generated/edited-contract parity. + +### Phase 9F — Instance And Static Methods + +- [x] Lower passed-object instance methods from the completed self position and + Phase 8 handoff. Preserve native argument order when `self` is not the first + native slot. +- [x] Support explicit binding names and type-bound root-target metadata without + exporting the private concrete target as a duplicate module function. +- [x] Lower supported static methods without fabricating `self`; attach them to + the type object with their completed export and descriptor flags. +- [x] Reuse ordinary function argument/result plans for scalar, string, array, + handle, and derived transfers. A method cannot widen an unsupported ordinary + call lane. +- [x] Replay reduced `fclasses_f90` vector methods first, then `vector_store` + handle methods and static factory, with exact source/generated-`.pyi` runtime + and artifact parity. + +### Phase 9G — Class-Owned Overload Dispatch + +- [x] Complete ordered candidates and exact runtime predicates for each + class-owned overload set. Candidate selection may inspect only typed Python + argument facts named by the plan, never invoke candidates speculatively. +- [x] Reuse one overload matching vocabulary for constructors, methods, + operators, and later Phase 11 module generics while keeping their owners and + call actions distinct. +- [x] Detect indistinguishable signatures before emission and produce stable + no-match diagnostics listing the public overload and accepted signatures. +- [x] Validate native target, Python export, argument/result plans, passed-object + position, and overload kind across binding and bridge views. +- [x] Add focused method-overload tests for primitive kinds, ranks, derived + subclasses, keyword normalization, exact no-match, and ambiguity; keep broad + defined-operator/module-generic promotion in Phase 11. + +### Phase 9H — Finalization And Constructor Failure Safety + +- [x] Route normal owned-instance deallocation, constructor failure, and + native-constructor failure through the same Phase 8 destroy/release action, + guarded by an explicit uncommitted/committed lifecycle state. +- [x] Prove finalization occurs exactly once for successfully constructed + owners, once for native storage allocated before a rejected constructor call, + and never for borrowed children or native-owned module objects. +- [x] Prove child-to-parent retention survives method/property access and that + deleting the parent first delays only the parent's owning finalizer. +- [x] Replay `fconstructors_f90` and `fborrowed_finalizer_f90`, including forced + Python argument failures and repeated garbage collection. + +### Phase 9I — Inheritance And Scalar Polymorphic Input Dispatch + +- [x] Complete canonical base/extension relationships, storage compatibility, + inherited fields, inherited methods, overrides, Python base symbols, and + module initialization order before planning. +- [x] Construct base and derived wrappers with the same Phase 8 opaque storage + contract while preserving exact runtime type identity and safe unbound base + method calls on derived instances. +- [x] For each supported scalar input-only polymorphic dummy, enumerate the + accepted concrete class identities and one concrete native call variant per + identity. Reject unknown or abstract runtime classes before the native call. +- [x] Keep polymorphic results, mutable dummies, arrays, descriptor-backed + polymorphic scalars, unlimited polymorphism, and unenumerated extensions on + exact blockers; inheritance must not silently widen them. +- [x] Replay `finheritance_f90` for `issubclass`, `isinstance`, inherited field + access, override dispatch, unbound base calls, and base/circle/box + polymorphic inputs through source and generated-`.pyi` routes. + +### Phase 9J — Production Routing, Documentation, And Closure + +- [x] Add support-report lanes for class registration, default constructors, + bound constructors, constructor overloads, instance methods, static methods, + class overloads, finalizers, inheritance, and scalar polymorphic input. +- [x] Add one reduced compiled direct-plan node per dependency-closed lane, then + update its migration-matrix row only after artifact and runtime parity. +- [x] Prove eligible class units select the production wrapper-plan route and + never call `semantic_ir_to_codegen_ast()`; an unsupported class decision must + keep the whole generation unit on one exact blocker without partial fallback. +- [x] Synchronize constructor/method/inheritance user docs, semantic `.pyi` + reference, source map, feature matrix, subject README, and checklist coverage + with the implemented class contract. +- [x] Run focused policy/plan/backend tests, all affected existing class wrapper + nodes through source/generated-`.pyi` modes, the wrapper suite excluding + LAPACK, the wrapper complexity checker, documentation checks, whitespace, + and the required static-analysis suite. +- [x] Close Phase 9 only when every supported constructor/method/inheritance + unit routes directly, all Phase 8 field/storage/lifecycle decisions remain + unchanged, and every remaining callback, derived-array, polymorphic, or + ambiguous-overload case has an exact Phase 10/11 or unsupported-policy + blocker. + +Closure evidence (2026-07-16): focused semantic, lowering, routing, and direct +Phase 8-10 plan tests passed 184 tests after the final policy refactor. The +complete local wrapper suite excluding LAPACK passed 449 tests in source and +generated-contract modes. The wrapper complexity checker, Ruff lint/format, +Bandit, Vulture, explicit-`origin/main` Radon policy, and advisory Radon +complexity/maintainability commands passed. The CI-only `--base-ref auto` +Radon lookup could not resolve outside CI, so the required explicit-base run +was used. No LAPACK test was run locally. + +### Phase 9 Expansion Gate + +- [x] Inventory class creation/destruction, constructor categories, + instance/static/type-bound methods, overloads, inheritance/polymorphism, + decorator effects, module initialization, legacy owners, semantic fixtures, + and passing runtime oracles. +- [x] Define the Phase 8/9/10/11 ownership boundaries and keep all class + implementation rows unchecked. +- [x] Split implementation into dependency-ordered Phase 9A-J sub-lanes with + explicit policy, plan, validation, lowering, artifact, compiled parity, + production routing, documentation, and closure gates. + +## Phase 10 — Callbacks And Trampolines + +Expansion status: complete. Implementation status: complete. Immediate +callbacks are covered by focused policy/plan/artifact tests, existing compiled +runtime oracles, production routing, and broad non-LAPACK wrapper-suite +evidence below. + +Scope: immediate callback argument validation, call-scoped context lifetime, +external Fortran adapter procedures, C trampolines, scalar/string/array/derived +argument and result conversion, permissive reference writeback, same-thread +re-entry and GIL handling, callback cleanup, and the documented fatal error +boundary. + +### Phase 10 Boundary And Explicit Non-Scope + +Phase 10 composes ordinary call transfers completed in Phases 2-9 but does not +reinterpret them. A callback signature is transport-facing: it describes the +procedure ABI that native Fortran calls, including argument order, +value/reference transport, rank, shape, character length, and result +representation. It deliberately does not repeat native callback `intent`. +Normal wrapper projection and callback adapter projection remain distinct +completed records. + +Named `@prototype` declarations are the single callback-signature authority. +Callback arguments reference a prototype by name; bare prototype arguments use +reference transport and permissive writable Python storage, while `Value(T)` +is the only transport override. Prototypes are semantic-only declarations and +never become Python runtime exports. Post-IR policy selects either an implicit +external adapter declaration or a named explicit declaration from completed +prototype characteristics. Lowering does not reconstruct that decision or +duplicate native `intent`. + +The supported callback contract is deliberately call-scoped: + +- the Python callable is validated and retained before the native call, placed + in one thread-local context stack for that callback site, and released after + the native call returns; +- nested callback-taking calls on the same entering Python thread are allowed; +- each C trampoline validates the entering thread, acquires the GIL, converts + completed adapter arguments, invokes the current Python callable, converts + or copies back results, releases the GIL, and returns to its Fortran adapter; +- `Value(T)` uses value conversion; scalar reference storage, fixed-length + character storage, arrays, and derived objects use permissive copy-in/out + storage already asserted by the runtime tests; and +- a Python exception, invalid callback return, missing context, or cross-thread + invocation prints the Python error and aborts the host process. The direct + path must not fabricate a fallback result or continue native execution. + +The following remain outside Phase 10: + +- stored callbacks, callback registration/unregistration, procedure-pointer + fields, callbacks invoked after the wrapped call, optional dummy procedures, + null procedure pointers, asynchronous callbacks, and cross-thread callback + execution; +- persistent callable ownership, callback teardown during object/library + destruction, and callback use as a synchronization mechanism; +- callbacks whose signature is incomplete, assumed-rank, has a runtime-only + character length, or otherwise lacks the exact ABI facts required by the + adapter and trampoline; +- callback-specific coercion, recovery, exception-result, or argument + reordering policies not present in the public contract or legacy tests; and +- module generic/operator orchestration that merely contains a callback-taking + candidate; its callback transfer may be reusable, but public generic routing + remains Phase 11. + +### Phase 10 Existing Oracle And Inventory + +The public callback guide/reference, generated semantic `.pyi` contracts, and +existing source/generated-`.pyi` runtime assertions are the behavioral oracle. + +| Existing unit | Phase 10 behavior to preserve | Required reduced slice | +| --- | --- | --- | +| `callbacks/test_scalar_callbacks.py::test_immediate_scalar_dummy_procedure_calls_python_callback[*]` | scalar result/void callbacks, callable validation, balanced references, nested same-thread re-entry, held-GIL wrapper envelope, and thread-local context | first context, trampoline, scalar-value, and cleanup slice | +| `callbacks/test_scalar_callbacks.py::test_callback_exception_prints_traceback_and_aborts_host_process[*]` | callback exception, wrong result, and wrong signature print a Python error and terminate the subprocess | fatal-boundary slice after scalar success | +| `callbacks/test_array_callbacks.py::test_immediate_dummy_procedure_converts_array_arguments_and_results[*]` | writable array view, shaped array result, outer-output identity, and reference writeback | array argument/result slice | +| `callbacks/test_all_callback_shapes.py::test_immediate_callbacks_cover_all_supported_argument_shapes[*]` | scalar value override, rank-zero scalar storage, fixed strings, arrays, derived values, reference writeback, and one combined call envelope | cross-kind closure slice | +| `callbacks/test_derived_callbacks.py::test_immediate_dummy_procedure_converts_derived_arguments_and_results[*]` | callback-local borrowed derived input plus wrapper-owned derived result conversion | derived slice after Phase 9 construction | +| `callbacks/test_callback_generated_pyi_contracts.py` | named prototypes, reference-default and `Value(T)` transport, shape, character storage, cross-module identity, and result annotations round-trip exactly | semantic-contract parity slice | +| `semantics/conversion/pyi/test_types_and_values.py` callback cases | prototype declarations and references, `Value(T)`, exact argument names used by shapes, and unnecessary `Addr(...)` prototype forms | policy completion before planner work | + +### Phase 10 Plan Shape And Action Vocabulary + +Extend the existing argument/function plans; do not add a second function plan +or embed a legacy AST. + +- Add one `CallbackHandoffPlan` facet to each callable argument. It records the + callable owner, call-scoped lifetime, context symbol, context stack action, + entering-thread rule, GIL rule, Fortran adapter symbol, C trampoline symbol, + ordered callback argument plans, optional result plan, and fatal-error + action. +- Add a `CallbackTransferPlan` for each callback argument/result containing the + semantic type identity, object kind, value/reference ABI, rank/shape/length + roles, Python barrier action, permissive reference writeback or isolated + value action, borrowed-owner retention, and exact C ABI roles. +- Reuse ordinary scalar, string, array, and derived plan vocabulary where the + representation is identical. The native callback is the caller, so normal + Python-to-native argument projection cannot be silently reused in reverse. +- Add ordered function lifecycle phases `VALIDATE_CALLBACK`, `PUSH_CONTEXT`, + `ENTER_NATIVE`, `POP_CONTEXT`, and `RELEASE_CALLBACK`. Every failure edge + before native entry unwinds acquired references; the fatal trampoline edge + never returns. +- Keep backend-local adapter locals and temporary Python views inside the + selected implementation method. They are emitted-code details, not semantic + policy. + +Stable actions must describe behavior, not generated function names. At +minimum distinguish: + +- callable/context: `VALIDATE_CALLABLE`, `RETAIN_CALLABLE`, `PUSH_CONTEXT`, + `POP_CONTEXT`, `RELEASE_CALLABLE`; +- callback ABI: `VALUE`, `REFERENCE`, `DATA_AND_SHAPE`, + `DATA_AND_LENGTH`, and `DERIVED_ADDRESS`; +- adapter transfer: `COPY_IN`, `COPY_OUT`, `COPY_IN_OUT`, `BORROW_READ_ONLY`, + and `BORROW_WRITABLE`; +- trampoline runtime: `REQUIRE_ENTERING_THREAD`, `ACQUIRE_GIL`, `CALL_PYTHON`, + `RELEASE_GIL`, and `ABORT_WITH_PYTHON_ERROR`; and +- result handling: `RETURN_SCALAR`, `RETURN_ARRAY_ADDRESS`, + `RETURN_DERIVED_ADDRESS`, `RETURN_VOID`, and `REJECT_RESULT`. + +### Mandatory Phase 10 Migration Algorithm + +For every dependency-closed sub-lane: + +1. Capture the documented behavior, one passing source/generated-`.pyi` + legacy unit, and its callback-related binding, bridge, adapter, trampoline, + and runtime assertions. +2. Complete callable validity, signature order, ABI roles, reference + writeback/value isolation, + shape/length dependencies, result handling, context lifetime, thread/GIL + rules, cleanup, and fatal behavior before wrapper planning. +3. Project those facts into the existing function/argument/lifecycle plans plus + the smallest callback-specific facets. +4. Validate the binding, bridge, adapter, and trampoline role graph before + either backend emits source. +5. Lower through typed action dispatch and small named methods. Do not trial a + callback or infer shape/transport from emitted locals. +6. Compare emitted artifacts and behavior with the runtime oracle; document + any safety improvement before changing observable behavior. +7. Add focused policy, editable-plan, validation, binding, bridge, printer, + source/generated-`.pyi`, subprocess-failure, and compiled runtime tests. +8. Promote production routing only after the complete callback-taking + generation unit passes direct-plan parity with no callback fallback. + +### Phase 10A — Semantic Callback Completion + +- [x] Add completed post-IR callback records for callable signature order, + object kind, value/reference ABI, shape/length roles, result representation, + call scope, context lifetime, same-thread rule, GIL rule, cleanup, and + fatal-error behavior. +- [x] Preserve generated and edited named prototypes exactly. Reject an + incomplete prototype reference, unnecessary `Addr`, optional procedure, + stored/procedure-pointer lifetime, unavailable mandatory native interface, + or unsupported result with the owner path and one exact reason. +- [x] Complete callback signature/result/ownership policy before wrapper + planning; lowering may only project the completed callback record. +- [x] Add policy/readiness tests for reference-default transport, `Value(T)`, + and retained unsupported forms before planner changes. + +### Phase 10B — Typed Callback Plan And Validation + +- [x] Add callback handoff, transfer, result, context, and lifecycle facets to + the existing function plan and reference ordinary datatype plans instead of + copying them. +- [x] Project adapter/trampoline symbols and ABI roles once. Do not synthesize + backend handler names or rediscover dimension/length dependencies from + emitted variables. +- [x] Validate unique callback sites, exact argument order, role availability, + transport/writeback, dtype/rank/shape/length agreement, derived type identity, + result compatibility, context balance, and validate/push/pop/release order. +- [x] Add direct plan-edit tests proving invalid callback roles, unbalanced + lifecycle, or cross-backend disagreement fail before emission. + +### Phase 10C — Context, Trampoline, GIL, And Scalar Values + +- [x] Emit one thread-local stack per callback site, callable validation and + strong-reference retention before native entry, reverse-order pop/release + after return, and cleanup on every ordinary pre-entry failure. +- [x] Emit one C trampoline and separately linked external Fortran adapter from + the completed ABI; validate the entering thread and context before Python + conversion. +- [x] Acquire/release the GIL inside the trampoline and keep the outer + callback-taking wrapper on the legacy-observed held-GIL envelope. +- [x] Lower void and scalar-value arguments/results first, then replay scalar + callback success, nested re-entry, non-callable rejection, and balanced + reference-count assertions in both build modes. + +### Phase 10D — Scalar Reference And Fixed-String Storage + +- [x] Lower every primitive scalar reference as copy-in/out rank-zero NumPy + storage; `Value(T)` remains an isolated Python scalar value. +- [x] Lower fixed-string references as rank-zero fixed-width bytes storage with + exact length, padding, and writeback. The semantic annotation remains + `String[n]` and carries no native direction. +- [x] Reject runtime-length callback strings before emission; no adapter-local + inference may change the representation. +- [x] Replay the scalar-storage and string-storage cases from the combined + callback fixture through source/generated-`.pyi` routes. + +### Phase 10E — Array Arguments And Results + +- [x] Lower array callback arguments from completed dtype, rank, shape, + ordering, contiguity, and alignment facts. Reference arrays expose writable + storage and copy back in adapter order. +- [x] Lower fixed-shape array results through one validated returned-address + ABI and assign them into the native adapter result. Reject incomplete shape + or unsupported ownership before emission. +- [x] Preserve output-array Python identity in the outer ordinary call and do + not add a detached-copy fallback. +- [x] Replay `fcallback_array_f90` plus the combined array-storage callback and + artifact assertions in both build modes. + +### Phase 10F — Derived Arguments And Results + +- [x] Reuse the exact Phase 8/9 type identity, opaque wrapper, owner-retention, + and destroy/release actions for callback-local derived wrappers. Do not + expose aggregate layout or introduce callback-specific storage ownership. +- [x] Borrow callback input wrappers only for the callback invocation; convert + supported callback results to the completed native result storage and + release temporary wrapper ownership exactly once. +- [x] Validate exact runtime class/type identity before using a returned + derived address. Polymorphic, descriptor-backed, or unsupported derived + callback forms retain exact blockers. +- [x] Replay `fcallback_derived_f90` and the combined derived callback after the + Phase 9 constructor/class route is green. + +### Phase 10G — Fatal Errors, Re-entry, And Cleanup + +- [x] Route Python exceptions, argument-call mismatch, invalid callback result, + missing context, and cross-thread entry through one + traceback-plus-`abort()` action. Never return a fabricated value. +- [x] Prove nested same-thread callback calls use stack discipline and restore + the previous callable/context after the inner call. +- [x] Prove ordinary validation or setup failures before native entry release + every retained reference, and successful calls leave the callable reference + count unchanged. +- [x] Run fatal cases in subprocesses for both source/generated-`.pyi` builds + and assert the documented traceback/error text plus nonzero termination. + +### Phase 10H — Production Routing, Documentation, And Closure + +- [x] Add support-report lanes for callback context, scalar value/storage, + fixed strings, arrays, derived values, result conversion, same-thread + re-entry, and fatal errors. +- [x] Add one reduced compiled direct-plan node per dependency-closed lane and + update its migration-matrix row only after artifact and runtime parity. +- [x] Prove eligible callback units select the production wrapper-plan route; + unsupported callback policy must keep the whole generation unit on one + exact blocker. +- [x] Synchronize callback guide/reference, semantic `.pyi` reference, feature + matrix, callback README, source map, and checklist coverage with the direct + implementation. +- [x] Run focused policy/plan/backend tests, every callback wrapper node in + source/generated-`.pyi` modes, the wrapper suite excluding LAPACK, the + wrapper complexity checker, documentation checks, whitespace, and the + required static-analysis suite. +- [x] Close Phase 10 only when every supported immediate callback unit routes + directly, no callback plan falls back after generation starts, and every + stored/optional/asynchronous/cross-thread or incomplete callback form has an + exact retained blocker. Stop before Phase 11. + +Closure evidence (2026-07-16): callback policy, editable-plan validation, +binding/bridge artifacts, scalar/string/array/derived conversion, nested +same-thread re-entry, reference cleanup, and subprocess fatal-boundary tests +all passed through the direct route. The same 184-test focused batch and +449-test non-LAPACK wrapper replay used for Phase 9 closure cover the complete +immediate-callback matrix. Required static checks passed with the explicit +Radon base noted above, and implementation stopped before Phase 11. + +### Phase 10 Expansion Gate + +- [x] Inventory the public callback contract, semantic prototype records, + legacy lowering/codegen owners, source/generated-`.pyi` runtime fixtures, + context lifetime, re-entry/GIL behavior, exception/abort behavior, and every + supported scalar/string/array/derived argument-result combination. +- [x] Define the Phase 9/10/11 boundary and retain explicit blockers for stored, + optional, asynchronous, cross-thread, incomplete-signature, and unsupported + callback forms. +- [x] Split implementation into dependency-ordered Phase 10A-H sub-lanes with + policy, typed plan, validation, lowering, compiled parity, production + routing, documentation, and closure gates. + +## Phase 11 — Cross-Cutting Wrapper Suite Completion + +Implementation status: complete. The pre-Phase-11 ledger contained 236 +wrapper-plan nodes, five dual-route array parity nodes, 113 passing legacy-route +nodes, 95 non-generating nodes, and two deferred real-library nodes. The final +forced-plan sweep passed 435 of 449 non-real-library nodes before obsolete +dual-route artifact assertions were removed; its two shared implementation +gaps were Fortran-ordered strided ndarray validation and static `nopass` method +dispatch, both now resolved through existing policy/runtime paths. + +The ordered output aggregator now combines direct and hidden native results +with visible scalar, string, array, and derived writeback. It converts each +value once in public result order and releases every earlier Python reference +if a later conversion or tuple allocation fails; the former single-result and +"native result plus writeback" blockers are removed. + +Scope: existing wrapper tests whose generation units combine completed semantic +lanes or exercise build and runtime behavior rather than introducing one new +datatype lane. + +Implement in these dependency-ordered waves: + +1. reconcile the five reduced array dual-route nodes and remove stale Phase 7 + exclusion bookkeeping where their completed actual-source policy now permits + production routing; +2. migrate source/semantic-`.pyi` build modes, edited contracts, external + symbols, multiple-source linkage, and independent native bundles through one + shared route and planner; +3. migrate mixed scalar/string/array/handle/derived/module/class generation + units without adding per-test or per-datatype fallback; +4. migrate naming, generic interfaces, defined operators, OpenMP/runtime policy, + and remaining public-surface orchestration; and +5. require the live nondeferred ledger to contain only `wrapper-plan` or + justified `not-applicable` nodes before Phase 12 begins. + +- [x] Reconcile every remaining `legacy` or `dual-route` matrix row by owning + test area: `build_from_source`, `build_from_pyi`, `edit_pyi_contracts`, + `external_routines`, `multiple_files`, `naming`, `runtime_behavior`, and + `real_libraries`. +- [x] Group remaining rows into dependency-ordered waves by their actual + unsupported owner paths. Do not implement a broad test directory as one + special case and do not add per-test backend fallbacks. +- [x] For every newly discovered semantic or backend gap, expand the applicable + earlier lane or add an explicit sub-lane here, then follow the complete + policy -> plan -> backend -> emission -> compiled parity -> route sequence. +- [x] Prove source-driven and semantic-`.pyi`-driven builds use the same route + selector and wrapper planner while retaining their existing build assertions. +- [x] Prove edited-policy contracts, external symbols, multiple-source builds, + naming/generic interfaces, runtime policies, recursion, OpenMP, and real + library-independent native bundles preserve their existing assertions + through the wrapper-plan route. +- [x] Keep non-wrapper-generating tests, including layout and generated-`.pyi` + checks, marked `not-applicable` to route selection but passing in the same + suite. +- [x] Run every `tests/wrapper` test except + `test_real_blas_lapack.py` locally and in CI as the pre-cutover gate. +- [x] Finish this phase only when every nondeferred matrix row is either + `wrapper-plan` or justified `not-applicable`; no nondeferred row may remain + `legacy` or `dual-route`. BLAS/LAPACK rows remain + `deferred-real-library` until Phase 12. + +Closure evidence (2026-07-16): the Phase 11 ledger contains 344 canonical +wrapper-plan nodes, 95 justified non-generating nodes, two deferred +real-library nodes, and no legacy or dual-route node. The complete local +pre-cutover suite outside the shared BLAS/LAPACK file passed all 439 collected +tests. Mixed outputs use the ordered aggregator, Fortran-ordered strided array +validation reuses the shared array-actual runtime path, and static `nopass` +methods reuse the completed class invocation path; no per-test route or +backend fallback was added. + +## Phase 12 — Cutover And Removal + +Implementation status: complete. Local BLAS evidence is recorded below; +LAPACK execution remains intentionally CI-only. + +Local verification boundary: run the BLAS generation unit locally. Do not run +the LAPACK generation unit locally; make its wrapper-plan invocation runnable +in GitHub Actions and use that job for LAPACK parity and cutover evidence. + +External-interface parameter lists preserve native ABI order, while their +declarations may be topologically ordered from the plan's explicit array +extent-reference roles. This permits a later scalar extent dummy to be +declared before an earlier array dummy without reordering the native call. + +Cutover contract: source builds, semantic-`.pyi` builds, Makefile generation, +manifest replay, and strict-name validation all use completed policy -> +`WrapperPlan` -> `WrapperCodeGenerator`. The build API has no route selector, +rollback flag, or silent fallback; an unsupported owner path fails before any +backend or legacy lowering runs. + +- [x] Re-audit collected Python test nodes under `tests/wrapper` and reconcile + them with the migration matrix. No test may be missing from the matrix. +- [x] After every other migration row is complete, restore the full + `test_real_blas_lapack.py` run and any required native-cache preparation in + local opt-in verification and GitHub Actions. +- [x] Run BLAS locally through the canonical route using its existing contract, + import, ABI, and runtime assertions. Run the equivalent exact LAPACK node in + the dedicated GitHub Actions real-library matrix; do not run it locally. +- [x] Require every wrapper-generating test row to be `wrapper-plan`; no row + remains `legacy`, `dual-route`, or `deferred-real-library`. +- [x] Configure the complete `tests/wrapper` suite in CI with ordinary tests in + the main matrix and the full BLAS/LAPACK nodes in the cached real-library + matrix. +- [x] Confirm no wrapper build lane uses the old + `semantic_ir_to_codegen_ast()` path. The old lowering is no longer a supported + test owner and receives no focused compatibility coverage. +- [x] Remove route support tracking and fallback diagnostics; whole-generation + units now either validate and generate one plan or fail on exact owner-path + support diagnostics before emission. +- [x] Retain rollback only until the live ledger is reconciled, then remove it + in one cutover without compatibility flags or per-function fallback. +- [x] Do not move modified isolated nodes or printers back into the legacy + package during migration. After final cutover, remove the legacy package + pieces proven unused and keep `x2py.wrapper_codegen` as the canonical + generator rather than performing a second package rename. +- [x] Keep semantic `.pyi` emission under `x2py.wrapper_codegen.printers` and + retire focused tests of the old semantic AST, bridge, binding, and printer + implementation before deleting the legacy package. +- [x] Remove the temporary legacy route and its route diagnostics after every + live generation unit is supported; do not replace it with compatibility + shims or per-function fallback. +- [x] Remove migration-only dual-route orchestration after the complete existing + wrapper suite proves the wrapper-plan route and legacy rollback is no longer + supported. Keep the existing behavioral fixtures and assertions. +- [x] Keep source printers only for the remaining generated source fragments they + still own, or replace them with narrower emitters once the model layer is no + longer needed. + +Closure evidence (2026-07-16): the final live ledger contains 346 canonical +wrapper-plan nodes, 75 justified non-generating nodes, and zero legacy, +dual-route, or deferred nodes. The complete local suite outside the shared +real-library file passed 419 tests; the exact BLAS full-library node passed +locally; and the exact BLAS and LAPACK nodes are runnable as independent legs +of the cached GitHub Actions real-library matrix. LAPACK was intentionally not +run locally, so its runtime result remains CI evidence. Focused semantic and +compiled class/module policy tests passed 80 tests, all wrapper-codegen tests +passed 352 tests, and documentation plus structural layout checks passed 1,142 +tests. Ruff lint/format, Bandit, Vulture, the wrapper-codegen complexity check, +the Radon policy against explicit base `main`, advisory Radon complexity and +maintainability reports, and `git diff --check` all passed. + +## Verification + +- [x] Documentation changes run + `python3 -m pytest -q tests/docs/test_examples.py tests/docs/test_structure.py` + and `git diff --check`. +- [x] Wrapper-plan code changes run the affected existing `tests/wrapper` nodes, + the minimal intermediate contract tests required above, and the required + static-analysis suite from `AGENTS.md`. +- [x] Wrapper-codegen implementation changes pass + `python3 tools/check_wrapper_codegen_complexity.py` with no handler waiver. +- [x] Runtime wrapper tests cover every changed generated behavior. +- [x] Every migrated lane completed legacy-oracle comparison before cutover; + final tests now exercise only the canonical wrapper-plan route and retain the + existing behavior and ABI-relevant call assertions. +- [x] Structural dependency tests prove complete generator isolation: no + imports from `x2py.wrapper_codegen` to `x2py.codegen` or in the reverse + direction. +- [x] BLAS and LAPACK full-library wrapper tests remained excluded locally and in + GitHub Actions throughout Phases 0-11. At the explicit Phase 12 gate, enable + BLAS locally and in GitHub Actions, enable LAPACK only in GitHub Actions, and + keep local LAPACK execution disabled. + +## Completion Record + +- [x] The final report for each lane names the plan actions added, the binding + and bridge handlers they dispatch to, and the handoff specs validated. +- [x] No unsupported wrapper lane uses old lowering/codegen; focused tests now + target completed policy, `WrapperPlan`, `WrapperCodeGenerator`, or compiled + public behavior rather than `ir2ast.py` and `x2py.codegen` internals. +- [x] The final cutover report includes the completed `tests/wrapper` migration + matrix and confirms every wrapper-generating row uses the wrapper-plan route. +- [x] The final report includes focused verification commands and results. +- [x] The final report includes the changed-stage breakdown required by + `AGENTS.md` and names every test file added or updated with the behavior it + covers. + +## Post-Cutover Legacy Codegen Removal + +The legacy `x2py.codegen` package, `x2py/semantics/ir2ast.py`, and the obsolete +`x2py/compiling/python_wrapper.py` pipeline are removed together. No alias, +fallback, compatibility import, or rejection-only test preserves that route. + +Required behavior remains with its current owner: completed semantic policy +tests for semantic decisions, `tests/wrapper_codegen/` for plans and direct +source generation, and compiled `tests/wrapper/` cases for public Python +behavior and native ABI outcomes. Static-analysis baselines cover only source +that remains in the repository. +## Session Continuation Protocol + +The stable continuation prompt is: + +```text +Continue implementing the wrapper-plan migration checklist. +``` + +On continuation: read this checklist and `AGENTS.md`; inspect the dirty +worktree; choose the first unchecked dependency-closed item; replay the +existing passing wrapper test before extending a lane; implement code and tests +together; run required verification; and check items only from live evidence. +Do not reset unrelated user changes, infer missing policy in lowering, or use a +new fallback after direct plan generation starts. diff --git a/docs/old_docs/developper_guide.md b/docs/old_docs/developper_guide.md index a8c5cfa0e..e019def0b 100644 --- a/docs/old_docs/developper_guide.md +++ b/docs/old_docs/developper_guide.md @@ -75,7 +75,7 @@ Use these documentation roles consistently: | [fortran_parser.md](fortran_parser.md) | Maintainer inventory for the Fortran frontend | | [semantics.md](semantics.md) | Accepted semantic IR and datatype contract | | [pyi_format.md](pyi_format.md) | User-visible semantic `.pyi` syntax and roadmap | -| [wrapper_design_notes.md](wrapper_design_notes.md) | Clearly deferred wrapper policy, not current runtime support | +| [wrapper_design_notes.md](wrapper_design_notes.md) | Clearly deferred wrapper policy, not current native binding support | When adding a user example: @@ -210,7 +210,7 @@ implementation files. | Fortran wrapper orchestration | `x2py/wrapping.py` | `tests/wrapper/fortran/native_build/test_build_modes.py`, `tests/wrapper/fortran/multi_source/test_multi_source_builds.py` | | Semantic IR to codegen AST | `x2py/semantics/ir2ast.py` | `tests/semantics/test_ir2ast.py`, `tests/wrapper/` | | Fortran-to-C bridge and CPython binding | `x2py/codegen/bridges/fortran_to_c.py`, `x2py/codegen/bindings/c_to_python.py` | `tests/wrapper/` subject suites | -| Native compilation and runtime support | `x2py/compiling/`, `x2py/stdlib/x2py_runtime/` | `tests/wrapper/fortran/native_build/test_runtime_abi.py`, `tests/wrapper/fortran/native_build/test_build_modes.py` | +| Native compilation and binding support | `x2py/compiling/`, `x2py/binding_support/` | `tests/wrapper/fortran/native_build/test_runtime_abi.py`, `tests/wrapper/fortran/native_build/test_build_modes.py` | | Public API exports | `x2py/__init__.py` | `tests/parser/test_parser_public_entrypoints.py`, `tests/parser/c/test_c_public_api_skeleton.py` | | Executable Markdown examples | `README.md`, `docs/*.md` | `tests/tools/test_documentation_examples.py` | @@ -729,7 +729,7 @@ The main ownership boundaries are: reference handling, and CPython wrapper construction; - `x2py/codegen/printers/{fcode,ccode,cpythoncode}.py`: source rendering only; - `x2py/compiling/`: compiler commands and shared-library linking; and -- `x2py/stdlib/x2py_runtime/`: native runtime support copied into each build. +- `x2py/binding_support/`: native binding support copied into each build. Do not move semantic ownership or projection policy into printers. Do not infer source dependencies: multi-source builds compile in caller order, and the first diff --git a/docs/old_docs/examples.md b/docs/old_docs/examples.md index c70a561d9..53e04a29d 100644 --- a/docs/old_docs/examples.md +++ b/docs/old_docs/examples.md @@ -203,8 +203,10 @@ Exact NumPy scalars are part of the native contract. Passing ordinary Python numbers where a specific native dtype is required raises `TypeError` rather than silently changing the ABI conversion. -With no `--out-dir`, x2py writes intermediates under `__x2py__` beside the -first source and writes the extension beside that source. Use `--verbose` to +With no `--out-dir`, x2py writes intermediates and the ABI-suffixed extension +under `__x2py__` in the current working directory, while a direct CLI build +writes its stable `.so` alias there unless `--out` gives it an explicit path. +Use `--verbose` to print the direct compiler and linker commands. Use `--strict-wrapper-names` to reject public names that need Python keyword escaping or collision suffixes. @@ -726,10 +728,13 @@ from x2py import assess_semantic_wrap_readiness, pyi_text_to_semantic_module module = pyi_text_to_semantic_module( """ -from typing import Callable +from x2py.contracts import prototype + +@prototype +def objective(value: Float64) -> Float64: ... def integrate( - objective: Callable[[Float64], Float64], + callback: objective, x0: Float64 ) -> Float64: ... """, @@ -818,16 +823,18 @@ def fill_matrix( ### Complete Callback Signature ```python -from typing import Callable +from x2py.contracts import prototype + +@prototype +def objective(value: Float64) -> Float64: ... def integrate( - objective: Callable[[Float64], Float64], + callback: objective, x0: Float64 ) -> Float64: ... ``` -`Callable[..., Float64]` is accepted syntax but remains semantically -incomplete because the callback argument types are unknown. +The prototype names every callback argument and its result explicitly. ### Preserved Projection Metadata @@ -852,8 +859,8 @@ def integrate(objective: Procedure, x0: Float64) -> Float64: ... ``` This is blocked because callback argument order, argument types, and return -type are incomplete. Replacing `Procedure` with a complete supported -`Callable[[...], Return]` supplies the semantic signature. +type are incomplete. Replacing `Procedure` with a complete named prototype +supplies the semantic signature. ### Missing Compile-Time Constant diff --git a/docs/old_docs/fortran_wrapper.md b/docs/old_docs/fortran_wrapper.md index c5c72707a..df1b4c568 100644 --- a/docs/old_docs/fortran_wrapper.md +++ b/docs/old_docs/fortran_wrapper.md @@ -107,7 +107,7 @@ ordered Fortran source files -> merged public wrapper module and collision-safe Python names -> codegen AST -> Fortran bind(C) bridge - -> C/CPython binding and x2py runtime support + -> C/CPython binding and native binding support -> compile user sources and generated sources -> link one Python extension module ``` @@ -115,7 +115,7 @@ ordered Fortran source files The Fortran bridge converts non-interoperable Fortran contracts into a stable C ABI. The generated C layer validates Python and NumPy objects, manages Python references and wrapper-owned temporaries, calls the bridge, and projects native -results onto the documented Python API. The runtime support supplies shared +results onto the documented Python API. The native binding support supplies shared array, error, allocation, and ownership helpers. Typical generated artifacts are: @@ -124,7 +124,7 @@ Typical generated artifacts are: | --- | --- | | `bind_c__wrapper.f90` | Fortran-to-C ABI bridge | | `_wrapper.c` and `.h` | CPython extension binding | -| `x2py_runtime/` | Shared native runtime support | +| `binding_support/` | Header-only native binding support | | user and generated `.o`/`.mod` files | Native build intermediates | | `..so` | Importable extension on Linux | @@ -132,8 +132,10 @@ The extension name comes from the first generated semantic module. For a multi-source build, x2py merges the public surface into that extension and compiles sources in caller-supplied order. -Without `--out-dir`, x2py uses a private `__x2py__` build directory beside the -source and places the importable extension beside the source file. Generated +Without `--out-dir`, x2py writes generated artifacts, including the ABI-suffixed +extension, in a private `__x2py__` build directory in the current working +directory. A direct CLI build writes its stable `.so` import alias in +the current working directory unless `--out` gives it an explicit path. Generated Fortran and C wrapper sources remain build artifacts; users do not edit them to change the Python API. @@ -1318,11 +1320,11 @@ python3 -m x2py mesh.f90 solver.f90 --makefile --out-dir build --json make -f build/Makefile.x2py -j4 X2PY_FFLAGS=-O3 X2PY_CFLAGS=-O3 ``` -The Makefile covers user sources, generated wrappers, runtime support, and the +The Makefile covers user sources, generated wrappers, the header-only native binding support, and the shared-library link. It records resolved compilers and exposes `FC`, `CC`, `X2PY_LD`, `X2PY_FFLAGS`, `X2PY_CFLAGS`, and `X2PY_LDFLAGS`. User Fortran -sources are conservatively chained in supplied order; independent generated C -and runtime work may run in parallel. This target expects GNU Make and a POSIX +sources are conservatively chained in supplied order; generated bridge and C +binding work may run in parallel. This target expects GNU Make and a POSIX shell. Runtime tests: [`test_multi_source_builds.py`](../tests/wrapper/fortran/multi_source/test_multi_source_builds.py), diff --git a/docs/old_docs/pyi_format.md b/docs/old_docs/pyi_format.md index 156e2048c..1fd35bacf 100644 --- a/docs/old_docs/pyi_format.md +++ b/docs/old_docs/pyi_format.md @@ -217,8 +217,9 @@ files while the generated bridge is compiled: ``` Archives do not normally contain `.mod` files, so module directories remain -separate inputs. Standalone `@external` procedures require no `.mod` file because -the bridge emits their interface from the semantic contract. +separate inputs. Standalone `@external` procedures require no `.mod` file +because the bridge emits their implicit external declaration or required +explicit interface from the semantic contract. Required link cases are: @@ -363,13 +364,13 @@ The public annotations use semantic names, not raw C or Fortran spellings: | Complex | `Complex64`, `Complex128`, `Complex256` | | Text | `String` | | User types | class names and imported type names | -| Callables | `Callable`, `Callable[..., T]`, `Callable[[A, B], T]` | +| Callback prototypes | named `@prototype` declarations referenced by name | `Unknown` is intentionally rejected in `.pyi` annotations. Generated stubs must resolve or block unsupported source types instead of emitting unknown contracts. Current C callback placeholders such as `CFunctionPointer` can appear in -generated stubs when source callback policy is incomplete; edit them to a full -`Callable[[...], ...]` contract before expecting readiness to pass. +generated stubs when source callback policy is incomplete; replace them with a +complete named prototype before expecting readiness to pass. ## Storage Contracts @@ -450,7 +451,7 @@ Generated canonical metadata: | `Allocatable` | Fortran allocatable array storage | | `Pointer` | Fortran pointer array storage | | `PointerAssociation("runtime")` | pointer association is a runtime state rather than a declaration-time constant | -| `Name("native-name")` | source name cannot be represented directly as the Python target name | +| `SourceName("native-name")` | source name cannot be represented directly as the Python target name | | `FortranCharacterLength("n")` | Fortran character storage length for `String` contracts | | `FortranAllocatable` | Fortran scalar character storage is allocatable | | `Aliased` | native storage may be exposed across the Python boundary as an alias | @@ -463,7 +464,7 @@ Loaded compatibility metadata: | Metadata | Meaning | | --- | --- | -| `ORDER_C` | explicit C-oriented storage; this is also the default for plain multidimensional arrays | +| `ORDER_C` | explicit C-oriented storage in a Fortran contract | | `Contiguous` | source provenance says the array is contiguous | | `ArrayCategory("...")` | source array category provenance | | `SourceDims(...)` | source declaration dimensions | @@ -975,12 +976,12 @@ is a user contract applied to a declaration that was otherwise available to the wrapper, so the declaration remains printed and loadable as wrapper input. Names that are not valid Python identifiers are represented with `var[...]` for -data declarations, or with `Annotated[..., Name("native-name")]` for callable +data declarations, or with `Annotated[..., SourceName("native-name")]` for callable arguments: ```python var["class"]: Int32 -def f(class_: Annotated[Int32, Name("class")]) -> None: ... +def f(class_: Annotated[Int32, SourceName("class")]) -> None: ... ``` ## Projection Metadata @@ -1021,7 +1022,7 @@ Generated `.pyi` currently covers these exact-contract areas: | C primitive scalars | compiler-probed semantic dtype names when a target report is supplied | | Functions/subroutines | exact native argument order and direct return type | | Fortran scalar storage | `T`, `T[()]`, `Addr(Arg(...))`, `Returns[...]` | -| Arrays | shaped storage with extents, strided axes, `ORDER_F` for multidimensional Fortran arrays | +| Arrays | shaped storage with extents and strided axes; multidimensional order defaults from the selected native language | | Allocatable borrowed views | derived-type fields and target-backed module arrays, with `None` for unallocated storage | | Constants | `Final[T]` module variables | | C and Fortran enums | module-level `Final[...]` integer constants | @@ -1039,7 +1040,7 @@ Loaded but usually not generated from source today: | Area | Loaded behavior | | --- | --- | -| `Callable[[...], ...]` | complete callback/procedure signature metadata | +| named `@prototype` | complete callback/procedure signature metadata | | `Addr[n](T)` for `n > 1` | direct low-level pointer topology | | `ORDER_ANY` | edited orientation-independent array contract | | generic `Annotated` constraints | preserved semantic constraints | @@ -1071,7 +1072,7 @@ The loader intentionally rejects syntax that would be ambiguous or stale: Near-term format work: 1. Make C and Fortran callbacks/procedure pointers first-class by preserving - complete `Callable[[...], ...]` contracts from source. + complete named prototypes from source. 2. Add explicit pointer ownership, borrow, nullability, output-buffer and copy/readback policy so pointer-heavy C APIs can move beyond blockers. 3. Strengthen Fortran `character(len=...)` with length, kind, hidden-length ABI diff --git a/docs/old_docs/pyi_wrapper_checklist.md b/docs/old_docs/pyi_wrapper_checklist.md index 2df922d15..428b2aa47 100644 --- a/docs/old_docs/pyi_wrapper_checklist.md +++ b/docs/old_docs/pyi_wrapper_checklist.md @@ -217,8 +217,9 @@ different public API or runtime contract. `@external` generation and runtime parity. - [ ] One source containing several standalone procedures generates external declarations for all of them and exposes each at the extension root. -- [ ] `@external` makes the bridge emit an explicit interface and no module - `use`; a module procedure makes the bridge emit the correct `use `. +- [ ] `@external` makes the bridge emit an implicit external declaration or a + required explicit interface and no module `use`; a module procedure makes + the bridge emit the correct `use `. - [ ] `@external` composes with `@bind("native_name")`: the native external is called while the wrapper declaration and root export may use different names. - [ ] A handwritten external `.pyi` plus native artifacts builds without source diff --git a/docs/old_docs/quality.md b/docs/old_docs/quality.md index 3278f79a3..6f2ef0ebf 100644 --- a/docs/old_docs/quality.md +++ b/docs/old_docs/quality.md @@ -127,7 +127,7 @@ removed as redundant maintenance overhead. **Role:** generates edge cases for parsers, AST transforms, semantic IR, and code generation. -**Bugs found:** generated code-generation cases exposed quoted `Name(...)` +**Bugs found:** generated code-generation cases exposed quoted `SourceName(...)` emission. Generated preprocessing inputs also aligned raw Fortran and C macro handling around compiler-required errors. @@ -285,7 +285,7 @@ The `Fuzz` workflow runs deeper discovery every Monday and by manual dispatch: | --- | --- | --- | --- | | 2026-05-31 | Initial stack integration | Added configuration, CI, documentation, and Hypothesis tests. | Continue staged strictness rollout. | | 2026-05-31 | Bandit | Reviewed low-severity findings and confirmed no medium- or high-severity findings. | Re-review when command trust boundaries change. | -| 2026-05-31 | Hypothesis code generation | Added generated native-name escaping, stable synthetic-import ordering, and semantic-IR-to-Pyi parse-back invariants; fixed quoted `Name(...)` emission. | Keep storing minimized failures. | +| 2026-05-31 | Hypothesis code generation | Added generated native-name escaping, stable synthetic-import ordering, and semantic-IR-to-Pyi parse-back invariants; fixed quoted `SourceName(...)` emission. | Keep storing minimized failures. | | 2026-06-01 | Ruff formatting rollout | Formatted the historical Python tree and changed CI to `ruff format --check .`. | Continue complexity-policy ratchets. | | 2026-06-01 | Radon and Ruff complexity policy | Added `tools/check_radon_policy.py`, made the staged Radon policy blocking in CI, and lowered Ruff McCabe from `50` to `45`. | Continue hotspot refactors and later threshold ratchets toward `20`. | | 2026-06-02 | Historical mutation-derived tests | Added direct Fortran parser contracts and fixed the directory namespace encoding bug. | Keep the tests as normal regression coverage. | diff --git a/docs/old_docs/semantics.md b/docs/old_docs/semantics.md index 5b0977b9d..693546a5d 100644 --- a/docs/old_docs/semantics.md +++ b/docs/old_docs/semantics.md @@ -324,7 +324,7 @@ The converter does not silently invent wrapper policy. It attaches - unresolved typedef or unknown type references; - legacy parser reports carrying macro-dependent declarations; - variadic functions; -- function pointer/callback signatures without edited `.pyi` `Callable` +- function pointer/callback signatures without a resolved named prototype policy; - mutable numeric or `void *` pointer parameters without ownership, scalar-storage, raw-address, or array policy; @@ -399,11 +399,11 @@ use `Annotated[T[...], Constraint, ...]`. replacement projection, where the argument remains visible and a `Returns["name", T]` item carries the post-call value. -Plain multidimensional array notation is C-oriented (`ORDER_C`) by default. -Under the current Fortran generation policy, every multidimensional Fortran -array contract emits `ORDER_F`, including stride-aware assumed-shape arrays. -Rank-one storage has no C-versus-Fortran order distinction, so no order marker -is emitted for vectors. +Plain multidimensional array notation follows the selected native language: +Fortran contracts default to `ORDER_F` and C contracts to `ORDER_C`. +Generated contracts omit that default order; an order annotation records only +an intentional non-default layout. Rank-one storage has no C-versus-Fortran +order distinction, so no order marker is emitted for vectors. `ArrayCategory(...)`, `SourceDims(...)`, `LowerBounds(...)` and `Contiguous` are not part of newly generated canonical array annotations. They described @@ -849,7 +849,7 @@ only when a real array storage contract is known. ## Design Proposal: Self-Contained C Semantic `.pyi` Runtime Contract -> **Status: design only, not implemented runtime support.** x2py currently +> **Status: design only, not implemented native binding support.** x2py currently > parses C, converts the supported subset to semantic IR, emits and loads > semantic `.pyi`, and reports readiness. It does not currently generate, > lower, compile, or execute C wrappers. Every runtime behavior, wrapper error, @@ -1032,20 +1032,17 @@ represents pointer-backed array storage; do not additionally wrap it in `Addr(...)`. For multidimensional storage, order is orthogonal to rank, dimensions and -stride capability. `Annotated[Float64[:, :], ORDER_F]` denotes a rank-two -dense Fortran-contiguous array, while -`Annotated[Float64[::, ::], ORDER_F]` denotes a rank-two -Fortran-oriented strided array. Bare `Float64[::, ::]` retains -the default `ORDER_C` orientation, and -`Annotated[Float64[::, ::], ORDER_ANY]` imposes no C/F -orientation restriction. `Annotated[Float64[...][1:4], ORDER_F]` expresses -the corresponding Fortran-oriented rank-polymorphic contract. These spellings -define the semantic format; they are explicit because `ORDER_F` and -`ORDER_ANY` are non-default in a C-origin stub. Accepting either in a -runnable C Phase 1 wrapper requires the corresponding native routine to -accept that storage layout directly. For a rank-one array, `ORDER_C` and -`ORDER_F` do not distinguish storage, contiguous or strided, so no order -constraint is written. +stride capability. In a C contract, `Annotated[Float64[:, :], ORDER_F]` +denotes a rank-two dense Fortran-contiguous array, while +`Annotated[Float64[::, ::], ORDER_F]` denotes a rank-two Fortran-oriented +strided array. Bare `Float64[::, ::]` uses the selected native language's +default orientation, and `Annotated[Float64[::, ::], ORDER_ANY]` imposes no +C/F orientation restriction. `Annotated[Float64[...][1:4], ORDER_F]` +expresses the corresponding Fortran-oriented rank-polymorphic contract. +These spellings define the semantic format; `ORDER_F`, `ORDER_C`, and +`ORDER_ANY` are written only when they differ from the selected language's +default. For a rank-one array, `ORDER_C` and `ORDER_F` do not distinguish +storage, contiguous or strided, so no order constraint is written. For a multidimensional strided annotation, `ORDER_F` is orientation metadata, not a requirement that NumPy report `F_CONTIGUOUS`; non-unit strides remain part of the contract. @@ -1293,16 +1290,17 @@ later Pythonic adaptations. #### 6.4 Contiguity Without an explicit layout or stride form, array annotations such as `T[:]`, -`T[:, :]`, `T[n]`, and `T[...]` require C-contiguous numeric storage; a -generated C stub does not repeat this as `ORDER_C`. Explicit non-default -forms such as `Annotated[T[:, :], ORDER_F]`, -`Annotated[T[::, ::], ORDER_F]`, or -`Annotated[T[::, ::], ORDER_ANY]` are exact interfaces when -the native routine accepts that layout and all required metadata remains -visible in the signature. A bare multidimensional stride form such as -`T[:, ::]` is also exact when native metadata is visible, but retains -the implicit `ORDER_C` orientation. Automatic packing, copy-back, or -derivation of native metadata is a later Pythonic transformation. +`T[:, :]`, `T[n]`, and `T[...]` require the selected native language's +default numeric storage order: Fortran-contiguous for Fortran and +C-contiguous for C. Generated stubs do not repeat that default. Explicit +non-default forms such as `Annotated[T[:, :], ORDER_F]` in a C contract, +`Annotated[T[:, :], ORDER_C]` in a Fortran contract, or +`Annotated[T[::, ::], ORDER_ANY]` are exact interfaces when the native +routine accepts that layout and all required metadata remains visible in the +signature. A bare multidimensional stride form such as `T[:, ::]` is also +exact when native metadata is visible, but retains the language-derived +default orientation. Automatic packing, copy-back, or derivation of native +metadata is a later Pythonic transformation. For rank one, `T[:]` and `T[n]` are also the canonical Fortran-contiguous spelling; write `T[::]` when contiguity is not required. diff --git a/docs/old_docs/tutorial.md b/docs/old_docs/tutorial.md index f20af24f4..b43fbc2ac 100644 --- a/docs/old_docs/tutorial.md +++ b/docs/old_docs/tutorial.md @@ -52,7 +52,7 @@ ordered Fortran sources -> semantic IR -> codegen AST -> generated Fortran bind(C) bridge - -> generated C/CPython binding and runtime support + -> generated C/CPython binding and native binding support -> compiled and linked Python extension ``` @@ -264,8 +264,10 @@ assert value == np.float64(7.5) The exact NumPy scalar types are intentional. The wrapper validates the native ABI contract instead of silently converting arbitrary Python numeric objects. -Without `--out-dir`, intermediate files go into `__x2py__` beside the first -source and the extension is placed beside that source. Use `--verbose` to print +Without `--out-dir`, intermediate files and the ABI-suffixed extension go into +`__x2py__` in the current working directory, while a direct CLI build writes +its stable `.so` alias there unless `--out` gives it an explicit path. Use +`--verbose` to print the executed compiler and linker commands. ### 6. Understand The Generated Boundary @@ -277,9 +279,9 @@ The build lowers semantic IR through two native layers: 2. A generated C/CPython binding validates Python objects, manages ownership and references, invokes the bridge, and creates Python or NumPy results. -The x2py runtime support is compiled with those generated sources. The final -link combines user objects, the Fortran bridge, the CPython binding, and the -runtime into one extension module. Generated sources are build artifacts; the +The header-only native binding support is compiled as part of the generated +CPython binding. The final link combines user objects, the Fortran bridge, and +the CPython binding into one extension module. Generated sources are build artifacts; the public behavior is the documented semantic and wrapper contract. For a build-system-controlled workflow, generate sources and a GNU Make build @@ -558,7 +560,8 @@ Python return values. The loader accepts semantic interface syntax such as: ```python -from typing import Callable, Final +from typing import Final +from x2py.contracts import prototype nmax: Final[Int32] = 32 @@ -566,15 +569,18 @@ class state: count: Int32 values: Float64[nmax] +@prototype +def objective(value: Float64) -> Float64: ... + def integrate( - objective: Callable[[Float64], Float64], + callback: objective, x0: Float64 ) -> Float64: ... ``` -A complete `Callable[[...], Return]` can resolve a callback-signature -readiness blocker. A placeholder such as `Procedure` or -`Callable[..., Return]` remains incomplete because argument types are unknown. +A complete named prototype resolves a callback-signature readiness blocker. A +placeholder such as `Procedure` remains incomplete because its argument and +result types are unknown. Supported projection metadata such as `@native_call(...)` is parsed and preserved. The source-driven Fortran wrapper executes the built-in projection @@ -670,7 +676,7 @@ Use x2py for the behavior implemented and tested today: - generated and compiled CPython extensions from one or more ordered Fortran source files; -- generated Fortran `bind(C)` bridges, C/CPython bindings, and runtime support +- generated Fortran `bind(C)` bridges, C/CPython bindings, and native binding support for the contracts in the [Fortran wrapper guide](fortran_wrapper.md); - wrapper-relevant Fortran and C source-fact extraction; - compiler-preprocessed CLI workflows; diff --git a/docs/old_docs/wrapper_design_notes.md b/docs/old_docs/wrapper_design_notes.md index 475535c9d..e34c60d89 100644 --- a/docs/old_docs/wrapper_design_notes.md +++ b/docs/old_docs/wrapper_design_notes.md @@ -32,7 +32,7 @@ before generated wrappers should treat them as supported behavior. | Gap | Current risk | Proposed direction | | --- | --- | --- | -| Function pointers and callbacks | The parser can capture function-pointer shape, but semantic conversion does not yet preserve a complete callable contract that wrappers can use safely. | Round-trip callback signatures as a first-class semantic callable form, such as a dedicated callback type or `Callable[[...], ...]` plus native callback metadata. Keep wrapper readiness blocked until lifetime, threading, exception, context-pointer, and unregister policy is supplied. | +| Function pointers and callbacks | The parser can capture function-pointer shape, but some C callback declarations still lack a complete wrapper policy. | Round-trip callback signatures as named `@prototype` declarations and keep wrapper readiness blocked until lifetime, threading, exception, context-pointer, and unregister policy is supplied. | | Pointer ownership and array extents | Raw pointers, pointer-to-pointer values, unknown extents, output buffers, and arrays of pointers are ambiguous without user policy. | Keep exact pointer topology in semantic IR. Require explicit `.pyi` ownership, borrow, output, shape, nullability, and copy/readback policy before projecting to Python containers or NumPy arrays. | | Unions | `CUnion` identifies the native type, but it does not say which member is active or whether by-value union ABI is safe. | Continue representing named and anonymous unions explicitly with `CUnion`; require active-member/discriminant policy for high-level access. Prefer a compiled shim or target layout proof for by-value union calls; otherwise keep a readiness blocker. | | Bitfields | Bit width is parser-visible, but Python field access needs target layout, signedness, padding, and read/write rules. | Preserve bit width, declared base type, containing aggregate, and layout-sensitive attributes. Generate access through a compiled C shim or target layout probe; block direct field projection when layout cannot be proven. | @@ -61,12 +61,21 @@ The supported target is the API surface needed to produce or validate wrappers: functions, variables, structs, enums, typedefs, constants, arrays, pointers, callbacks, and the metadata needed for readiness decisions. -Generated CPython extension builds copy their bundled C/Python support sources -into an `x2py_runtime/` directory inside the build output. The generated C -extension includes `x2py_runtime/python_runtime.h`. These files are an -implementation detail of the generated extension, but their names are -intentionally x2py-specific so they do not look like user source or a generic -C wrapper. +Generated CPython extension builds copy their bundled C/Python support header +into a `binding_support/` directory inside the build output. The generated C +extension includes `binding_support/x2py_binding.h`. This header is an +implementation detail of the generated extension, but its name is intentionally +x2py-specific so it does not look like user source or a generic C wrapper. + +The support header is header-only: each helper has internal linkage and is +eligible for inlining when the generated binding translation unit is compiled. +There is no separately compiled or linked support object. It exposes a +deliberately small `x2py_*` mechanical API: scalar type matching, scalar +unpacking, scalar creation as a Python or NumPy object, and release of a +bridge-owned allocation. The generated binding passes the completed NumPy type, +layout, ownership, and mutation decisions into those operations. Native support +must not infer a layout, accept a different dtype, or choose ownership behavior +from a value at runtime; those are completed wrapper-plan decisions. Generated CPython extensions should expose useful NumPy-style docstrings on the Python-visible API. The CPython wrapper layer owns this generation because it has @@ -91,8 +100,8 @@ attributes. Verbose wrapper builds should print the exact compiler command lines they run, not only the source or target being compiled. The printed command should be shell-quoted so users can copy it to reproduce object compilation, generated -wrapper compilation, runtime support compilation, and final shared-library -linking. +wrapper compilation (including its header-only native binding support), and +final shared-library linking. Normal C parsing uses a real compiler preprocessor first. Macro expansion, conditional compilation, token paste, stringify, and include resolution belong diff --git a/docs/user/examples/recipes/build-and-import-cli.md b/docs/user/examples/recipes/build-and-import-cli.md index 215b1e315..c1f5cd927 100644 --- a/docs/user/examples/recipes/build-and-import-cli.md +++ b/docs/user/examples/recipes/build-and-import-cli.md @@ -29,13 +29,12 @@ end module fruntime_abi_f90 ```bash python3 -m x2py tests/data/fortran/wrapper/fruntime_abi_f90.f90 \ - --wrap \ --out-dir build/fruntime_abi \ --json ``` -Recognizable Fortran sources default to `--wrap` when no inspection stage is -selected, so this is equivalent: +Recognizable Fortran sources select the wrapper build when no inspection stage +is selected, so this is equivalent: ```bash python3 -m x2py tests/data/fortran/wrapper/fruntime_abi_f90.f90 \ diff --git a/docs/user/examples/recipes/build-multiple-fortran-sources.md b/docs/user/examples/recipes/build-multiple-fortran-sources.md index 32ddca8dd..dbd26c6f2 100644 --- a/docs/user/examples/recipes/build-multiple-fortran-sources.md +++ b/docs/user/examples/recipes/build-multiple-fortran-sources.md @@ -20,7 +20,6 @@ merged extension: python3 -m x2py \ tests/data/fortran/wrapper/first_api.f90 \ tests/data/fortran/wrapper/second_api.f90 \ - --wrap \ --out-dir build/multi_api \ --json ``` diff --git a/docs/user/examples/recipes/generate-editable-makefile.md b/docs/user/examples/recipes/generate-editable-makefile.md index ac0fca4e4..762355fd3 100644 --- a/docs/user/examples/recipes/generate-editable-makefile.md +++ b/docs/user/examples/recipes/generate-editable-makefile.md @@ -17,13 +17,12 @@ manifest is the source of truth used to generate the Makefile. ```bash python3 -m x2py tests/data/fortran/wrapper/fruntime_abi_f90.f90 \ - --wrap \ --makefile \ --out-dir build/fruntime_abi \ --json ``` -This writes generated wrapper sources, runtime support, dependency files, and +This writes generated wrapper sources, the header-only native binding support, dependency files, and `build/fruntime_abi/Makefile.x2py`. For a semantic `.pyi` contract with native implementation sources, use the same @@ -31,7 +30,6 @@ mode with explicit native inputs: ```bash python3 -m x2py contracts/fruntime_abi_f90.pyi \ - --wrap \ --native-fortran-sources native/fruntime_abi_f90.f90 \ --native-fortran-flags="-O3 -fopenmp" \ --out-dir build/fruntime_abi \ @@ -68,11 +66,11 @@ X2PY_C_DOCS_END --> ## Notes - `--makefile` generates the build plan without compiling immediately. -- `--makefile` is a wrapper-build option and must be used with `--wrap`. +- `--makefile` selects the editable wrapper-build mode directly. - `--makefile` and `--verbose` are mutually exclusive. - `.pyi` Makefile generation is replayable through - `python3 -m x2py --build-manifest build/fruntime_abi/x2py-build.json --wrap --makefile` + `python3 -m x2py --build-manifest build/fruntime_abi/x2py-build.json --makefile` or buildable through - `python3 -m x2py --build-manifest build/fruntime_abi/x2py-build.json --wrap`. + `python3 -m x2py --build-manifest build/fruntime_abi/x2py-build.json`. - User Fortran sources remain in caller-provided order. Generated independent objects may be built in parallel by Make. diff --git a/docs/user/examples/recipes/semantic-pyi-contracts.md b/docs/user/examples/recipes/semantic-pyi-contracts.md index 8dd3318a0..dba18a36c 100644 --- a/docs/user/examples/recipes/semantic-pyi-contracts.md +++ b/docs/user/examples/recipes/semantic-pyi-contracts.md @@ -34,7 +34,6 @@ you provide the native artifacts explicitly: ```bash python3 -m x2py path/to/module.pyi \ - --wrap \ --native-objects path/to/module.o path/to/support.a \ --native-include-dir path/to/mod-files path/to/vendor-mod-files \ --out-dir build/module diff --git a/docs/user/examples/recipes/use-python-inspection-apis.md b/docs/user/examples/recipes/use-python-inspection-apis.md index ecb656a02..3812819e3 100644 --- a/docs/user/examples/recipes/use-python-inspection-apis.md +++ b/docs/user/examples/recipes/use-python-inspection-apis.md @@ -108,10 +108,13 @@ from x2py import assess_semantic_wrap_readiness, pyi_text_to_semantic_module module = pyi_text_to_semantic_module( """ -from x2py.contracts import Callable, Float64 +from x2py.contracts import Float64, prototype + +@prototype +def objective(value: Float64) -> Float64: ... def integrate( - objective: Callable[[Float64], Float64], + callback: objective, x0: Float64 ) -> Float64: ... """, diff --git a/docs/user/getting-started/beginner-workflow.md b/docs/user/getting-started/beginner-workflow.md index a128d1ae5..ae8049ea0 100644 --- a/docs/user/getting-started/beginner-workflow.md +++ b/docs/user/getting-started/beginner-workflow.md @@ -38,7 +38,7 @@ under version control. Do not commit `build/`. @@ -109,7 +109,7 @@ You normally do not need to open generated files. When debugging, expect | Artifact | Purpose | | --- | --- | -| `x2py_runtime/` | shared runtime support sources | +| `binding_support/` | header-only native binding support | | `.o` and `.mod` files | native intermediates | | `.` | importable extension | diff --git a/docs/user/getting-started/first-wrapped-function.md b/docs/user/getting-started/first-wrapped-function.md index 11b22b7af..7094deabc 100644 --- a/docs/user/getting-started/first-wrapped-function.md +++ b/docs/user/getting-started/first-wrapped-function.md @@ -17,7 +17,7 @@ Reuse the same `scale.f90` input from the [README Quick Start](../../../README.md#quick-start). The generated Python call accepts two `numpy.float64` values and returns a -`numpy.float64` result. +Python `float` result. ## Build @@ -43,11 +43,11 @@ import scale result = scale.scale(np.float64(3.0), np.float64(2.5)) -assert isinstance(result, np.float64) -assert result == np.float64(7.5) +assert isinstance(result, float) +assert result == 7.5 ``` -The checked call returns `numpy.float64(7.5)`. +The checked call returns the Python value `7.5`. ## Inspect The Generated Signature @@ -77,14 +77,17 @@ caller passes references. The semantic `.pyi` is a native contract, not an ordinary pure-Python type stub. Do not edit it during this first workflow; the Semantic `.pyi` Format reference explains the complete grammar later. +Fortran `intent` is not printed into the semantic `.pyi`. It helps generate the +initial Python argument/result projection, but the visible signature, +`Returns[...]`, and ordered `@native_call` list are the wrapper authority after +the contract is loaded. The compiled Fortran procedure retains its own `intent`. + ## Failure Mode: Wrong Scalar Type Native scalar arguments use exact NumPy dtypes. A plain Python `float` is not a replacement for `numpy.float64` at this boundary: ```python -from x2py.contracts import raises - scale.scale(3.0, 2.5) # raises TypeError ``` @@ -107,6 +110,6 @@ generated `.pyi` contract. ## Evidence The linked `scale.f90` input is checked against the repository fixture by -[`test_documentation_examples.py`](../../../tests/docs/test_examples.py). +[`test_examples.py`](../../../tests/docs/test_examples.py). The default extension name and `7.5` runtime result are checked by [`test_build_modes.py`](../../../tests/wrapper/fortran/build_from_source/test_build_modes.py). diff --git a/docs/user/getting-started/index.md b/docs/user/getting-started/index.md index 88f1bcbd2..93853ad65 100644 --- a/docs/user/getting-started/index.md +++ b/docs/user/getting-started/index.md @@ -59,6 +59,6 @@ failures are routed through [Troubleshooting](../troubleshooting/index.md). The standalone example used throughout this section is checked against its fixture by -[`test_documentation_examples.py`](../../../tests/docs/test_examples.py), +[`test_examples.py`](../../../tests/docs/test_examples.py), and its `7.5` runtime result is checked by [`test_build_modes.py`](../../../tests/wrapper/fortran/build_from_source/test_build_modes.py). diff --git a/docs/user/getting-started/verification.md b/docs/user/getting-started/verification.md index c5bcc12e1..cdb3c6704 100644 --- a/docs/user/getting-started/verification.md +++ b/docs/user/getting-started/verification.md @@ -77,10 +77,10 @@ python3 -m x2py scale.f90 \ The command must create: - an importable `scale` extension under `build/verify`; and -- generated native bridge, object, runtime-support, and extension files. +- generated native bridge, object, native-support, and extension files. Import the extension from that build directory: @@ -139,7 +139,7 @@ the full GitHub Actions matrix is the final cross-version evidence. ## Evidence The linked `scale.f90` input is checked against the repository fixture by -[`test_documentation_examples.py`](../../../tests/docs/test_examples.py). +[`test_examples.py`](../../../tests/docs/test_examples.py). Native artifact placement and runtime calls are checked by [`test_build_modes.py`](../../../tests/wrapper/fortran/build_from_source/test_build_modes.py) and diff --git a/docs/user/guide/allocatables.md b/docs/user/guide/allocatables.md index fca9a0a27..0557adaba 100644 --- a/docs/user/guide/allocatables.md +++ b/docs/user/guide/allocatables.md @@ -56,8 +56,11 @@ def maybe_resize(values: Allocatable[Float64[:]] | None = ...) -> None: ... That spelling is valid only for optional callable arguments. Do not use `Allocatable[T[...]] | None` for module variables, derived-type fields, or -function results; those surfaces return a present handle, and unallocated state -is represented inside that handle. +function results; those surfaces return a present handle. Module variables, +fields, allocatable output dummies, function results, and handles changed by +later operations may be unallocated. The handle then reports +`allocated is False` and `to_numpy() is None`; this is descriptor state, not an +absent optional argument. Passing a handle to `Allocatable[T[...]]` passes the native descriptor. Passing the same allocated handle to a normal `T[...]` argument uses ordinary Fortran @@ -71,17 +74,18 @@ rejected for `Allocatable[T[...]]` descriptor parameters because a NumPy array does not carry a native allocatable descriptor. `h.to_numpy()` is the explicit extraction operation. It returns `None` when the -handle is unallocated. When policy proves live aliasing is safe, it returns a -borrowed NumPy view; when live aliasing is unsafe but copying is supported, it -returns a read-only detached copy. Users can call `.copy()` on any returned -NumPy array when they need independent lifetime. +handle is unallocated. Otherwise, it returns a live mutable NumPy view of the +current native allocation. It never creates an automatic detached snapshot or +copy. Users who need independent storage must explicitly call `.copy()` on the +returned array. A borrowed view is a NumPy array that points at storage Python does not own. Mutating the view mutates the owner. Deallocating or reallocating the owner can -make existing views stale, so copy the view when Python needs an independent -lifetime. Each fresh extraction starts at the array's current native lower -bounds; changing lower bounds during native reallocation must not offset the -first element exposed to NumPy. +make an existing view stale. Accessing a stale view is unsupported and may +crash the process; discard it and call `to_numpy()` again after the native state +changes. Each fresh extraction inspects the current descriptor and starts at +the current native lower bounds. Changing lower bounds during native +reallocation must not offset the first element exposed to NumPy. An allocatable array returned by a function or hidden output is different from a borrowed module or field handle. x2py transfers the result into persistent @@ -140,9 +144,10 @@ def update_scale( Passing `None` creates a present but unallocated call-local descriptor. Omitting a defaulted scalar descriptor argument creates native optional absence, so `present(scale)` is false. Passing a value creates a present allocated -call-local descriptor. An unallocated function result or projected output -becomes `None`. Ordinary scalar projection rules still apply: `intent(out)` uses -`Allocatable(Return("name", j))`, while `intent(inout)` uses +call-local descriptor. A projected output becomes `None` when its descriptor is +unallocated, including an allocatable scalar function result. Ordinary scalar +projection rules still apply: +`intent(out)` uses `Allocatable(Return("name", j))`, while `intent(inout)` uses `Allocatable(Arg(i))` plus a matching `Returns["name", T] | None` readback. The singular `result=Allocatable(Return(j))` mapping describes the native function result and places it among any other Python results. @@ -166,7 +171,7 @@ Create `allocations.f90`: module storage implicit none real(8), allocatable, target :: shared_values(:) - real(8), allocatable :: snapshot_values(:) + real(8), allocatable :: plain_values(:) contains function make_values(count) result(values) integer(4), intent(in) :: count @@ -195,27 +200,27 @@ contains shared_values = [(1.0_8 * index, index = 1, count)] end subroutine allocate_shared - subroutine allocate_snapshot(count) + subroutine allocate_plain(count) integer(4), intent(in) :: count integer(4) :: index - if (allocated(snapshot_values)) deallocate(snapshot_values) - allocate(snapshot_values(count)) - snapshot_values = [(3.0_8 * index, index = 1, count)] - end subroutine allocate_snapshot + if (allocated(plain_values)) deallocate(plain_values) + allocate(plain_values(count)) + plain_values = [(3.0_8 * index, index = 1, count)] + end subroutine allocate_plain subroutine release_shared() if (allocated(shared_values)) deallocate(shared_values) end subroutine release_shared - subroutine scale_snapshot(scale) + subroutine scale_plain(scale) real(8), intent(in) :: scale - snapshot_values = scale * snapshot_values - end subroutine scale_snapshot + plain_values = scale * plain_values + end subroutine scale_plain - subroutine release_snapshot() - if (allocated(snapshot_values)) deallocate(snapshot_values) - end subroutine release_snapshot + subroutine release_plain() + if (allocated(plain_values)) deallocate(plain_values) + end subroutine release_plain real(8) function shared_sum() result(total) total = sum(shared_values) @@ -224,14 +229,15 @@ end module storage ``` Inspecting `allocations.f90` prints allocatable array handles for module -storage, descriptor results, and descriptor arguments. Metadata such as -`Aliased` can still wrap the handle to describe owner or transfer policy: +storage, descriptor results, and descriptor arguments. `Aliased` remains a +language-neutral fact that native storage may be externally aliased or +addressed. It does not change `to_numpy()` extraction semantics: ```python from x2py.contracts import Addr, Aliased, Allocatable, Annotated, Arg, Float64, Int32, Returns, native_call shared_values: Annotated[Allocatable[Float64[:]], Aliased] -snapshot_values: Allocatable[Float64[:]] +plain_values: Allocatable[Float64[:]] @native_call([Addr(Arg(0))]) def make_values( @@ -248,24 +254,27 @@ def allocate_shared( ) -> None: ... @native_call([Addr(Arg(0))]) -def allocate_snapshot( +def allocate_plain( count: Int32 ) -> None: ... def release_shared() -> None: ... -def scale_snapshot( +@native_call([Addr(Arg(0))]) +def scale_plain( scale: Float64 ) -> None: ... -def release_snapshot() -> None: ... +def release_plain() -> None: ... def shared_sum() -> Float64: ... ``` -`snapshot_values` is not written as `Snapshot[...]`. For arrays, detached -copies are an extraction policy of the allocatable handle, not a separate -public array type. Whole-object snapshots are a separate derived-object feature. +`plain_values` and `shared_values` have the same extraction behavior: a fresh +`to_numpy()` call returns a live view of the current allocation or `None`. +`Aliased` remains present on `shared_values` because the native declaration +supplies the corresponding addressability fact. It does not change the +allocatable-array extraction mode. Build it: @@ -298,21 +307,22 @@ view = shared.to_numpy() view[0] = np.float64(10.0) assert api.shared_sum() == np.float64(15.0) -api.allocate_snapshot(np.int32(3)) -snapshot = api.snapshot_values.to_numpy() -np.testing.assert_array_equal(snapshot, np.array([3.0, 6.0, 9.0], dtype=np.float64)) -assert not snapshot.flags.writeable +api.allocate_plain(np.int32(3)) +plain_view = api.plain_values.to_numpy() +plain_copy = plain_view.copy() +plain_view[0] = np.float64(12.0) -api.scale_snapshot(np.float64(2.0)) -np.testing.assert_array_equal(snapshot, np.array([3.0, 6.0, 9.0], dtype=np.float64)) +api.scale_plain(np.float64(2.0)) +np.testing.assert_array_equal(plain_copy, np.array([3.0, 6.0, 9.0], dtype=np.float64)) np.testing.assert_array_equal( - api.snapshot_values.to_numpy(), - np.array([6.0, 12.0, 18.0], dtype=np.float64), + api.plain_values.to_numpy(), + np.array([24.0, 12.0, 18.0], dtype=np.float64), ) ``` -Do not access `view` after `api.release_shared()`; native deallocation makes -the previous borrowed view stale. +Do not access `view` after `api.release_shared()`, or `plain_view` after +`api.release_plain()` or another reallocation. Native storage changes make the +previous views stale, and accessing a stale view is unsupported and may crash. ## Output And Function Results @@ -350,8 +360,14 @@ Allocatable character arrays use fixed-width NumPy bytes storage. Create ```fortran module character_names implicit none - character(len=:), allocatable :: stored_names(:) contains + function make_names() result(names) + character(len=:), allocatable :: names(:) + + allocate(character(len=3) :: names(2)) + names = [character(len=3) :: "red", "sky"] + end function make_names + subroutine replace_names(names) character(len=:), allocatable, intent(inout) :: names(:) integer :: count @@ -372,14 +388,15 @@ end module character_names ``` The generated `.pyi` represents a fixed-length rank-one character array as -`String[4][::]`. A deferred-length allocatable rank-one array uses the two-axis +`String[n][::]`, where `n` is its fixed element length. A deferred-length +allocatable rank-one array uses the two-axis handle spelling `Allocatable[String[:][:]]`, so the element width can come from the native allocation at runtime: ```python from x2py.contracts import Allocatable, Returns, String -stored_names: Allocatable[String[:][:]] +def make_names() -> Allocatable[String[:][:]]: ... def replace_names( names: Allocatable[String[:][:]] @@ -403,7 +420,7 @@ sys.path.insert(0, "build/character_allocatables") import character_allocatables api = character_allocatables.character_names -names = api.stored_names +names = api.make_names() assert api.replace_names(names) is names assert names.to_numpy().dtype.itemsize == 5 assert names.to_numpy().tolist() == [b"red ", b"blue "] @@ -415,20 +432,30 @@ handle-typed parameter. When extracting character storage, x2py uses NumPy bytes dtype `S`; Unicode (`U`) and object (`O`) arrays are not descriptor-handle substitutes. +Projected writable descriptor mutation requires a handle with persistent +wrapper-owned standard-descriptor storage, such as the owned result returned by +`make_names()`. A borrowed module handle can be passed to a read-only descriptor +argument through descriptor facts, but it cannot be passed to a projected +writable descriptor argument: native mutation of a call-local reconstructed +descriptor would not update the module handle reliably. + ## Module Handles And Views An allocatable module array is native-owned. Reading the Python attribute returns an `Allocatable[T[...]]` handle, not `ndarray | None`. The module's allocation routines create and release the storage. `h.to_numpy()` returns the -current view, detached copy, or `None` according to completed policy and current -allocation state. When the Fortran declaration has `target`, the generated -`.pyi` marks the handle with `Aliased`. `Aliased` is not an ownership mode; it -says x2py may expose the native storage through an alias. - -A plain allocatable module array remains wrappable. Its handle can still report -unallocated state. If policy cannot expose a live view safely, `to_numpy()` -returns a read-only detached copy when that path is implemented, or wrapper -readiness blocks with a clear diagnostic. +current live view or `None` according to the current allocation state. When the +Fortran declaration has `target`, the generated `.pyi` marks the handle with +`Aliased`. `Aliased` is not an ownership mode or an extraction selector; it +records native addressability for pointer association, raw-address, foreign-pointer, +and related policy. + +A plain allocatable module array has the same extraction contract as an +`Aliased` one. The wrapper uses the completed descriptor mechanism to inspect +the current allocation without copying. If the backend cannot expose a live +view through a supported mechanism, wrapper readiness blocks with a clear +diagnostic. Call `.copy()` explicitly when independent Python-owned storage is +required. A supported allocatable component belongs to its containing native derived-type instance. The generated wrapper owns that native instance. The field exposes an @@ -447,10 +474,28 @@ independent = view.copy() ## Limitations -- Allocatable scalar derived-type argument replacement is blocked. +- A wrapper-owned allocatable scalar derived result can be passed to a + compatible ordinary, target, allocatable, allocatable-target, input-only + pointer, or value dummy. The generated typed holder preserves the same Python + object and writes allocation changes back to that holder. +- An allocatable scalar derived module variable is a live nullable field proxy, + and it can satisfy a compatible allocatable dummy through a scoped + `move_alloc` transaction. The allocation is moved into an exact typed local + holder, passed to the native procedure, and restored exactly once; no object + address substitutes for the module descriptor and no descriptor crosses the + interoperable boundary. +- The complete ordinary, `TARGET`, `ALLOCATABLE`, `ALLOCATABLE,TARGET`, + `POINTER`, and `VALUE` compatibility rules—including empty state, + multi-argument cleanup, and deliberate errors—are in the later Wrapping + Derived Types guide under “Scalar Actuals And Native Dummies.” - Mutable scalar deferred-length character storage is blocked. -- Borrowed views require a proved native or wrapper owner and `Aliased` - storage when the owner is a module variable. +- Plain derived module objects use typed module-specific member access; + `Aliased` is needed only for policies that require a direct native address. + Allocatable module-array handles use their standard descriptor path and have + the same live-view extraction contract with or without `Aliased`. +- Borrowed module handles do not provide the persistent direct descriptor + handoff required by projected writable descriptor arguments; use an owned + result handle for that operation. - An edited `.pyi` cannot relabel a native-owned descriptor as Python-owned. Use an implemented owned-result handle or copy an extracted NumPy value when Python needs independent storage. diff --git a/docs/user/guide/arrays.md b/docs/user/guide/arrays.md index 38de3d558..2e31a941f 100644 --- a/docs/user/guide/arrays.md +++ b/docs/user/guide/arrays.md @@ -35,12 +35,12 @@ contains values = values + 1.0_8 end subroutine shift - function automatic_vector(size) result(values) - integer(4), intent(in) :: size - real(8) :: values(size) + function automatic_vector(count) result(values) + integer(4), intent(in) :: count + real(8) :: values(count) integer(4) :: index - values = [(2.0_8 * index, index = 1, size)] + values = [(2.0_8 * index, index = 1, count)] end function automatic_vector end module array_ops ``` @@ -48,25 +48,25 @@ end module array_ops Inspecting `arrays.f90` prints these array contracts: ```python -from x2py.contracts import Addr, Annotated, Arg, Float64, Int32, ORDER_F, native_call +from x2py.contracts import Addr, Arg, Float64, Int32, native_call @native_call([Addr(Arg(0)), Addr(Arg(1)), Arg(2)]) def scale_matrix( rows: Int32, columns: Int32, - values: Annotated[Float64[rows, columns], ORDER_F] + values: Float64[rows, columns] ) -> None: ... @native_call([Addr(Arg(0)), Arg(1)]) def shift( size: Int32, - values: Float64[size - 1 - 0 + 1] + values: Float64[size] ) -> None: ... @native_call([Addr(Arg(0))]) def automatic_vector( - size: Int32 -) -> Float64[size]: ... + count: Int32 +) -> Float64[count]: ... ``` Build it: @@ -109,8 +109,10 @@ an automatic rank-one result. Other supported contracts can use `Float64[:]`, `Float64[3]`, `Float64[::]`, `Float64[Flat]`, or `Float64[...]`. The element name maps to an exact NumPy dtype; see [Data Types](data-types.md). -Dimension expressions constrain extents. Python remains zero-indexed even when -the native declaration has non-default lower bounds. +Dimension expressions constrain extents, not source lower/upper-bound +spellings. The native dimension `0:size-1` therefore becomes the public extent +`size`. Python remains zero-indexed even when the native declaration has +non-default lower bounds. ## Validation @@ -132,6 +134,31 @@ runs. Use `numpy.asfortranarray` or `order="F"` for a multidimensional contract that requires Fortran orientation, as shown by `matrix` in the complete example. +Layout annotations describe a deliberate non-default storage representation; +they do not request an automatic conversion. Plain multidimensional +Fortran-facing arrays already pass Fortran-contiguous storage with their logical +axes unchanged. `ORDER_C` passes the same C-contiguous data address without +copying and constructs the Fortran bridge view with reversed axes. For example, +a C-order Python shape `(2, 3)` is a Fortran bridge shape `(3, 2)` over the +same six elements. Use `ORDER_C` only when the native operation intentionally +accepts that transposed storage view. + +Add `COPY_F` when Python should accept C-contiguous storage but native Fortran +must observe the same logical axes in Fortran order: + +```python +values: Annotated[Float64[:, :], ORDER_C, COPY_F] +``` + +The binding owns this complete representation lifecycle. It creates an +F-contiguous NumPy temporary before the call, passes that ordinary F-order +buffer through the unchanged bridge path, copies values back into the original +C-order array after the call, and releases the temporary. Projected results +return the original C-order object. Native `intent(in)` remains a property of +the native procedure call; neither the semantic `.pyi` nor the bridge temporary +needs a separate direction annotation for `COPY_F`. The bridge performs neither +half of this argument conversion. + Rank-one contiguous arrays can satisfy their documented contiguous contract without a meaningful row/column distinction. Legacy fixed-form array contracts are contiguous-only. A modern Fortran dummy is stride-aware only when its diff --git a/docs/user/guide/callbacks.md b/docs/user/guide/callbacks.md index f123b9562..62557a1b3 100644 --- a/docs/user/guide/callbacks.md +++ b/docs/user/guide/callbacks.md @@ -9,16 +9,16 @@ status: maintained # Callbacks x2py supports Python callbacks invoked immediately during one wrapped native -call. Callback annotations are Fortran-facing: they describe the procedure -signature that Fortran will call, then x2py lifts those arguments into Python -objects for the callback invocation. +call. A semantic `.pyi` declares each native callback shape once as a named +prototype, then callback-taking procedures refer to that prototype by name. This is the opposite direction from a normal `.pyi` function signature. A normal function signature describes how Python calls the wrapper; x2py may -lower that call into any compatible native bridge shape. A `Callable[[...], T]` -argument describes how Fortran calls the callback adapter, so the argument -order, value/reference passing, explicit or missing callback intent, rank, -shape, character length, and result shape are part of the callback contract. +lower that call into any compatible native bridge shape. A `@prototype` +declaration describes how native code calls the callback adapter, so argument +order, value/reference passing, rank, shape, character length, and result shape +are part of the callback contract. Native callback direction is deliberately +not repeated in semantic `.pyi`. ## Complete Callback Example @@ -74,82 +74,61 @@ is unsupported unless the value is explicitly copied. ## Callback Values -Callback argument wrappers are valid only inside `Callable[[...], T]`: +Callback arguments use ordinary semantic types. Native code passes them by +reference unless the prototype applies the one ABI override, `Value(T)`: ```python -from x2py.contracts import Callable, Float64, In, InOut, Int32, Out, PassByRef - -callback: Callable[ - [ - Int32, - PassByRef(Float64), - In(Float64), - Out(Float64), - InOut(Float64), - In(Float64[n]), - ], - None, -] +from x2py.contracts import Float64, Int32, Value, prototype + +@prototype +def update_values( + count: Int32, + scale: Value(Float64), + values: Float64[count], +) -> None: ... + +def apply_update(callback: update_values, count: Int32) -> None: ... ``` -The wrappers mean: +The spellings mean: | Callback spelling | Fortran callback dummy | Python callback object | | --- | --- | --- | -| `Int32` | scalar `value` dummy | Python scalar value | -| `PassByRef(Float64)` | scalar by reference with missing intent | rank-zero NumPy scalar storage | -| `In(Float64)` | scalar by reference with `intent(in)` | Python scalar value | -| `Out(Float64)` | scalar by reference with `intent(out)` | rank-zero NumPy scalar storage | -| `InOut(Float64)` | scalar by reference with `intent(inout)` | rank-zero NumPy scalar storage | -| `Float64[n]` | array with missing intent | NumPy array view | -| `In(Float64[n])` | array with `intent(in)` | read-only NumPy array view | -| `Out(Float64[n])` | array with `intent(out)` | writable NumPy array view | -| `InOut(Float64[n])` | array with `intent(inout)` | writable NumPy array view | +| `Int32` | scalar reference dummy | writable rank-zero NumPy scalar storage | +| `Value(Float64)` | scalar `value` dummy | Python scalar value | +| `Float64[n]` | array reference dummy | writable NumPy array view | +| `point_t` | derived reference dummy | generated wrapper object | +| `Value(point_t)` | derived `value` dummy | generated wrapper over the call-local value copy | Array callback arguments require exact dtype, rank, declared shape, alignment, and required Fortran contiguity. Derived values use the generated wrapper class. -Array, scalar-storage, character-storage, and derived output/inout callback -values are copied back before the callback adapter returns. +Reference arguments are exposed permissively. Mutable scalar, array, character, +and derived storage is written back before the callback adapter returns. A +`Value(...)` argument is a native copy, so Python mutation cannot replace the +caller's original object. ## Character Callback Arguments -Read-only fixed-length character callback arguments use Python `str`: - -```python -from x2py.contracts import Callable, In, String - -callback: Callable[[In(String[8])], None] -``` - -The callback receives a Python string with exactly eight encoded bytes. Writable -character callback arguments cannot use plain `String[8]`, because Python -strings are immutable. Use mutable rank-zero fixed-width bytes storage instead: +Fixed-length character callback arguments use their ordinary semantic spelling: ```python -from x2py.contracts import Callable, InOut, Out, String +from x2py.contracts import String, prototype -callback: Callable[[InOut(String[8][()])], None] -callback: Callable[[Out(String[8][()])], None] +@prototype +def label_callback(label: String[8]) -> None: ... ``` -The callback receives a NumPy scalar bytes array, for example an object with -shape `()` and dtype `S8`. The Python callback writes through that storage: +Because reference callbacks are permissive, the callback receives mutable +rank-zero NumPy bytes storage with shape `()` and dtype `S8`. The Python +callback reads or writes through that storage: ```python def rewrite_label(label): label[...] = b"done " ``` -Generated `.pyi` contracts use `String[n][()]` automatically for Fortran -callback character dummies with `intent(out)` or `intent(inout)`. Manual -contracts reject these writable immutable forms: - -```python -from x2py.contracts import Callable, InOut, Out, String - -Callable[[Out(String[8])], None] # invalid -Callable[[InOut(String[8])], None] # invalid -``` +The semantic callback annotation remains `String[n]`; the callback adapter +chooses mutable scalar storage without encoding native direction in `.pyi`. A non-callable argument raises `TypeError` before native execution. @@ -182,31 +161,44 @@ survive such failures. - asynchronous or cross-thread callback invocation; and - persistent callback ownership during object or library teardown. -## Future Contract Policy - -Future callback contract work should enrich adapter policy, not reshape the -callback call signature. A `Callable[[...], T]` must continue to describe the -Fortran procedure interface that Fortran calls: argument order, value/reference -passing, storage shape, character length, and result type. The Python callable -can adapt argument names or order itself, so callback contracts should not grow -normal wrapper features such as argument reordering or hidden native-call -projection. - -The useful future work is explicit policy for how the adapter crosses the -Fortran-to-Python-to-Fortran boundary: - -- copy-in, copy-out, borrowed-view, and zero-copy choices; +## Adapter Policy + +The post-IR policy stage completes how the adapter crosses the +Fortran-to-Python-to-Fortran boundary before wrapper generation begins. It +records value versus reference ABI, permissive reference writeback, array shape, +fixed character length, exact derived type identity, call-scoped context +lifetime, entering-thread enforcement, GIL entry, cleanup, and the fatal error +action. The Python binding and Fortran bridge only lower those completed actions. + +A `@prototype` declaration is a compile-time semantic declaration, not a +Python runtime export. Its declaration order, value/reference transport, +storage shape, character length, and result type define the callback transport +contract. The Python callable may adapt argument names itself, so prototypes do +not use normal wrapper projection such as `@native_call`. + +Post-IR policy selects the weakest correct native declaration from the complete +prototype. Classic scalar, explicit-shape, and assumed-size signatures use an +implicit external declaration. Signatures with optional or descriptor +arguments, polymorphism, or non-scalar/descriptor results use the +named native prototype through an explicit declaration. That path imports the +real interface, including direction facts deliberately omitted from semantic +`.pyi`, so its compiler module file must be available. `Value(...)` alone does +not force the explicit path when a typed external declaration is sufficient. + +Future work may add user-selectable policy for: + +- borrowed-view, detached-copy, and zero-copy choices; - dtype conversion, overflow checks, and result coercion; - fixed-length character encoding, padding, truncation, and writeback rules; - ownership and lifetime rules for arrays, scalar storage, derived wrappers, and temporary callback values; -- writeback protocols for output or inout values that cannot be mutated +- writeback protocols for values that cannot be mutated directly by the Python object currently passed to the callback; and - callback-specific error/result policy beyond the current fatal native callback boundary. -This is planned design work, not current support. The semantic `.pyi` wrapper -roadmap later tracks the callback-adapter policy work. +These choices are not currently user-selectable; unsupported forms remain +blocked instead of selecting a different backend behavior. ## Evidence And Troubleshooting diff --git a/docs/user/guide/data-types.md b/docs/user/guide/data-types.md index cf673ad79..c53e436ec 100644 --- a/docs/user/guide/data-types.md +++ b/docs/user/guide/data-types.md @@ -90,7 +90,7 @@ The tables below summarize the currently verified Fortran-to-Python type mapping | supported logical storage | `Bool` | `bool` or `numpy.bool_` as documented by the generated contract | `numpy.bool_` | | scalar character | `String` or `String[n]` | `str` | fixed-width NumPy bytes, such as `S8` | | derived type | generated class name | instance of that generated class | arrays of derived types are unsupported | -| dummy procedure | `Callable[[...], T]` | Python callable with the exact argument/result contract | not applicable | +| dummy procedure | named `@prototype` reference | Python callable matching the named prototype | not applicable | ## Source Kind Names @@ -107,8 +107,9 @@ native storage instead of silently narrowing it. ## Scalar Values And Native Storage -A bare numeric semantic type is a Python-visible value. `T` is the -read-only value form. Its default native boundary is pass-by-value. +A bare numeric semantic type is a Python-visible value. `T` is a value +contract, not a copy of Fortran `intent`. Its default native boundary is +pass-by-value. `Addr(Arg(...))` in `@native_call` means x2py converts that Python-visible value to call-local native scalar storage and passes the storage address to native code. @@ -117,7 +118,11 @@ Use `T[()]` when the Python API itself exposes safe caller-provided scalar storage as a rank-0 NumPy array. Use `Addr(T)` only when the caller passes a raw address such as `array.ctypes.data`. For raw array addresses such as `Addr(Float64[n])`, every extent must be a fixed literal or a visible argument; -the address value itself does not carry shape. +the address value itself does not carry shape. x2py does not reject zero or +negative integer addresses or validate that raw-address extents are positive. +It reports integer-to-pointer overflow, but otherwise the caller is responsible +for supplying a live address and a valid pointee shape before native code uses +either value. Arrays and strings are storage-like at the native boundary. `Float64[n]` already passes the NumPy data address, and `String[8]` already passes the address of @@ -165,9 +170,10 @@ Array annotations combine an element dtype with rank and shape: Plain `:` records a dense axis. `::` records an axis where the contract allows runtime strides; x2py reads that exact slice spelling from the `.pyi` source -before Python AST normalization. Multidimensional dense arrays are validated -against their documented orientation, such as `ORDER_F` for Fortran-oriented -storage. +before Python AST normalization. Multidimensional dense arrays in a +Fortran-facing contract use Fortran orientation by default; generated contracts +omit `ORDER_F`. Explicit order metadata appears only when a contract +deliberately requests a non-default representation. The wrapper validates exact dtype, native byte order, rank, known extents, alignment, layout, and writeability before entering native code. It does not @@ -232,7 +238,7 @@ finalization behavior. ## Unsupported Widths And Forms The semantic format can represent names such as `Float128` and `Complex256`, -but representation in a `.pyi` file is not a runtime support claim. Current +but representation in a `.pyi` file is not a native binding support claim. Current Fortran wrapper generation blocks: - real storage wider than 64 bits; diff --git a/docs/user/guide/distribution.md b/docs/user/guide/distribution.md index 2a08227b4..4061430e7 100644 --- a/docs/user/guide/distribution.md +++ b/docs/user/guide/distribution.md @@ -36,7 +36,7 @@ python3 -m x2py src/scale.f90 --out-dir build/scale python3 python/check_scale.py ``` -The asserted result remains `numpy.float64(7.5)`, as shown with the original +The asserted result remains the Python value `7.5`, as shown with the original source in the packaging example. Record the required Python and NumPy versions, compiler family, compiler flags, diff --git a/docs/user/guide/editing-semantic-pyi-contracts.md b/docs/user/guide/editing-semantic-pyi-contracts.md index b2d841307..61157fafe 100644 --- a/docs/user/guide/editing-semantic-pyi-contracts.md +++ b/docs/user/guide/editing-semantic-pyi-contracts.md @@ -40,14 +40,13 @@ Build the edited entry contract with the same native implementation artifacts: ```bash python3 -m x2py contracts/edited_solver/__init__.pyi \ - --wrap \ --native-objects build/solver.o \ --native-include-dir build/mod \ --out-dir build/edited-solver ``` -The explicit `--wrap` is required here because the entry input is a semantic -`.pyi` contract, not a Fortran source file. +The semantic `.pyi` entry contract selects the wrapper build automatically; +the native artifact options provide the implementation to compile or link. The entry `.pyi` is the sole semantic input to wrapper generation. x2py does not reparse the native source to restore a removed declaration, projection, or @@ -169,7 +168,10 @@ Each candidate is an independent declaration. Removing one candidate narrows runtime dispatch without removing the generic name: ```python -from x2py.contracts import Float64, Int32, overload +from x2py.contracts import Float64, Int32, overload, private + +@private +def convert_integer(value: Int32) -> Int32: ... @overload("convert_integer") def convert(value: Int32) -> Int32: ... @@ -251,7 +253,13 @@ def norm2(values: Float64[:]) -> Float64: ... Link every Python overload to one concrete native specific: ```python -from x2py.contracts import Float64, Int32, overload +from x2py.contracts import Float64, Int32, overload, private + +@private +def scale_integer(value: Int32) -> Int32: ... + +@private +def scale_real(value: Float64) -> Float64: ... @overload("scale_integer") def scale(value: Int32) -> Int32: ... @@ -264,7 +272,10 @@ To rename the Python overload group while calling an existing native generic, preserve the native generic explicitly: ```python -from x2py.contracts import Int32, overload +from x2py.contracts import Int32, overload, private + +@private +def convert_integer(value: Int32) -> Int32: ... @overload("convert_integer", generic="convert") def convert_number(value: Int32) -> Int32: ... @@ -279,9 +290,13 @@ silently choose a native procedure. An edited class may bind `__init__` to one concrete native initializer: ```python -from x2py.contracts import Addr, Arg, Int32, Pass, bind, native_call +from x2py.contracts import Addr, Arg, Int32, Pass, bind, native_call, private class state: + @private + @native_call([Pass(), Addr(Arg(0))]) + def init_state(self, size: Int32) -> None: ... + @bind("init_state") @native_call([Pass(), Addr(Arg(0))]) def __init__(self, size: Int32) -> None: ... @@ -360,6 +375,14 @@ def scalar_status( Removing the explicit `status` parameter and adding the result projection are one edit. A projection must map every required native argument exactly once; incomplete, duplicate, or out-of-range mappings are contract errors. +The semantic `.pyi` intentionally has no `intent` annotation. For a generated +starter contract, source `intent` only helps select the default visible +arguments and projected results. After loading, the signature, `Returns[...]`, +and exhaustive `@native_call` list are authoritative. An output dummy may +remain caller-supplied storage, while a projected output exists only when the +contract explicitly requests that projection. The bridge may use permissive +writable local storage; the called native procedure retains and enforces its +own direction. ### Make mutation replacement-only @@ -381,6 +404,11 @@ and returns a different NumPy array. The original remains unchanged. The compiled evidence is [`test_policy_dispatch_contracts.py`](../../../tests/wrapper/fortran/edit_pyi_contracts/test_policy_dispatch_contracts.py). +Mutability is a property of this argument boundary, not of `Float64`, +`String[n]`, or another datatype in isolation. The same datatype may be an +input value, caller-owned writable storage, or a replacement-only value in +different procedures, so datatype spelling cannot replace `Immutable`. + `Immutable` plus `Transfer("borrowed_view")` on a writable value is contradictory: one requests replacement-only semantics and the other requests a writable shared view. The contract fails instead of selecting one silently. @@ -391,20 +419,24 @@ The annotation is runtime policy, not merely an IDE hint. Supported edits can tighten or broaden validation without changing the native ABI: ```python -from x2py.contracts import Annotated, Float64, ORDER_F +from x2py.contracts import Float64 def solve( - matrix: Annotated[Float64[3, 3], ORDER_F], + matrix: Float64[3, 3], rhs: Float64[3], ) -> Float64[3]: ... ``` +Plain multidimensional arrays in a Fortran semantic `.pyi` use `ORDER_F` by +default. Generated contracts omit that default order. Use explicit layout +metadata only to request a non-default Python storage representation. + The wrapper validates exact dtype, rank, shape, layout, writeability, byte order, alignment, and zero-sized-array rules required by the selected backend path. Examples of supported changes include: - `Float64[:, :]` to `Float64[3, 3]` to require one shape; -- `ORDER_F` to `ORDER_ANY` when the native path is implemented for either +- `ORDER_ANY` when the native path is implemented for either contiguous orientation; - `T | None` or a default `= ...` for a genuinely optional native argument; - `Allocatable`, `Pointer`, `Aliased`, or `PointerPolicy(...)` when those @@ -429,10 +461,10 @@ Use `@raises(...)` to turn a projected native status into a Python exception: from x2py.contracts import Float64, Int32, Returns, String, raises @raises(status="status", message="message", success=0) -def solve(values: Float64[:]) -> Returns[ - "result", Float64[:], - "status", Int32, - "message", String, +def solve(values: Float64[:]) -> tuple[ + Returns["result", Float64[:]], + Returns["status", Int32], + Returns["message", String], ]: ... ``` @@ -446,10 +478,13 @@ allows it. Add `@hold_gil` when native code must call Python synchronously or otherwise requires the current Python thread to retain the GIL: ```python -from x2py.contracts import Callable, Float64, In, hold_gil +from x2py.contracts import Float64, hold_gil, prototype + +@prototype +def scalar_callback(value: Float64) -> Float64: ... @hold_gil -def invoke_callback(callback: Callable[[In(Float64)], Float64]) -> Float64: ... +def invoke_callback(callback: scalar_callback) -> Float64: ... ``` These decorators change wrapper runtime policy; they do not change the native @@ -462,7 +497,7 @@ Ownership edits use a complete policy triple: ```python from x2py.contracts import Annotated, Destruction, Float64, Ownership, Transfer -Annotated[ +values: Annotated[ Float64[:], Ownership("native"), Transfer("borrowed_view"), @@ -504,12 +539,13 @@ module_values: Annotated[ ``` Python receives a persistent `AllocatableArray` for the module descriptor. -`handle.to_numpy()` may expose a zero-copy view because `Aliased` proves the -required addressability. NumPy must not free the data. A native +`handle.to_numpy()` exposes a current live view for both plain and `Aliased` +module allocatables. `Aliased` preserves native addressability for other policy; +it is not the extraction switch. NumPy must not free the data. A native allocate/deallocate routine controls the allocation, and a later native -deallocation or reallocation makes previous views stale. The same handle then -reports `allocated is False`; the module attribute itself does not become -`None`. +deallocation or reallocation makes previous views stale. Accessing stale views +is unsupported and may crash. The same handle then reports `allocated is +False`; the module attribute itself does not become `None`. Lifecycle: diff --git a/docs/user/guide/error-handling.md b/docs/user/guide/error-handling.md index 0e87c6433..22d40223e 100644 --- a/docs/user/guide/error-handling.md +++ b/docs/user/guide/error-handling.md @@ -44,7 +44,7 @@ Generate an editable contract package: python3 -m x2py solver.f90 --pyi --out contracts/solver ``` -In `contracts/solver/solver_api.pyi`, keep the generated native types and add +In `contracts/solver/solver.pyi`, keep the generated native types and add the explicit status policy: ```python @@ -61,9 +61,8 @@ Build that contract against the same simple native source: ```bash python3 -m x2py contracts/solver/__init__.pyi \ - --wrap \ --native-fortran-sources solver.f90 \ - --out-dir build/solver \ + --out-dir build/solver ``` The success outputs are consumed, while a nonzero status becomes diff --git a/docs/user/guide/fortran-wrapper.md b/docs/user/guide/fortran-wrapper.md index ee494ac8f..72c315d96 100644 --- a/docs/user/guide/fortran-wrapper.md +++ b/docs/user/guide/fortran-wrapper.md @@ -126,9 +126,9 @@ ordered Fortran source files -> Fortran parser project model -> compiler-dependent kind and storage probes -> semantic modules and readiness blockers - -> source-root export tree preserving native module namespaces - -> codegen AST - -> generated native bridge and Python binding + -> post-IR policy completion + -> ordered wrapper plan preserving native module namespaces and ABI slots + -> direct native-bridge and Python-binding lowering -> compile and link one Python extension module ``` @@ -139,10 +139,10 @@ ordered Fortran source files -> Fortran parser project model -> compiler-dependent kind and storage probes -> semantic modules and readiness blockers - -> source-root export tree preserving native module namespaces - -> codegen AST - -> Fortran bind(C) bridge - -> C/CPython binding and x2py runtime support + -> post-IR policy completion + -> ordered wrapper plan preserving native module namespaces and ABI slots + -> direct Fortran bind(C) bridge lowering + -> direct C/CPython binding lowering and native binding support -> compile user sources and generated sources -> link one Python extension module ``` @@ -153,11 +153,16 @@ binding validates arguments, manages wrapper-owned temporaries, calls native code, and projects results onto the documented Python API. Shared runtime support supplies array, error, allocation, and ownership helpers. +There is no separate codegen-AST conversion stage. Post-IR completion freezes +object kind, storage, ownership, mutation, output projection, and native-call +policy; wrapper planning orders those completed decisions, and the binding and +bridge generators dispatch them directly into emitted source. + @@ -165,7 +170,7 @@ Typical generated artifacts are: | Artifact | Purpose | | --- | --- | -| `x2py_runtime/` | Shared native runtime support | +| `binding_support/` | Header-only native binding support | | user and generated `.o`/`.mod` files | Native build intermediates | | `..so` | Importable extension on Linux | @@ -185,14 +190,18 @@ When a folder contains only standalone BLAS/LAPACK-style procedures, separate artifacts. -Without `--out-dir`, x2py uses a private `__x2py__` build directory beside the -source and places the importable extension beside the source file. Generated +Without `--out-dir`, x2py writes generated artifacts, including the ABI-suffixed +extension, in a private `__x2py__` build directory in the current working +directory. A direct CLI build writes its stable `.so` import alias in +the current working directory unless `--out` gives it an explicit path. Generated wrapper sources remain build artifacts; users do not edit them to change the Python API. @@ -202,11 +211,11 @@ ownership, and destruction, is explained later in Editing Semantic `.pyi` Contracts. The complete grammar appears later in the Semantic `.pyi` Format reference. The normal CLI build is source-driven: recognizable Fortran sources build -wrappers without a stage flag and cannot be combined with `--pyi`. For the -implemented `.pyi` subset, pass `--wrap` with a semantic `.pyi` file and native -build artifacts such as `.o`, `.a`, or `.so` inputs. In that mode the `.pyi` is -the Python API source of truth; native source is not reparsed during wrapper -generation. +wrappers without a stage flag and cannot be combined with `--pyi`. A semantic +`.pyi` entry contract also selects the wrapper stage automatically when its +native build artifacts, such as `.o`, `.a`, or `.so` inputs, are supplied. In +that mode the `.pyi` is the Python API source of truth; native source is not +reparsed during wrapper generation. The current `.pyi` build subset requires the contract filename stem to match the native Fortran module name. Supply the native module file directory as an @@ -214,7 +223,6 @@ include directory when the generated bridge contains `use `: ```bash python3 -m x2py path/to/module.pyi \ - --wrap \ --native-objects path/to/module.o \ --native-include-dir path/to/mod-files \ --out-dir build/module @@ -240,14 +248,13 @@ replayed directly: ```bash python3 -m x2py contracts/module.pyi \ - --wrap \ --native-fortran-sources native/module.f90 \ --native-fortran-flags="-O3 -fopenmp" \ --out-dir build/module \ --makefile -python3 -m x2py --build-manifest build/module/x2py-build.json --wrap -python3 -m x2py --build-manifest build/module/x2py-build.json --wrap --makefile +python3 -m x2py --build-manifest build/module/x2py-build.json +python3 -m x2py --build-manifest build/module/x2py-build.json --makefile ``` Edited `.pyi` contracts may expose the native call shape directly. If every @@ -278,10 +285,19 @@ Runtime tests: [`test_pyi_wrapper_builds.py`](../../../tests/wrapper/fortran/bui [`test_policy_dispatch_contracts.py`](../../../tests/wrapper/fortran/edit_pyi_contracts/test_policy_dispatch_contracts.py). Use `--verbose` to execute a build while printing every exact, shell-escaped -compiler and linker command. Verbose builds also print elapsed time for each -compiler/linker command and for the wrapper creation, printing, and compilation -stages. Use `--wrap --makefile` to generate an editable `Makefile.x2py` without -compiling. These modes are mutually exclusive. +compiler and linker command. It first announces binding, bridge, and header +source-text generation on separate lines without paths, because those files do +not exist yet. Each line is printed immediately before its separate lowering +and printing operation, followed by `Timing: ...` for that operation. It then announces each written artifact with its output path +(`Write bridge source: ...` and `Write native support: ...`), each native, bridge, and binding compilation with its +source and object path (`Compile bridge source: source -> object`), and the final +extension path before linking (`Create shared library: ...`). The exact +shell-escaped command follows each compilation or link announcement, so it can +be copied to reproduce that step. Verbose builds print elapsed time for policy +completion, each source-text generation, every compilation, +and linking, followed by total build time; writing generated files has no separate +timing. Use `--makefile` to generate an editable +`Makefile.x2py` without compiling. These modes are mutually exclusive. | Value | Who destroys it | When | | --- | --- | --- | | Python scalar or string | Python | When Python references are gone. | -| Copy-return or detached-copy NumPy array | NumPy or its generated base capsule | When Python references are gone. | +| Copy-return or explicitly detached NumPy array | NumPy or its generated base capsule | When Python references are gone. | | Caller-supplied NumPy array | The Python caller | According to normal Python lifetime. | | Wrapper-owned derived instance | Generated wrapper deallocator | When the owning wrapper is collected. | | Borrowed nested component | The parent wrapper | When parent and all borrowed children are gone. | @@ -622,7 +639,32 @@ Python immutable scalars cannot expose native in-place mutation. Scalar `intent(out)` values are hidden and returned as new Python values, while mutable semantics for strings use replacement projection as described below. -Runtime tests: [`test_verified_baseline.py`](../../../tests/wrapper/fortran/scalars/test_verified_baseline.py). +Editable semantic contracts distinguish three numeric scalar boundaries: + +- `Float64` accepts a scalar value. If a writable native reference is projected + back with `Returns["value", Float64]`, x2py copies into call-local storage and + returns the mutated replacement; the original Python scalar is unchanged. +- `Float64[()]` accepts caller-owned rank-zero NumPy storage. x2py validates its + exact dtype, native byte order, alignment, rank, and writeability, then passes + its data address so native `out` or `inout` mutation remains visible in the + same array. +- `Addr(Float64)` accepts an integer raw address and forwards it without copying + or owning the pointee. For a NumPy buffer, pass `value.ctypes.data`. + +```python +storage = np.array(3.5, dtype=np.float64) +update_storage(storage) + +raw_storage = np.array(4.5, dtype=np.float64) +update_raw(raw_storage.ctypes.data) +``` + +`Addr(Arg(i))` inside `@native_call(...)` is different from `Addr(T)`: it tells +the wrapper to take the address of its converted call-local scalar. It does not +make the Python caller pass an address. + +Runtime tests: [`test_verified_baseline.py`](../../../tests/wrapper/fortran/scalars/test_verified_baseline.py) +and [`test_scalar_boundary_plan.py`](../../../tests/wrapper/fortran/scalars/test_scalar_boundary_plan.py). ## Generic Procedure Interfaces @@ -809,6 +851,24 @@ use ordinary return annotations; hidden allocatable array outputs use `Allocatable[T[...]]` handles whose unallocated state remains inside the handle. +### Generated Docstrings + +Generated modules, functions, classes, constructors, methods, overloads, and +properties expose compact NumPy-style docstrings derived from the same completed +wrapper plan as the executable code. `help(module.function)` therefore reports +the Python-visible signature rather than the native dummy list, including +hidden outputs, ordered tuple results, optional omission versus a present +`None`, constrained array shape and layout, handle ownership, and native-status +exceptions. + +Module docstrings index their public functions, module attributes, and classes. +Class docstrings index the public constructor, fields, methods, and overloads; +the individual constructor, method, overload, and property descriptors also +carry focused docstrings. Private wrapper helper names and internal bridge roles +are never shown. Module attributes are documented in the module docstring +because Python extension modules do not provide portable per-attribute +descriptor docstrings. + Runtime tests: [`test_output_arguments.py`](../../../tests/wrapper/fortran/function_calls/test_output_arguments.py), [`test_native_call_examples.py`](../../../tests/wrapper/fortran/function_calls/test_native_call_examples.py). @@ -847,7 +907,8 @@ unallocated or unassociated state. Hidden scalar or derived-type `Return(...)` outputs are different: the wrapper requests them with native temporary storage, so they are present and returned on every call. -Runtime tests: [`test_optional_arguments.py`](../../../tests/wrapper/fortran/function_calls/test_optional_arguments.py). +Runtime tests: [`test_optional_arguments.py`](../../../tests/wrapper/fortran/function_calls/test_optional_arguments.py), +[`test_scalar_writeback_plan.py`](../../../tests/wrapper/fortran/function_calls/test_scalar_writeback_plan.py). Non-default lower bounds are preserved when computing shape constraints; they @@ -1261,8 +1349,10 @@ Private components are omitted from Python descriptors. Allocatable fields use descriptor access; that retention does not make the wrapper owner of a pointer target. Arrays of derived types are blocked. -Runtime tests: [`test_derived_type_boundaries.py`](../../../tests/wrapper/fortran/derived_types/test_derived_type_boundaries.py) -and [`test_derived_type_methods.py`](../../../tests/wrapper/fortran/derived_types/test_derived_type_methods.py). +Runtime tests: [`test_derived_type_boundaries.py`](../../../tests/wrapper/fortran/derived_types/test_derived_type_boundaries.py), +[`test_derived_type_methods.py`](../../../tests/wrapper/fortran/derived_types/test_derived_type_methods.py), +and the direct Phase 8 object and field evidence in +[`test_phase8_derived_plan.py`](../../../tests/wrapper/fortran/derived_types/test_phase8_derived_plan.py). ## Inheritance And Polymorphism @@ -1362,9 +1452,14 @@ class settings: The target method must have the same Python call shape and return type. A public target remains callable as a method; `@private` keeps the signature in the -standalone `.pyi` but exposes only construction to users. Fortran generic -constructor interfaces and overloaded runtime `tp_init` lowering are not yet -mapped; they report explicit blockers. +standalone `.pyi` but exposes only construction to users. + +An edited contract can instead declare multiple `__init__` overload links. +The wrapper allocates the ordinary Phase 8 native owner once, selects an exact +candidate from completed dtype/rank/class predicates, invokes that target, and +commits ownership only after it succeeds. Selection never calls candidates to +see which one works. Indistinguishable candidates fail during generation and a +runtime call with no match raises `TypeError` before native entry. ### Finalization @@ -1379,6 +1474,9 @@ native execution terminates the process. Runtime tests: [`test_constructors_and_finalizers.py`](../../../tests/wrapper/fortran/derived_types/test_constructors_and_finalizers.py) and [`test_borrowed_finalizers.py`](../../../tests/wrapper/fortran/derived_types/test_borrowed_finalizers.py). +Bound and overloaded constructors are covered by +[`test_phase9_bound_constructors.py`](../../../tests/wrapper/fortran/derived_types/test_phase9_bound_constructors.py) +and [`test_phase9_class_overloads.py`](../../../tests/wrapper/fortran/naming/test_phase9_class_overloads.py). ## Module Variables, Constants, Saved State, And Common Blocks @@ -1456,6 +1554,8 @@ covered in [Runtime Errors, The GIL, OpenMP, And Concurrency](#runtime-errors-th Runtime tests: [`test_module_state.py`](../../../tests/wrapper/fortran/module_state/test_module_state.py) and [`test_common_blocks.py`](../../../tests/wrapper/fortran/module_state/test_common_blocks.py). +Scalar module-variable route parity is covered by +[`test_scalar_module_variable_plan.py`](../../../tests/wrapper/fortran/module_state/test_scalar_module_variable_plan.py). ## Fortran Enums @@ -1638,10 +1738,11 @@ X2PY_C_DOCS_END --> @@ -1790,12 +1890,11 @@ For semantic `.pyi` builds, Makefile mode writes `x2py-build.json` before ```bash python3 -m x2py contracts/solver.pyi \ - --wrap \ --native-fortran-sources native/solver.f90 \ --out-dir build/solver \ --makefile -python3 -m x2py --build-manifest build/solver/x2py-build.json --wrap +python3 -m x2py --build-manifest build/solver/x2py-build.json ``` Runtime tests: [`test_multi_source_builds.py`](../../../tests/wrapper/fortran/multiple_files/test_multi_source_builds.py), @@ -1843,6 +1942,11 @@ class_(np.int32(4)) # Python name # native call uses native_class_entry ``` +Python escaping changes only the public Python surface. For example, a native +Fortran variable named `lambda` remains `lambda` in native code and is exposed +to Python as `lambda_`. Generated native symbols are checked against their own +target-language restrictions, not Python's keyword list. + ### Collisions Every normalized public name must be unique in its namespace. Module members @@ -1857,6 +1961,19 @@ class__2 class__3 ``` +Generated native symbols use the same readable duplicate convention, while a +target-language reserved word gets an explicit wrapper suffix: + +```text +value # first generated symbol +value_2 # another generated symbol named value +module_x2py # generated symbol whose original spelling is a native reserved word +module_x2py_2 # collision with the escaped spelling +``` + +The generator does not invent semantic names for collisions: the source name +and deterministic suffix make generated artifacts easy to trace and reproduce. + Generated helper names use an internal namespace, so a user procedure named `get_value` does not collide with the internal accessor for a variable named `value`. @@ -1870,9 +1987,11 @@ Runtime tests: [`test_visibility_naming.py`](../../../tests/wrapper/fortran/nami ## Immediate Python Callbacks x2py supports dummy procedures invoked during the wrapped call. It resolves -local explicit interfaces and named abstract interfaces into a complete -callable contract containing argument order, types, intents, array ranks and -shapes, derived-type references, and optional result type. +local explicit interfaces and named abstract interfaces into named +`@prototype` declarations containing argument order, types, value/reference +transport, array ranks and shapes, derived-type references, and result type. +Source `intent` remains in the native interface when that interface must be +imported, but it is not repeated in the semantic prototype. ```fortran abstract interface @@ -1898,12 +2017,13 @@ thread are supported. ### Callback Values -- scalars use the matching Python numeric conversion; +- value scalars use the matching Python numeric conversion, while reference + scalars use writable rank-zero NumPy storage; - arrays require exact dtype, rank, declared shape, alignment, and Fortran contiguity; - derived values require the generated wrapper type; -- array and derived `intent(out)` or `intent(inout)` values are copied back - before the adapter returns; and +- reference scalar, array, character, and derived storage is handled + permissively and written back before the adapter returns; and - temporary NumPy views and borrowed derived wrappers passed to the callback are valid only during that callback invocation. @@ -1975,8 +2095,6 @@ def solve( ``` ```python -from x2py.contracts import raises - solve(values) # returns None when status == 0 solve(bad_values) # raises RuntimeError(message) otherwise ``` @@ -2016,7 +2134,7 @@ uses the normal GIL-release policy. For GNU Fortran, pass OpenMP flags to both compile and link steps: ```bash -python3 -m x2py parallel_api.f90 --wrap --makefile --out-dir build +python3 -m x2py parallel_api.f90 --makefile --out-dir build make -f build/Makefile.x2py \ X2PY_FFLAGS=-fopenmp \ X2PY_LDFLAGS=-fopenmp @@ -2048,15 +2166,6 @@ This chapter groups behavior for which implementation or policy is incomplete. These items are not enabled by parser support or by editing metadata unless the backend contract described here is also implemented. -### Output Projection Metadata Is Not The Sole Codegen Source - -Semantic IR preserves explicit projection mappings, and the documented output -behaviors are implemented and runtime-tested. Wrapper generation does not yet -consume those semantic mappings as the single authoritative mechanism for every -projection path. Some output decisions are still represented by the established -lowered argument/result structures. This is an internal integration gap, not a -different user-visible tuple or mutation contract. - ### Pointer Views, Results, And Reassociation Module and derived-field pointer handles can expose borrowed NumPy views when @@ -2106,13 +2215,13 @@ wrappers: | Subject | Blocked form | Missing contract | | --- | --- | --- | -| Allocatables | Allocatable scalar derived-type replacement | Whole-object construction, replacement, finalization, and destruction. | +| Allocatables | Passing a module allocatable scalar derived object to an allocatable dummy | The object address is not the concrete allocatable descriptor; use a wrapper-owned allocatable result holder. | | Arrays | Assumed type `type(*)` | Runtime dtype and descriptor policy. | | Arrays | Character arrays not representable as fixed-width bytes dtype | Encoding, ABI, allocation, and ownership. | | Arrays | Derived-type arrays | Element layout, construction, destruction, aliasing, and copy/view behavior. | -| Pointers | Pointer-array results and unproved reassociation or ownership-changing operations | Stable result owner storage, target lifetime, or explicit operation policy. | +| Pointers | Pointer-array or scalar-derived pointer results and unproved reassociation or ownership-changing operations | Stable result owner storage, target lifetime, descriptor identity, or explicit operation policy. | | Polymorphism | Results, mutable dummies, arrays, allocatable/pointer scalars, `class(*)` | Dynamic type, allocation, replacement, and ownership. | -| Constructors | Generic constructor interfaces and overloaded runtime initialization | Deterministic Python constructor selection and lowering. | +| Constructors | Incomplete or indistinguishable constructor overload sets | Every candidate needs a complete exact runtime signature and compatible owner lifecycle. | | Characters | Mutable scalar allocatable character dummies and deferred-length mutable fields | Allocation, encoding, replacement, and destruction. | | Kinds | Real wider than 64 bits, complex wider than 128 bits, wider explicit logical storage | Portable NumPy round-trip without silent precision loss. | | Callbacks | Stored, optional, cross-thread, or procedure-pointer callbacks | Persistent ownership, thread, exception, nullability, and teardown. | diff --git a/docs/user/guide/generic-interfaces.md b/docs/user/guide/generic-interfaces.md index 9d1987168..066309971 100644 --- a/docs/user/guide/generic-interfaces.md +++ b/docs/user/guide/generic-interfaces.md @@ -20,6 +20,8 @@ Create `generic.f90`: ```fortran module conversions implicit none + private + public :: convert interface convert module procedure convert_integer module procedure convert_real diff --git a/docs/user/guide/memory-management.md b/docs/user/guide/memory-management.md index 7e8222d7d..614f28214 100644 --- a/docs/user/guide/memory-management.md +++ b/docs/user/guide/memory-management.md @@ -23,7 +23,7 @@ guess from datatype or intent. | Wrapper-owned instance | A generated Python extension object owns one native derived instance. | [`points.f90` result](wrapping-derived-types.md#complete-derived-type-example) | | Native-owned storage | Native module state or another native owner controls allocation and release. | [`allocations.f90` module handle](allocatables.md#complete-allocatable-example) | | Borrowed view or child | Python refers to storage owned by a module or containing wrapper. | [`points.f90` nested child](wrapping-derived-types.md#complete-derived-type-example) | -| Detached copy (`snapshot_copy` policy) | Python receives copied current native state, without a live view. | [`allocations.f90` handle extraction](allocatables.md#complete-allocatable-example) | +| Detached copy (`snapshot_copy` policy) | Python receives copied current native state where an explicit value-copy policy requires it. Native-array-handle `to_numpy()` and derived module-object reads do not select this behavior. | Explicit copy-result contracts | | Call-local association | Native code may refer to Python storage only during one wrapped call. | [`pointers.f90` input](pointers.md#complete-pointer-example) | Those linked pages contain the full source, build commands, and asserted @@ -41,8 +41,8 @@ attached to one canonical source listing. 7. Missing owner, lifetime, release, shape, dtype, mutability, nullability, or aliasing facts block generation. 8. Addressability is an object-origin fact: generated constructors allocate pointer-backed instances, while pre-existing derived module objects need - `Aliased` or another completed live-borrow policy before they can be - exposed. + either proved `Aliased` addressability or typed module-specific bridge + operations. A backend must not fabricate an address. 9. Native array descriptor state lives in `Allocatable[T[...]]` and `Pointer[T[...]]` handles; borrowed NumPy views and detached NumPy copies are explicit extraction results from `to_numpy()`. @@ -89,14 +89,14 @@ it does not transfer native release responsibility. - Allocatable array results use wrapper-owned handles whose finalizer releases x2py-owned descriptor storage; - Pointer-array handle results block readiness until owner storage, target - lifetime, descriptor extraction, and destroy behavior are implemented; plain - derived module variables without `Aliased` or another completed policy also - block readiness; and + lifetime, descriptor extraction, and destroy behavior are implemented; +- plain and `Aliased` derived module variables remain live native-owned objects + through module-specific or address-backed mechanisms respectively; and - Borrowed views extracted from handles **share native storage** until native invalidation. -Whole-object `Snapshot[T]` contracts are future-only and are not emitted or -accepted by the active semantic `.pyi` format. +Native-array-handle extraction remains live-view-or-`None`; callers use +`.copy()` on an extracted NumPy view for independent array storage. Return projection and ownership are one contract. An edited `.pyi` cannot ask for copy-return without a projected replacement, or combine immutable storage diff --git a/docs/user/guide/packaging.md b/docs/user/guide/packaging.md index 1172f7e7f..033a89aef 100644 --- a/docs/user/guide/packaging.md +++ b/docs/user/guide/packaging.md @@ -68,7 +68,7 @@ outside project-specific build scripts. ## Generated Artifacts An output directory can contain native object and module files, generated -wrapper sources, runtime support, build metadata, and the importable extension. +wrapper sources, header-only native binding support, build metadata, and the importable extension. These files are build products. Do not edit them as the source of the public API; change the native source or an intentional semantic `.pyi` contract. @@ -83,15 +83,14 @@ flags: ```bash python3 -m x2py src/scale.f90 \ - --wrap \ --makefile \ --out-dir build/scale make -f build/scale/Makefile.x2py X2PY_FFLAGS=-O3 X2PY_CFLAGS=-O3 ``` -Makefile mode is an explicit wrapper submode, so the command keeps `--wrap`. -Makefile mode and verbose direct compilation are separate modes. The generated +`--makefile` selects the editable wrapper-build submode directly. Makefile mode +and verbose direct compilation are separate modes. The generated Makefile expects GNU Make and a POSIX-style shell. Semantic `.pyi` Makefile builds also write `x2py-build.json`, which can regenerate or replay the build. diff --git a/docs/user/guide/pointers.md b/docs/user/guide/pointers.md index 49ea697ba..51f758343 100644 --- a/docs/user/guide/pointers.md +++ b/docs/user/guide/pointers.md @@ -57,18 +57,19 @@ not carry a native pointer descriptor. `p.to_numpy()` is the explicit extraction operation. It returns `None` when the handle is unassociated. When descriptor extraction is supported, it returns the -current target view and may expose strided targets. If descriptor extraction is -unavailable, policy must choose a contiguous-only path, an explicit copy -fallback, or a readiness diagnostic; x2py must not guess compiler-specific -descriptor layout. A generated handle without an extraction path raises a clear -unavailable-operation error instead of fabricating a view. +current target view and may expose strided targets. It never creates an +automatic detached snapshot or copy. If no supported live-view mechanism can +expose the current target, policy completion or readiness fails explicitly; +x2py does not guess compiler-specific descriptor layout or fall back to a copy. Any NumPy view returned by `p.to_numpy()` is tied to the pointer target at the time of extraction. After native code nullifies, reassociates, deallocates, or -otherwise changes that target, discard older views and call `p.to_numpy()` again -or copy the data before the target-changing operation. Each fresh extraction -starts at the target's current native lower bounds rather than assuming a -fixed Fortran lower bound. +otherwise changes that target, discard older views and call `p.to_numpy()` +again. Accessing a stale view is unsupported and may crash. Users who need +independent storage must call `.copy()` before the target-changing operation. +Each fresh extraction inspects the current descriptor and starts at the +target's current native lower bounds rather than assuming a fixed Fortran lower +bound. `p.nullify()` is the default pointer descriptor operation. `allocate(shape)`, `deallocate()`, and `resize(shape)` are exposed only when completed pointer @@ -175,14 +176,31 @@ The generated semantic contract distinguishes the module descriptor, an ordinary array parameter, and a pointer-descriptor parameter: ```python -from x2py.contracts import Aliased, Annotated, Float64, Pointer, PointerAssociation +from x2py.contracts import ( + Aliased, + Annotated, + Destruction, + Float64, + Ownership, + Pointer, + PointerAssociation, + Transfer, +) storage: Annotated[Float64[3], Aliased] values: Annotated[Pointer[Float64[:]], PointerAssociation("runtime")] def associate_values() -> None: ... def sum_array(actual: Float64[::]) -> Float64: ... -def sum_pointer(actual: Pointer[Float64[:]]) -> Float64: ... +def sum_pointer( + actual: Annotated[ + Pointer[Float64[:]], + PointerAssociation("runtime"), + Ownership("caller"), + Transfer("call_local"), + Destruction("none"), + ] +) -> Float64: ... ``` Build it: @@ -251,9 +269,9 @@ copy for `Pointer[T[...]]` results. ## Pointer Fields And Module Variables -Pointer-backed fields and module variables expose `Pointer[T[...]]` handles. -Their runtime Python class is `PointerArray`. Scalar pointers never produce a -`PointerArray`; they remain ordinary `T | None` values at the Python boundary. +Pointer-backed array fields and module variables expose `Pointer[T[...]]` +handles. Their runtime Python class is `PointerArray`. A scalar pointer to a +derived object instead returns its generated live wrapper or `None`. The containing object or module does not automatically own the pointer target. Derived-field handles keep the parent wrapper alive for descriptor access, but that retention is not target ownership. Plain `Pointer[T[...]]` has a default @@ -263,6 +281,20 @@ wrapper, and generated module operations address the native module variable. Neither path invents target ownership or enables ownership-changing operations that completed pointer policy did not allow. +An associated scalar derived module pointer can be passed to an ordinary, +target, input-only pointer, or value dummy through its current target. For a +reassociable pointer dummy, x2py uses a typed local pointer holder and restores +the final association—associated, reassociated, allocated, disassociated, or +deallocated—to the module pointer exactly once. A wrapper-owned pointer result +uses the same persistent holder component directly. The holder owns its +association variable, not an unknown target, and its destructor never +deallocates native-owned target storage. Nullification or reassociation makes an +older payload proxy stale, and later field access raises `ReferenceError`. + +The later Wrapping Derived Types guide gives the canonical “Scalar Actuals And +Native Dummies” matrix, including the `INTENT(IN)` exception for nonpointer +actuals, empty-state behavior, module transactions, and multi-argument cleanup. + ## Unsupported Forms - pointer array `intent(out)` and `intent(inout)` reassociation without a @@ -271,7 +303,9 @@ that completed pointer policy did not allow. - unknown target owners or release responsibility; - persistent associations to Python storage after return; and - stale-view invalidation after target reassociation, nullification, or - deallocation. + deallocation; and +- scalar-derived pointer targets whose lifetime or release responsibility is + neither native nor tied to a retained known owner. Semantic `.pyi` metadata can record these policy facts, but metadata does not implement a missing runtime path. @@ -281,6 +315,9 @@ implement a missing runtime path. Scalar pointer inputs, outputs, inout readback, nullable results, array pointer handles, descriptor views, normal array-actual handoff, and dtype rejection are exercised by [`test_pointers.py`](../../../tests/wrapper/fortran/derived_types/test_pointers.py). +Scalar derived module-pointer state, reassociation writeback, wrapper pointer +holders, stale proxy rejection, and multi-argument cleanup are exercised by +[`test_scalar_derived_actual_dummy_matrix.py`](../../../tests/wrapper/fortran/derived_types/test_scalar_derived_actual_dummy_matrix.py). The scalar `out` and `inout` parity cases are exercised by [`test_allocatable_views.py`](../../../tests/wrapper/fortran/module_state/test_allocatable_views.py). diff --git a/docs/user/guide/wrapping-derived-types.md b/docs/user/guide/wrapping-derived-types.md index e7706e07a..feaf4088c 100644 --- a/docs/user/guide/wrapping-derived-types.md +++ b/docs/user/guide/wrapping-derived-types.md @@ -89,23 +89,163 @@ assert container.origin.x == np.float64(12.0) - a function result is copied into a new wrapper-owned native instance before the native temporary expires. +When an edited semantic contract keeps a writable derived argument visible and +projects it with `Returns["name", T]`, the return value is the exact same Python +wrapper that the caller supplied. Native code mutates that wrapper's existing +storage; the binding does not construct a replacement object, copy the derived +value, or assume destruction responsibility. Omitting the projection leaves the +same mutation in place and returns only the other declared results (or `None`). + The complete example shows inout mutation and a wrapper-owned function result. +A native by-value argument is preserved in generated semantic contracts as +`@native_call([Value(Arg(0)), ...])`. Python still passes an existing `point` +wrapper. +The generated Fortran bridge imports the exact native type and performs the +typed by-value call. The foreign boundary never lays out or byte-copies the +aggregate, +so the same opaque mechanism applies to exact ordinary, `sequence`, and +`bind(C)` types. Polymorphic or unresolved types remain blocked. + +## Scalar Actuals And Native Dummies + +This section is the canonical compatibility reference for rank-zero, +monomorphic derived objects. It applies when a wrapper object—including a live +module attribute—is passed to another wrapped Fortran procedure. Arrays and +polymorphic objects follow different rules. + +An actual object has one of five relevant Fortran declaration forms: + +| Key | Actual declaration | +| --- | --- | +| `O` | `type(item) :: value` | +| `T` | `type(item), target :: value` | +| `A` | `type(item), allocatable :: value` | +| `AT` | `type(item), allocatable, target :: value` | +| `P` | `type(item), pointer :: value` | + +A native dummy has six forms: + +| Key | Dummy declaration | +| --- | --- | +| `O` | `type(item) :: arg` | +| `T` | `type(item), target :: arg` | +| `A` | `type(item), allocatable :: arg` | +| `AT` | `type(item), allocatable, target :: arg` | +| `P` | `type(item), pointer :: arg` | +| `V` | `type(item), value :: arg` | + +`OPTIONAL`, `INTENT`, rank, and the qualified native type are additional facts; +they are not extra declaration rows. In the table, “payload” requires an +allocated allocatable or associated pointer. “Holder” means persistent +wrapper-owned Fortran storage. “Scoped” means the originating module exposes an +address only for the duration of the synchronous native call. “Allocation +transaction” and “pointer transaction” write descriptor or association changes +back to the originating module variable before control returns to Python. + +| Actual and origin | `O` dummy | `T` dummy | `A` dummy | `AT` dummy | `P` dummy | `V` dummy | +| --- | --- | --- | --- | --- | --- | --- | +| `O`, wrapper-owned | direct reference | call-scoped target | incompatible | incompatible | input-only pointer adapter | typed value | +| `O`, module | scoped reference | scoped target | incompatible | incompatible | scoped input-only pointer adapter | scoped typed value | +| `T`, wrapper-owned | direct reference | direct target | incompatible | incompatible | input-only pointer adapter | typed value | +| `T`, module | module address | module target | incompatible | incompatible | input-only pointer adapter | typed value | +| `A`, wrapper holder | payload | payload as holder target | allocatable holder | allocatable holder | payload input-only pointer adapter | payload typed value | +| `A`, module | scoped payload | scoped payload target | allocation transaction | allocation transaction with call target | scoped payload input-only pointer adapter | scoped payload typed value | +| `AT`, wrapper holder | payload | payload as holder target | allocatable holder | allocatable holder | payload input-only pointer adapter | payload typed value | +| `AT`, module | module payload address | module payload target | target-preserving allocation transaction | target-preserving allocation transaction | payload input-only pointer adapter | payload typed value | +| `P`, wrapper holder | pointee | pointee target | incompatible | incompatible | pointer holder | pointee typed value | +| `P`, module | module pointee | module pointee target | incompatible | incompatible | pointer transaction | pointee typed value | + +“Incompatible” is a deliberate `TypeError` before native entry, not an +unimplemented Phase 8 fallback. A nonpointer actual can satisfy `P` only when +the pointer dummy is proved `INTENT(IN)`. A pointer dummy with no `INTENT`, or +with `INTENT(OUT)`/`INTENT(INOUT)`, may change association and therefore +requires a pointer actual. If the edited contract omits `INTENT` but an imported +Fortran interface is authoritative, x2py emits the target adapter and lets the +Fortran compiler enforce this rule. Without either source of authority, wrapper +generation reports an interface error. + +For payload calls, an unallocated `A`/`AT` actual or disassociated `P` actual +raises `ValueError` before native entry. Descriptor dummies `A`, `AT`, and `P` +instead accept empty state so the native procedure can allocate, deallocate, +nullify, or reassociate it. Empty state is still a present argument; only an +omitted optional argument or explicit optional `None` means absence. + +### Module Transactions And Multiple Arguments + +Module allocation and pointer state stays in Fortran. x2py uses one shared +typed allocatable holder and pointer holder per qualified native type. The +interoperable boundary carries only an opaque holder address and typed operation +pointers; no native descriptor crosses that boundary. + +For an allocatable module actual passed to `A` or `AT`, a module operation uses +`move_alloc` to place its allocation in a bridge-local typed holder. The native +procedure receives that holder component, and a cleanup operation moves the +final allocation back exactly once. For a module pointer passed to `P`, a local +pointer holder starts with the current association; cleanup restores its final +association to the module pointer exactly once. A module target needs neither +transaction: its durable native address is sufficient. + +A procedure may take any number of scalar-derived arguments. x2py validates all +slots first, acquires module origins in deterministic order, nests scoped +address producers, invokes the native procedure once, and restores transactions +in reverse order. It does not generate `2**N` call variants. Repeated read-only +use of the same object shares one acquisition; ambiguous writable aliasing is +rejected before any module state moves. + +This also applies when arguments are module variables from different Fortran +modules and have different qualified derived types. Each module variable owns a +separate bridge operation table and scoped callback; the binding validates the +table's qualified native type before the callback is invoked. There is no +shared type-specific callback slot, so one module variable cannot overwrite +another argument's transport. + +If a later acquisition or the native call reports a normal ABI error, cleanup +continues for every acquired origin and Python raises only after the Fortran +frames have returned. Concurrent or recursive use of the same active module +transaction raises `RuntimeError`. A restoration failure also raises +`RuntimeError` and poisons that module proxy rather than pretending its state is +usable. Process termination, `error stop`, signals, and invalid native pointers +cannot be converted into recoverable Python exceptions. + +A pointer holder owns its association variable, not its target. Native storage +remains native-owned unless completed policy identifies and retains a known +module, parent, or wrapper target. Destroying the Python holder nullifies its +component and releases only the holder; it never deallocates an unknown target. + ## Fields And Nested Components Public supported scalar fields become Python descriptors. Private fields are omitted. A nested scalar derived component is a borrowed child wrapper: it retains its parent owner and never destroys the component independently. +The same readable and, when policy permits, writable descriptor surface is used +for wrapper-owned instances, borrowed objects, and live module objects. A +target/addressable module object may use a direct native address; a plain module +object uses typed member getter and setter operations instead. Neither path +creates a detached whole-object copy. Allocatable fields expose `Allocatable[T[...]]` handles, and pointer-array fields expose `Pointer[T[...]]` handles. Each field handle retains the parent wrapper for descriptor access. Call `to_numpy()` to extract the current NumPy -view or detached copy selected by policy; discard old views after native -deallocation, reallocation, nullification, or reassociation. Pointer fields use -a conservative default operation policy, while ownership-changing operations -require explicit pointer policy. Arrays of derived types remain blocked because -element construction, destruction, layout, aliasing, and copy policy are -incomplete. +view or `None`; extraction never copies. Discard old views after native +deallocation, reallocation, nullification, or reassociation because accessing a +stale view is unsupported and may crash. Call `.copy()` explicitly for +independent NumPy storage. Pointer fields use a conservative default operation +policy, while ownership-changing operations require explicit pointer policy. +Arrays of derived types remain blocked because element construction, +destruction, layout, aliasing, and copy policy are incomplete. +Rank-zero allocatable or pointer components whose value is itself a derived +object use the same completed holder and ownership policy when their origin is +supported. Rank-zero allocatable and pointer module variables expose persistent +live descriptor proxies, including while the allocatable is unallocated or the +pointer is disassociated. Payload-field access then raises `ReferenceError`, +but the same proxy can still be passed to an `A`, `AT`, or `P` dummy so native +code can establish new state. Wrapper-owned allocatable and pointer results use +persistent typed holders with the same empty-state rule. +Their complete call compatibility and transaction rules are defined in +[Scalar Actuals And Native Dummies](#scalar-actuals-and-native-dummies). x2py +does not silently turn an unsupported origin into an owned object or detached +copy. ## Constructors @@ -136,6 +276,13 @@ deallocation. Native termination from a finalizer terminates the process. Supported extension types form a matching Python inheritance hierarchy. A scalar polymorphic input over a known hierarchy dispatches descendant-first. +Ordinary native `class(T)` arguments retain `Annotated[T, Polymorphic]` in the +semantic `.pyi` because that source fact selects the accepted dynamic-type +dispatch. The passed-object dummy of a type-bound procedure is different: its +class binding already proves that it is polymorphic, so generated contracts use +the plain declared type for that one argument and restore the fact when loading +the binding. + Polymorphic results, mutable polymorphic arguments, arrays, allocatable or pointer polymorphic scalars, `class(*)`, abstract instantiation, and deferred bindings are blocked. @@ -151,6 +298,10 @@ that Python-visible fields imply a stable binary layout. Scalar boundaries and nested lifetime are exercised by [`test_derived_type_boundaries.py`](../../../tests/wrapper/fortran/derived_types/test_derived_type_boundaries.py), +direct live object and field behavior by +[`test_phase8_derived_plan.py`](../../../tests/wrapper/fortran/derived_types/test_phase8_derived_plan.py), +the complete scalar actual/dummy matrix and multi-argument transactions by +[`test_scalar_derived_actual_dummy_matrix.py`](../../../tests/wrapper/fortran/derived_types/test_scalar_derived_actual_dummy_matrix.py), methods by [`test_derived_type_methods.py`](../../../tests/wrapper/fortran/derived_types/test_derived_type_methods.py), constructors/finalizers by diff --git a/docs/user/guide/wrapping-functions.md b/docs/user/guide/wrapping-functions.md index 0c5adfa58..eecac4345 100644 --- a/docs/user/guide/wrapping-functions.md +++ b/docs/user/guide/wrapping-functions.md @@ -62,12 +62,12 @@ Example (`function_results.f90`): module results implicit none contains - function squares(size) result(values) - integer(4), intent(in) :: size - real(8) :: values(size) + function squares(count) result(values) + integer(4), intent(in) :: count + real(8) :: values(count) integer(4) :: index - values = [(real(index, 8) * real(index, 8), index = 1, size)] + values = [(real(index, 8) * real(index, 8), index = 1, count)] end function squares end module results ``` @@ -79,8 +79,8 @@ from x2py.contracts import Addr, Arg, Float64, Int32, native_call @native_call([Addr(Arg(0))]) def squares( - size: Int32 -) -> Float64[size]: ... + count: Int32 +) -> Float64[count]: ... ``` Build it: diff --git a/docs/user/guide/wrapping-modules.md b/docs/user/guide/wrapping-modules.md index 6c9e175d4..3321955fd 100644 --- a/docs/user/guide/wrapping-modules.md +++ b/docs/user/guide/wrapping-modules.md @@ -80,8 +80,10 @@ the same extension observe the same native module storage. ## Module Arrays An allocatable module array is exposed as a persistent -`Allocatable[T[...]]` handle. `Aliased` permits live view extraction; without -that fact, completed policy may select a read-only detached extraction. The +`Allocatable[T[...]]` handle. Plain and `Aliased` declarations have the same +extraction behavior: a fresh `to_numpy()` call returns a live view of the +current allocation or `None`. `Aliased` preserves the native addressability +fact for other policy; it does not select view versus copy extraction. The module attribute remains a handle even when native storage is unallocated: ```python @@ -92,10 +94,11 @@ view = handle.to_numpy() view[0] = np.float64(5.0) ``` -For a borrowed extraction, mutation reaches native module storage. A later -native deallocation or reallocation invalidates old views; use `view.copy()` -first when Python needs an independent lifetime. The same handle object then -reports the new allocation state. +Mutation through either kind of view reaches native module storage. A later +native deallocation or reallocation may make old views stale; accessing stale +views is unsupported and may crash. Use `view.copy()` first when Python needs +an independent lifetime. The same handle object then reports the new allocation +state. Pointer-array module variables expose `Pointer[T[...]]` handles with a default conservative operation policy. Association inspection and `nullify()` are @@ -107,8 +110,8 @@ ownership-changing operations require explicit pointer policy. A derived-type module variable is not automatically addressable just because the same type can be constructed from Python. Python construction asks x2py to allocate a new pointer-backed native instance. A pre-existing module variable -has its own source attributes. `Aliased` on that declaration selects a live -borrowed wrapper: +has its own source attributes. Both plain and `Aliased` declarations are read +as live generated objects, but their bridge mechanisms differ: ```python from x2py.contracts import Aliased, Allocatable, Annotated, Float64 @@ -117,23 +120,20 @@ class box: values: Allocatable[Float64[:]] current: Annotated[box, Aliased] +plain_current: box ``` -Reading `module.current` returns a native-owned borrowed wrapper. The wrapper -does not copy or destroy `current`; it retains the module object's address and -allows supported component access such as `module.current.values`. That -component is an `Allocatable[T[...]]` handle retaining the wrapper; call -`to_numpy()` to obtain its current view. - -Without `Aliased` or another completed live-borrow policy, x2py blocks the -plain derived module variable before wrapper lowering. Whole-object -`Snapshot[T]` contracts are future-only; the active contract does not generate -or accept them as a detached fallback. +Reading either attribute returns a native-owned live object. The wrapper never +copies or destroys module storage. `current` may use its proved native address; +`plain_current` uses typed module-specific bridge operations instead of +fabricating an address. Supported component access such as `.values` returns an +`Allocatable[T[...]]` handle retaining the object; call `to_numpy()` to obtain +its current view. Whole object replacement through `module.current = other` is not exposed. -Mutate native module state through an `Aliased` borrowed object or a wrapped -native procedure, then call `.copy()` on an extracted view when Python needs a -detached value. +Mutate live native module state through its completed module-object policy or a +wrapped native procedure. Use `.copy()` on an extracted array view when +independent NumPy storage is required. ## Common Blocks @@ -154,10 +154,12 @@ code. - Private module declarations remain hidden. - Common-block variables have no generated attribute surface. -- Pointer state is exposed only when detached-copy policy is complete; general - borrowed pointer variables are blocked. -- Plain derived-type module variables are exposed only when the recursive - snapshot policy covers every field; `Aliased` is required for live borrowing. +- Pointer state is exposed only when association, target lifetime, and a live + descriptor-view mechanism are complete; there is no detached-copy fallback. +- Plain and `Aliased` derived-type module variables both use the live generated + object surface. Plain objects require complete typed member operations; + `Aliased` objects may use a direct native address. Unsupported members block + generation instead of changing the public representation. - Source ordering and external dependency discovery remain the caller's job. ## Evidence And Troubleshooting diff --git a/docs/user/guide/wrapping-subroutines.md b/docs/user/guide/wrapping-subroutines.md index c5ed5194b..550190733 100644 --- a/docs/user/guide/wrapping-subroutines.md +++ b/docs/user/guide/wrapping-subroutines.md @@ -174,7 +174,11 @@ contract, not the normal source-generated subroutine API. Editing Semantic reassociation without completed policy remain blocked. - Character arrays require fixed-width NumPy bytes dtype storage. Arrays of derived types are blocked. -- Allocatable scalar derived-type replacement is blocked. +- Wrapper-owned allocatable scalar derived results may be passed to compatible + dummies with same-object allocation-state writeback. Module allocatable and + pointer scalar objects use reversible typed transactions for compatible + descriptor dummies. The later Wrapping Derived Types guide gives the complete + “Scalar Actuals And Native Dummies” matrix. - Unsupported output combinations stop at readiness; code generation does not silently select another projection. diff --git a/docs/user/language-support/feature-matrix.md b/docs/user/language-support/feature-matrix.md index de7f91014..1445bb13c 100644 --- a/docs/user/language-support/feature-matrix.md +++ b/docs/user/language-support/feature-matrix.md @@ -37,19 +37,20 @@ inspection-only or partial support. | Defined operators and assignment overloads | Supported | [Defined operators](../guide/generic-interfaces.md#defined-operators) | [Bridge and binding generation](../../developer/source-map.md#common-change-routes) | [Defined operator tests](../../../tests/wrapper/fortran/naming/test_defined_operators.py) | Supported operators are those covered by the wrapper guide and runtime tests. | | Output arguments and multiple results | Supported | [Subroutine projection](../guide/wrapping-subroutines.md) | [Ownership and lowering](../../developer/source-map.md#common-change-routes) | [Output argument tests](../../../tests/wrapper/fortran/function_calls/test_output_arguments.py) | Tuple ordering and caller-provided array behavior follow the wrapper guide. | | Optional arguments | Supported | [Optional arguments](../guide/optional-arguments.md) | [Binding generation](../../developer/source-map.md#common-change-routes) | [Optional argument tests](../../../tests/wrapper/fortran/function_calls/test_optional_arguments.py) | Unsupported optional combinations must remain readiness blockers. | -| Allocatable array handles, descriptor arguments, and owned results | Supported | [Allocatables](../guide/allocatables.md) | [Ownership policy](../../developer/source-map.md#common-change-routes) | [Semantic `.pyi` parser tests](../../../tests/parsing/pyi/), [printer tests](../../../tests/codegen/printers/), [allocatable runtime tests](../../../tests/wrapper/fortran/module_state/test_allocatable_views.py) | Module and field handles borrow their owner; result handles own persistent descriptor storage. Allocatable scalar derived-type replacement remains blocked. | -| Pointer scalar projections and array handles | Partially supported | [Pointers](../guide/pointers.md) | [Ownership policy](../../developer/source-map.md#common-change-routes) | [Semantic `.pyi` parser tests](../../../tests/parsing/pyi/), [pointer tests](../../../tests/wrapper/fortran/derived_types/test_pointers.py), [scalar descriptor tests](../../../tests/wrapper/fortran/module_state/test_allocatable_views.py) | Descriptor arguments, module/field handles, strided descriptor views, and policy-gated operations are supported. Pointer-array results and reassociation without complete owner/lifetime policy remain blocked. | +| Allocatable array handles, descriptor arguments, and owned results | Supported | [Allocatables](../guide/allocatables.md) | [Ownership policy](../../developer/source-map.md#common-change-routes) | [Allocatable runtime tests](../../../tests/wrapper/fortran/module_state/test_allocatable_views.py), [scalar-derived matrix tests](../../../tests/wrapper/fortran/derived_types/test_scalar_derived_actual_dummy_matrix.py) | Array module/field handles borrow their owner; result handles own persistent descriptor storage. Wrapper-owned scalar-derived allocatables use typed holders; module scalar allocatables use reversible `move_alloc` transactions for compatible dummies. | +| Pointer scalar projections and array handles | Partially supported | [Pointers](../guide/pointers.md) | [Ownership policy](../../developer/source-map.md#common-change-routes) | [Pointer tests](../../../tests/wrapper/fortran/derived_types/test_pointers.py), [scalar descriptor tests](../../../tests/wrapper/fortran/module_state/test_allocatable_views.py), [scalar-derived matrix tests](../../../tests/wrapper/fortran/derived_types/test_scalar_derived_actual_dummy_matrix.py) | Descriptor arguments, module/field handles, strided views, wrapper-owned scalar-derived pointer holders, and module pointer reassociation transactions are supported. Unknown target ownership and unsupported pointer arrays remain blocked. | | Array-valued function results | Supported | [Array results](../guide/arrays.md#array-results) | [Array lowering](../../developer/source-map.md#common-change-routes) | [Array result tests](../../../tests/wrapper/fortran/arrays/test_array_results.py) | Ownership and dtype/shape behavior are limited to documented array result forms. | | NumPy array argument contracts | Supported | [Arrays](../guide/arrays.md) | [Bridge and binding generation](../../developer/source-map.md#common-change-routes) | [Array contract tests](../../../tests/wrapper/fortran/arrays/test_array_contracts.py), [multidimensional tests](../../../tests/wrapper/fortran/arrays/test_multidimensional_arrays.py) | Wrong dtype, rank, shape, contiguity, alignment, or mutability is rejected. | | Derived-type scalar boundaries and methods | Supported | [Derived types](../guide/wrapping-derived-types.md) | [Class lowering](../../developer/source-map.md#common-change-routes) | [Derived boundary tests](../../../tests/wrapper/fortran/derived_types/test_derived_type_boundaries.py), [method tests](../../../tests/wrapper/fortran/derived_types/test_derived_type_methods.py) | Derived-type arrays and some polymorphic forms are not included. | -| Default and keyword constructors with finalizers | Supported | [Constructors and finalizers](../guide/wrapping-derived-types.md#constructors) | [Ownership policy](../../developer/source-map.md#common-change-routes) | [Constructor/finalizer tests](../../../tests/wrapper/fortran/derived_types/test_constructors_and_finalizers.py), [borrowed finalizer tests](../../../tests/wrapper/fortran/derived_types/test_borrowed_finalizers.py) | Generic constructor interfaces and overloaded runtime initialization are blocked. | -| Module variables, constants, saved state, and common-block procedure state | Supported | [Wrapping modules](../guide/wrapping-modules.md) | [Module state route](../../developer/feature-to-code-map.md#workflow-feature-pointers) | [Module state tests](../../../tests/wrapper/fortran/module_state/test_module_state.py), [common-block tests](../../../tests/wrapper/fortran/module_state/test_common_blocks.py) | Common-block storage is not exported as Python variables. Plain derived module variables require `Aliased` or another completed live-borrow policy. | +| Default and keyword constructors with finalizers | Supported | [Constructors and finalizers](../guide/wrapping-derived-types.md#constructors) | [Ownership policy](../../developer/source-map.md#common-change-routes) | [Constructor/finalizer tests](../../../tests/wrapper/fortran/derived_types/test_constructors_and_finalizers.py), [borrowed finalizer tests](../../../tests/wrapper/fortran/derived_types/test_borrowed_finalizers.py) | Construction commits ownership only after initialization; borrowed wrappers never run an owning finalizer. | +| Generic constructor interfaces and overloaded runtime initialization | Supported | [Constructors](../guide/wrapping-derived-types.md#constructors) | [Class policy and lowering](../../developer/source-map.md#common-change-routes) | [Bound constructor tests](../../../tests/wrapper/fortran/derived_types/test_phase9_bound_constructors.py), [overload tests](../../../tests/wrapper/fortran/naming/test_phase9_class_overloads.py) | Candidates require distinguishable completed Python signatures; incomplete or ambiguous sets are blocked before emission. | +| Module variables, constants, saved state, and common-block procedure state | Supported | [Wrapping modules](../guide/wrapping-modules.md) | [Module state route](../../developer/feature-to-code-map.md#workflow-feature-pointers) | [Module state tests](../../../tests/wrapper/fortran/module_state/test_module_state.py), [scalar-derived matrix tests](../../../tests/wrapper/fortran/derived_types/test_scalar_derived_actual_dummy_matrix.py), [common-block tests](../../../tests/wrapper/fortran/module_state/test_common_blocks.py) | Common-block storage is not exported as Python variables. Rank-zero derived module objects use direct, scoped, allocation-transaction, or pointer-transaction handoff selected before lowering. | | Fortran enum constants | Supported | [Enumerations](../guide/enumerations.md) | [Semantic constants route](../../developer/source-map.md#common-change-routes) | [Enum tests](../../../tests/wrapper/fortran/scalars/test_fortran_enums.py) | No Python `Enum` or `IntEnum` classes are generated. | | Scalar character arguments, results, and fields | Supported | [Strings](../guide/data-types.md#strings) | [Character bridge route](../../developer/source-map.md#common-change-routes) | [Character argument tests](../../../tests/wrapper/fortran/strings/test_character_arguments.py), [edge-case tests](../../../tests/wrapper/fortran/strings/test_character_edge_cases.py) | Character arrays use fixed-width NumPy bytes dtype; mutable scalar deferred-length storage is blocked. | | Scalar kind coverage | Supported | [Data types](../guide/data-types.md) | [Fortran type probe](../../developer/source-map.md#hotspot-index) | [Scalar kind tests](../../../tests/wrapper/fortran/scalars/test_scalar_kinds.py) | Wider real, complex, and explicit logical storage is blocked without portable NumPy mapping. | | Caller-ordered multi-source builds, Makefiles, verbose mode, and output placement | Supported | [Packaging](../guide/packaging.md), [multi-source recipe](../examples/recipes/build-multiple-fortran-sources.md) | [Wrapper orchestration](../../developer/source-map.md#common-change-routes) | [Multi-source tests](../../../tests/wrapper/fortran/multiple_files/test_multi_source_builds.py), [compiler verbose tests](../../../tests/wrapper/fortran/build_from_source/test_compiler_verbose.py) | x2py does not discover, reorder, or resolve all external source dependencies. | | Visibility, naming, keyword escaping, and collision policy | Supported | [Visibility and naming](../guide/fortran-wrapper.md#visibility-naming-and-the-python-surface) | [Naming policy](../../developer/source-map.md#hotspot-index) | [Visibility/naming tests](../../../tests/wrapper/fortran/naming/test_visibility_naming.py) | Strict mode rejects names that default mode can normalize. | -| Immediate call-scoped Python callbacks | Supported | [Callbacks](../guide/callbacks.md) | [Callback bridge route](../../developer/source-map.md#common-change-routes) | [Scalar callback tests](../../../tests/wrapper/fortran/callbacks/test_scalar_callbacks.py), [array callback tests](../../../tests/wrapper/fortran/callbacks/test_array_callbacks.py), [derived callback tests](../../../tests/wrapper/fortran/callbacks/test_derived_callbacks.py) | Stored or asynchronous callbacks are unsupported. | +| Immediate call-scoped Python callbacks | Supported | [Callbacks](../guide/callbacks.md) | [Callback bridge route](../../developer/source-map.md#common-change-routes) | [Callback plan tests](../../../tests/wrapper_codegen/test_phase10_callbacks.py), [scalar callback tests](../../../tests/wrapper/fortran/callbacks/test_scalar_callbacks.py), [array callback tests](../../../tests/wrapper/fortran/callbacks/test_array_callbacks.py), [derived callback tests](../../../tests/wrapper/fortran/callbacks/test_derived_callbacks.py) | Direct wrapper-plan generation supports entering-thread callbacks only. Stored, optional, asynchronous, or cross-thread callbacks are unsupported. | | Runtime error projection, GIL policy, recursion, OpenMP path, and GNU ABI checks | Supported | [Error handling](../guide/error-handling.md) | [Runtime route](../../developer/source-map.md#common-change-routes) | [Runtime policy tests](../../../tests/wrapper/fortran/runtime_behavior/test_runtime_policies.py), [recursion tests](../../../tests/wrapper/fortran/runtime_behavior/test_runtime_recursion.py), [OpenMP tests](../../../tests/wrapper/fortran/runtime_behavior/test_openmp_runtime.py), [ABI tests](../../../tests/wrapper/fortran/build_from_source/test_runtime_abi.py) | OpenMP and ABI evidence is compiler/platform-specific; callers still own native synchronization. | | Fortran source wrapper builds | Supported | [Packaging](../guide/packaging.md), [CLI recipe](../examples/recipes/build-and-import-cli.md) | [Wrapper orchestration](../../developer/source-map.md#common-change-routes) | [Build modes](../../../tests/wrapper/fortran/build_from_source/test_build_modes.py), [runtime ABI](../../../tests/wrapper/fortran/build_from_source/test_runtime_abi.py) | Implemented for ordered Fortran source inputs. | @@ -82,8 +83,8 @@ X2PY_C_DOCS_END --> | Persistent callbacks and procedure pointers | Unsupported | [Callback limitations](../guide/callbacks.md#unsupported-forms) | [Callback route](../../developer/source-map.md#common-change-routes) | [Callback tests](../../../tests/wrapper/fortran/callbacks/test_scalar_callbacks.py), [readiness tests](../../../tests/semantics/readiness/) | Callbacks are valid only during the wrapped call. | | Advanced multi-source dependency discovery and external-library integration | Unsupported | [Packaging limits](../guide/packaging.md#limitations) | [Build orchestration](../../developer/source-map.md#common-change-routes) | [Multi-source tests](../../../tests/wrapper/fortran/multiple_files/test_multi_source_builds.py) | x2py does not infer dependency graphs, prebuilt module paths, or external library discovery. | | Blocked array forms | Unsupported | [Unsupported array forms](../guide/arrays.md#unsupported-forms) | [Readiness route](../../developer/source-map.md#common-change-routes) | [Readiness tests](../../../tests/semantics/readiness/), [diagnostics](../reference/diagnostic-codes.md) | Assumed type `type(*)`, arrays of derived types, and character arrays not representable as fixed-width bytes need missing runtime contracts. | -| Unsupported polymorphic forms | Unsupported | [Inheritance limits](../guide/wrapping-derived-types.md#inheritance-and-polymorphism) | [Semantic readiness](../../developer/source-map.md#common-change-routes) | [Readiness tests](../../../tests/semantics/readiness/) | Results, mutable dummies, arrays, allocatable/pointer scalars, and `class(*)` are blocked. | -| Generic constructor interfaces and overloaded runtime initialization | Unsupported | [Constructor limitations](../guide/wrapping-derived-types.md#constructors) | [Constructor route](../../developer/source-map.md#common-change-routes) | [Constructor tests](../../../tests/wrapper/fortran/derived_types/test_constructors_and_finalizers.py), [readiness tests](../../../tests/semantics/readiness/) | Deterministic Python constructor selection and lowering is not complete. | +| Unsupported polymorphic forms | Unsupported | [Inheritance limits](../guide/wrapping-derived-types.md#inheritance-and-polymorphism) | [Semantic readiness](../../developer/source-map.md#common-change-routes) | [Readiness tests](../../../tests/semantics/readiness/) | Results, mutable dummies, arrays, polymorphic allocatable/pointer scalars, and `class(*)` are blocked. | +| Ambiguous or incomplete constructor overload sets | Unsupported | [Constructor limitations](../guide/wrapping-derived-types.md#constructors) | [Constructor route](../../developer/source-map.md#common-change-routes) | [Constructor overload tests](../../../tests/wrapper/fortran/naming/test_phase9_class_overloads.py), [class-plan validation tests](../../../tests/wrapper_codegen/test_phase9_class_surfaces.py) | Candidates must have distinguishable exact runtime signatures and compatible native-owner lifecycles. | | Character arrays and mutable deferred-length character storage | Partially supported | [Strings](../guide/data-types.md#strings) | [Character bridge route](../../developer/source-map.md#common-change-routes) | [Character edge tests](../../../tests/wrapper/fortran/strings/test_character_edge_cases.py), [readiness tests](../../../tests/semantics/readiness/) | Character arrays use fixed-width NumPy bytes dtype. Fixed and allocatable deferred element length maps to dtype itemsize; Unicode/object arrays and mutable scalar deferred-length storage are unsupported. | | Wider-than-supported real, complex, and logical storage | Unsupported | [Datatype limits](../guide/data-types.md#unsupported-widths-and-forms) | [Type probing](../../developer/source-map.md#hotspot-index) | [Scalar kind tests](../../../tests/wrapper/fortran/scalars/test_scalar_kinds.py), [readiness tests](../../../tests/semantics/readiness/) | x2py blocks rather than silently losing precision or Boolean storage semantics. | diff --git a/docs/user/reference/callbacks.md b/docs/user/reference/callbacks.md index 3cc1ad720..86b6afc86 100644 --- a/docs/user/reference/callbacks.md +++ b/docs/user/reference/callbacks.md @@ -8,14 +8,13 @@ status: maintained # Callbacks Reference -Callback contracts are Fortran-facing. A normal generated function signature -describes how Python calls the wrapper; a `Callable[[...], T]` argument -describes the callback procedure signature that Fortran calls through the -generated adapter. +Callback contracts are native-facing. A normal generated function signature +describes how Python calls the wrapper; a `@prototype` declaration describes +the native callback signature invoked through the generated adapter. The callable argument list therefore preserves callback argument order, -value/reference calling, explicit or missing Fortran dummy `intent`, rank, -shape, character length, and result shape. +value/reference calling, rank, shape, character length, and result shape. It +does not repeat native callback direction. ## Immediate Callback Scope @@ -24,50 +23,61 @@ callable alive only for the active wrapped call, installs a callback context, passes a generated Fortran adapter to the native routine, and clears the context when the wrapped call returns. +Policy completion records the callback ABI, permissive reference writeback, context +lifecycle, entering-thread rule, GIL actions, and fatal action before wrapper +planning. Direct wrapper-plan generation then emits one typed Fortran adapter +and one native trampoline per callback site. Neither backend infers callback +transport, shape, or ownership from generated locals. + Native code must not store the callback or call it later. Stored procedure pointers, optional dummy procedures, asynchronous callbacks, and cross-thread callback invocation are unsupported. -## Callable Argument Forms +## Prototype Argument Forms -The callback wrappers are valid only inside `Callable[[...], T]` argument -lists. +Declare a named prototype once and use its name as the callback argument type: -| Spelling | Fortran callback dummy | Python callback object | -| --- | --- | --- | -| `Int32` | scalar `value` dummy | Python scalar | -| `In(Float64)` | scalar reference, `intent(in)` | Python scalar | -| `PassByRef(Float64)` | scalar reference, missing intent | rank-zero NumPy scalar storage | -| `Out(Float64)` | scalar reference, `intent(out)` | rank-zero NumPy scalar storage | -| `InOut(Float64)` | scalar reference, `intent(inout)` | rank-zero NumPy scalar storage | -| `Float64[n]` | array reference, missing intent | NumPy array view | -| `In(Float64[n])` | array reference, `intent(in)` | read-only NumPy array view | -| `Out(Float64[n])` | array reference, `intent(out)` | writable NumPy array view | -| `InOut(Float64[n])` | array reference, `intent(inout)` | writable NumPy array view | -| `In(point_t)` | derived reference, `intent(in)` | generated wrapper object | - -`Addr(...)` is invalid inside Fortran callback `Callable` signatures. Fortran -is the callback caller, so the contract must describe the Fortran callback -signature rather than a Python raw-address calling convention. +```python +from x2py.contracts import Float64, Int32, prototype -## Character Arguments +@prototype +def transform(count: Int32, values: Float64[count]) -> Float64[count]: ... + +def apply_transform(callback: transform, ...) -> ...: ... +``` -Read-only fixed-length character callback dummies use Python strings: +Prototype arguments use ordinary semantic types. Reference passing is the +default; `Value(T)` is the only callback argument ABI override. -```python -from x2py.contracts import Callable, In, String +| Spelling | Fortran callback dummy | Python callback object | +| --- | --- | --- | +| `Int32` | scalar reference dummy | writable rank-zero NumPy scalar storage | +| `Value(Float64)` | scalar `value` dummy | Python scalar | +| `Float64[n]` | array reference dummy | writable NumPy array view | +| `point_t` | derived reference dummy | generated wrapper object | +| `Value(point_t)` | derived `value` dummy | wrapper over the call-local value copy | + +`Addr(...)` is unnecessary inside callback signatures because reference is the +default. `Value(...)` is required only when the native callback dummy has the +Fortran `value` attribute. + +Prototypes are semantic declarations and never become Python runtime exports. +A prototype defined in another contract module is referenced through a normal +relative semantic import. That import supplies the signature identity and +complete transport contract. Post-IR policy decides whether the backend may use +an implicit external declaration or must import and use the named native +prototype. The explicit path obtains native direction from that real interface; +the semantic prototype does not repeat it. -callback: Callable[[In(String[8])], None] -``` +## Character Arguments -Writable fixed-length character callback dummies use mutable rank-zero bytes -storage: +Fixed-length character reference dummies use their ordinary type spelling: ```python -from x2py.contracts import Callable, InOut, Out, String +from x2py.contracts import String, prototype -callback: Callable[[Out(String[8][()])], None] -callback: Callable[[InOut(String[8][()])], None] +@prototype +def update_label(label: String[8]) -> None: ... ``` The Python callback receives a NumPy scalar bytes array, such as `np.ndarray` @@ -78,17 +88,15 @@ def update(label): label[...] = b"done " ``` -Generated `.pyi` contracts emit `String[n][()]` for Fortran callback -characters with `intent(out)` or `intent(inout)`. Manual contracts reject -`Out(String[n])`, `InOut(String[n])`, and `PassByRef(String[n])` because those -forms would expose an immutable Python `str` where Fortran expects writable -storage. +Generated `.pyi` contracts emit `String[n]` regardless of native callback +direction. Callback context makes reference character storage mutable without +adding direction metadata to the annotation. ## Results And Copy-Back Scalar callback results are converted from the Python return value. Array, -derived, scalar-storage, and character-storage output or inout callback -arguments are copied back before the Fortran adapter returns to native code. +derived, scalar-storage, and character-storage reference arguments are copied +back before the Fortran adapter returns to native code. Callback exceptions, invalid callback return conversion, and unsupported cross-thread callback execution are fatal at the callback boundary: x2py prints diff --git a/docs/user/reference/cli-commands.md b/docs/user/reference/cli-commands.md index 4d84f12a6..c5b8c78e6 100644 --- a/docs/user/reference/cli-commands.md +++ b/docs/user/reference/cli-commands.md @@ -148,16 +148,17 @@ commands, but new documentation and help output use `--print-limit`. ## Wrapper builds -With no explicit stage flag, Fortran source input builds a wrapper. `--wrap` -makes that build mode explicit. Semantic `.pyi` wrapper builds are available -only when native implementation inputs are supplied explicitly. +With no explicit inspection stage flag, recognizable Fortran source or +semantic `.pyi` input builds a wrapper. Manifest replay and `--makefile` also +select the wrapper stage automatically. `--wrap` remains an optional explicit +selector. | Option | Purpose | | --- | --- | | `--wrap` | Explicitly builds one Python extension module from Fortran source files or semantic `.pyi` contracts. | | `--makefile` | Generates wrapper sources and a GNU Make build without compiling. | | `--strict-wrapper-names` | Rejects Python wrapper names that require escaping or collision suffixes. | -| `--build-manifest PATH` | Replays a saved semantic `.pyi` wrapper build manifest. Use `--wrap` to build; add `--makefile` to regenerate the Makefile instead of compiling. | +| `--build-manifest PATH` | Replays a saved semantic `.pyi` wrapper build manifest. Add `--makefile` to regenerate the Makefile instead of compiling. | | `--native-fortran-sources PATH [PATH ...]` | Compiles one or more native Fortran implementation sources for a `.pyi` wrapper build without using them as semantic inputs. | | `--native-fortran-flags FLAG [FLAG ...]` | Adds one or more Fortran compiler flags to each source passed with `--native-fortran-sources`. | | `--native-objects PATH [PATH ...]` | Links one or more native object, static archive, or shared library paths into a `.pyi` wrapper build. | @@ -170,10 +171,12 @@ Important boundaries: - `--wrap` is mutually exclusive with `--parse`, `--semantics`, `--pyi`, and `--wrap-readiness`. -- `--makefile` is a wrapper-build option and requires `--wrap`. +- `--makefile` selects the editable wrapper-build mode. - For compiled wrapper builds, `--out NAME` selects the Python module name, - `PyInit_` symbol, JSON `module_name`, and final `NAME.so` path. Use - `--out-dir DIR` to choose the build directory. + `PyInit_` symbol, JSON `module_name`, and stable `NAME.so` alias in the + current directory. Use `--out-dir DIR` to choose where generated artifacts + and the ABI-suffixed extension are built. Give `--out` an explicit path to + place the stable alias elsewhere. - Wrapper `--out` requires a value and accepts `NAME` or `NAME.so`. - `--makefile` cannot be combined with `--out` because no shared library is compiled in that mode. @@ -187,8 +190,8 @@ Important boundaries: `--native-library="-lblas -llapack"`. - In `.pyi` Makefile mode, x2py writes `/x2py-build.json` first and generates `/Makefile.x2py` from that manifest. -- `--build-manifest PATH --wrap` builds from a saved manifest. - `--build-manifest PATH --wrap --makefile` regenerates `Makefile.x2py` from +- `--build-manifest PATH` builds from a saved manifest. + `--build-manifest PATH --makefile` regenerates `Makefile.x2py` from the manifest without positional contracts or repeated native flags. | `--json` | Prints JSON to stdout for inspection stages and wrapper build results. | | `--out [PATH]` | Writes inspection-stage output, selects the generated Fortran `.pyi` package directory, or names the wrapper Python module and final `.so`. | | `--out-dir DIR` | Selects the wrapper build output directory. | -| `--verbose` | Prints wrapper compiler commands, build steps, and elapsed time for each compiler/linker command and wrapper stage. | +| `--verbose` | Announces and completes binding, bridge, and header source-text generation in order, then each written artifact, source/object compilation pair, and final extension path before printing the exact compiler or linker command; it times each non-writing operation and reports total build time last. | | `--wrapper-compiler-debug` | Uses the compiler debug profile for direct wrapper builds instead of the default release profile. | | `--wrapper-fortran-flags FLAG...` | Appends flags to generated Fortran bridge compilation commands. | | `--no-color` | Disables ANSI color in parse diagnostics. | @@ -232,9 +235,9 @@ and for semantic `.pyi` builds the normalized replay `manifest`. | Check edited `.pyi` readiness | `python3 -m x2py path/to/module.pyi --wrap-readiness --json` | | Build a Fortran wrapper | `python3 -m x2py path/to/file.f` | | Build a Fortran wrapper with an explicit module and `.so` name | `python3 -m x2py path/to/file.f90 --out my_extension` | -| Generate an editable Makefile | `python3 -m x2py dependency.f90 api.f90 --wrap --makefile --out-dir build` | -| Generate a `.pyi` replay manifest and Makefile | `python3 -m x2py contracts/module.pyi --wrap --native-fortran-sources native/module.f90 --out-dir build --makefile --json` | -| Replay a `.pyi` manifest | `python3 -m x2py --build-manifest build/x2py-build.json --wrap` | +| Generate an editable Makefile | `python3 -m x2py dependency.f90 api.f90 --makefile --out-dir build` | +| Generate a `.pyi` replay manifest and Makefile | `python3 -m x2py contracts/module.pyi --native-fortran-sources native/module.f90 --out-dir build --makefile --json` | +| Replay a `.pyi` manifest | `python3 -m x2py --build-manifest build/x2py-build.json` | - unresolved typedef or unknown type references; - legacy parser reports carrying macro-dependent declarations; - variadic functions; -- function pointer/callback signatures without edited `.pyi` `Callable` +- function pointer/callback signatures without a resolved named prototype policy; - mutable numeric or `void *` pointer parameters without ownership, scalar-storage, raw-address, or array policy; @@ -439,11 +439,12 @@ storage, NumPy array storage, Python strings, raw address values, and generated wrapper instances. The native barrier distinguishes direct values, call-local addresses, caller/Python-backed storage addresses, raw addresses, packed array descriptors, and wrapper-owned native addresses. These decisions are semantic -policy. `ir2ast.py`, bindings, bridges, and printers may create backend-local -temporaries, but they must not infer or override a barrier action from datatype, -source-declaration direction, array category, aliasing, or memory-storage checks. +policy. Wrapper planning, binding/bridge lowering, and printers may create +backend-local temporaries, but they must not infer or override a barrier action +from datatype, source-declaration direction, array category, aliasing, or +memory-storage checks. -Parser-model conversion and codegen model traversal use the shared +Parser-model conversion and semantic/wrapper-model traversal use the shared `x2py.utilities.visitor.ClassVisitor` dispatcher and one configured `_` protocol. The default prefix is `_visit`; specialized visitors may choose clearer names such as `_print` or `_parse` while still using @@ -454,7 +455,7 @@ nodes. These tables are separate from model-node dispatch. ### Round Trips And Provenance -`x2py.pyi_parser` parses the documented semantic `.pyi` subset into Python AST. +`x2py.parsers.pyi` parses the documented semantic `.pyi` subset into Python AST. `convert_pyi_to_ir` converts that AST into the same public storage contracts emitted by the source semantic pipelines; `pyi_file_to_semantic_module` combines file parsing and conversion. Focused round-trip tests cover: @@ -517,7 +518,7 @@ X2PY_C_DOCS_END --> X2PY_C_DOCS_END --> -The normal `--wrap` workflow remains source-driven and accepts Fortran source -files. A `.pyi`-driven wrapper workflow is also available for the implemented -subset: pass the semantic `.pyi` file as the wrapper input and provide native -object, archive, shared-library, module, include, and link inputs with the -native artifact flags. This path treats the `.pyi` as the source of truth for -the Python API and does not reparse native source to reconstruct the contract. +The normal wrapper workflow accepts recognizable Fortran source without a +stage flag. A `.pyi`-driven wrapper workflow is also available for the +implemented subset: pass the semantic `.pyi` file as the wrapper input and +provide native object, archive, shared-library, module, include, and link +inputs with the native artifact flags. The `.pyi` input selects the wrapper +stage automatically and remains the source of truth for the Python API; native +source is not reparsed to reconstruct the contract. The implemented subset and remaining parity limits are stated in this reference and summarized later in Language Support. Status terms used below: -- **Generated**: emitted today by `--pyi` or `codegen.printers.pyi_printer`. -- **Loaded**: accepted today by `x2py.pyi_parser` and converted back to +- **Generated**: emitted today by `--pyi` or + `wrapper_codegen.printers.pyi_printer`. +- **Loaded**: accepted today by `x2py.parsers.pyi` and converted back to semantic IR. - **Readiness**: understood by the semantic readiness checker. - **Build input**: accepted by the `.pyi` wrapper build for the implemented subset when the required native artifacts are supplied. - **Roadmap**: design direction, not implemented wrapper behavior. +Parser-related pull requests that change `x2py/parsers/pyi/` or its focused +loading tests must update this reference. The parser-reference guard checks +that contract independently from the language parser references. + ## Contract Imports Every semantic `.pyi` control name is imported from `x2py.contracts`. This @@ -155,7 +161,7 @@ or an explicit call-local copy whose native mutation is discarded. A replacement requires a projected return such as `Returns["values", Float64[:]]`; the bridge and binding then emit the already-selected action without reconsidering the datatype, mutability, ownership, or storage mode. Unsupported combinations block -before `ir2ast.py`. +before wrapper planning and direct lowering. `@native_call(...)` and `Returns[...]` describe projection and native placement; they do not ask the backend to rediscover conversion policy. After `.pyi` @@ -341,11 +347,12 @@ from x2py.contracts import Float64, Int32, external def dgesv(a: Float64[:, :], b: Float64[:, :]) -> Int32: ... ``` -`@external` is immutable native-placement metadata. The bridge must generate a -matching explicit Fortran interface and call the external procedure without a -`use ` statement. The procedure therefore needs no Fortran `.mod` file, -but its defining object, archive, or shared library must be supplied to the -link. +`@external` is immutable native-placement metadata. The bridge calls the +external procedure without a `use ` statement. Classic +implicit-interface-compatible procedures use a compact `external` declaration; +features that require an explicit interface retain one. The procedure needs no +Fortran `.mod` file, but its defining object, archive, or shared library must be +supplied to the link. Python-visible renaming is separate from placement. `@bind` retains the native Fortran procedure name while the declaration uses a wrapper name: @@ -398,7 +405,6 @@ leaves: ```bash python3 -m x2py contracts/basic_subroutine/__init__.pyi \ - --wrap \ --native-objects basic_subroutine.o ``` @@ -480,9 +486,14 @@ def DAXPY( ``` `Float64[3, Flat]` maps to `real :: a(3, *)`, and -`Float64[3, 4, Flat]` maps to `real :: a(3, 4, *)`. The Python-visible flat -dimension remains unconstrained, but the explicit Fortran interface generated -from the `.pyi` uses `DX(*)`/`DY(*)` instead of assumed-shape descriptors. +`Float64[3, 4, Flat]` maps to `real :: a(3, 4, *)`. `Flat` is an axis marker, +not a request to collapse the whole array to rank one: `Float64[:, Flat]` +remains a rank-two Python and bridge contract. Because `real :: a(:, *)` is not +a legal Fortran assumed-size declaration, an external interface whose prefix +extent is known only at runtime uses the sequence-associated `a(*)` spelling +when that procedure requires an explicit interface; +the bridge view still has rank two and receives both runtime extents. The +Python-visible flat dimension remains unconstrained. | Complex | `Complex64`, `Complex128`, `Complex256` | | Text | `String` | | User types | class names and imported type names | -| Callables | `Callable`, `Callable[..., T]`, `Callable[[A, B], T]` | -| Callback argument interface wrappers | `PassByRef(T)`, `In(T)`, `Out(T)`, `InOut(T)` inside `Callable[[...], T]` only | +| Named callable prototypes | `@prototype` function declarations referenced by name | +| Prototype value override | `Value(T)` inside a `@prototype` declaration only | The Python argument may provide more storage than the declared explicit @@ -1041,14 +1081,25 @@ Use local constants or generated `Final[...]` names for shape symbols. ## Metadata With `Annotated` -`Annotated[...]` carries storage metadata and semantic constraints, not -source-language argument direction: +`Annotated[...]` carries storage metadata and semantic constraints. It does +not carry source-language argument direction or per-call value/reference +selection. The Fortran semantic pipeline supplies `ORDER_F` as the default +multidimensional layout. Generated contracts omit that default. Write explicit +layout metadata only when the Python-visible storage deliberately differs from +that Fortran representation, such as a row-major input accepted by a Fortran +wrapper. +Native call transport belongs to `@native_call`: a wrapped derived +object uses its normal reference handoff with `Arg(i)` and exact typed value +handoff with `Value(Arg(i))`. The Python API accepts the same opaque wrapper +object in both cases; the generated Fortran bridge performs the typed call, and +the binding never exposes or guesses aggregate layout. ```python -from x2py.contracts import Annotated, Float64, ORDER_F +from x2py.contracts import Annotated, COPY_F, Float64, ORDER_C def fill( - a: Annotated[Float64[:, :], ORDER_F], + a: Float64[:, :], + c_input: Annotated[Float64[:, :], ORDER_C, COPY_F], out: Float64[()], ) -> None: ... ``` @@ -1057,11 +1108,12 @@ Generated canonical metadata: | Metadata | Meaning | | --- | --- | -| `ORDER_F` | multidimensional Fortran-oriented storage | +| `COPY_F` | accept the declared C-contiguous Python layout, create an F-contiguous temporary with the same logical axes, and copy back after visible native mutation | | `PointerAssociation("runtime")` | pointer association is a runtime state rather than a declaration-time constant | -| `Name("native-name")` | source name cannot be represented directly as the Python target name | +| `SourceName("native-name")` | source name cannot be represented directly as the Python target name | | `Aliased` | native storage may be exposed across the Python boundary as an alias | -| `Immutable` | Python-visible value must not be mutated in place; writable native calls require a completed copy-in/copy-out replacement policy or an explicit call-local discarded-mutation policy | +| `Immutable` | Python-visible value must not be mutated in place; this is a use-site boundary policy rather than an intrinsic datatype property, and writable native calls require a completed copy-in/copy-out replacement policy or an explicit call-local discarded-mutation policy | +| `Polymorphic` | an ordinary derived argument is a native polymorphic `class(T)` dummy; the passed-object dummy of a type-bound procedure omits this metadata because the binding already proves it | | `Ownership("python" | "native" | "wrapper" | "caller" | "temporary" | "unknown")` | explicit owner override for the wrapper ownership policy | | `Transfer("copy_return" | "snapshot_copy" | "borrowed_view" | "call_local" | "in_place" | "by_value" | "wrapper_instance" | "blocked")` | explicit boundary transfer override for the wrapper ownership policy | | `Destruction("python_refcount" | "wrapper_dealloc" | "native_owner" | "caller" | "call_local" | "none" | "blocked")` | explicit destruction override for the wrapper ownership policy | @@ -1077,14 +1129,38 @@ Loaded compatibility metadata: | --- | --- | | `Contiguous` | source provenance says the array is contiguous | | `ArrayCategory("...")` | source array category provenance | -| `SourceDims(...)` | source declaration dimensions | -| `LowerBounds(...)`, `UpperBounds(...)` | source bound provenance | | `FortranAllocatable` | older scalar character allocatable metadata; generated contracts use `Allocatable[String]` | +Without `COPY_F`, `ORDER_C` is zero-copy and native Fortran observes the +reversed-axis storage view. With `COPY_F`, the binding performs both copy-in and +any required copy-out; the bridge receives an ordinary F-order buffer and does +not know that a representation conversion occurred. `COPY_F` is initially +limited to required, concrete-rank, dense numeric ndarray arguments. It does +not apply to `Flat`, assumed-rank or strided arrays, optional arrays, character +arrays, native descriptor arguments, or handle actuals. + +Semantic `.pyi` types never repeat the native procedure's `intent`. Native +source conversion may use the source declaration once to propose default +Python argument/result positions. The emitted Python signature, `Returns[...]` +items, and ordered `@native_call` mapping are the editable, authoritative +contract; users may retain the native positions or choose a different Python +projection. Wrapper policy is completed from that contract and does not retain +or re-infer native `intent`. + +Bridge entry dummies use the permissive omitted-`intent` default. The binding +may therefore use mutable call-local storage and perform completed copy-back +even when the native procedure has a more restrictive dummy. The compiled +native procedure's own explicit interface remains authoritative when the +bridge calls it. When an argument is projected with `Returns`, Python receives +the original C-order object after copy-back. + Persistent native descriptors use wrapper type syntax instead of descriptor metadata inside `Annotated[...]`: @@ -1113,6 +1189,12 @@ optional callable arguments, where `None` or omission maps to native use the handle type without `| None`; unallocated or unassociated state lives inside the present handle. +For this optional-descriptor ABI, the binding always supplies the bridge with a +valid standard descriptor. For an omitted Python value it establishes a local +unallocated or unassociated placeholder descriptor, while a separate completed +presence action selects a native call that omits the native dummy. The bridge +never forwards or inspects placeholder storage as a present native argument. + Procedure boundaries keep the Python value type in the annotation and put the native descriptor conversion in `@native_call`. Both scalar descriptor kinds are nullable: Python passes `T | None`, where `None` creates a present but @@ -1186,8 +1268,9 @@ maybe_value: Float64 | None `Float64 | None` does not imply a native allocatable or pointer descriptor. For array descriptors, use `Allocatable[T[...]]` and `Pointer[T[...]]`. -`Annotated[T[...], Allocatable]`, `Annotated[T[...], Pointer]`, and -`Snapshot[T]` are not active public descriptor spellings. +`Annotated[T[...], Allocatable]` and `Annotated[T[...], Pointer]` are not +active public descriptor spellings. Derived module objects remain live objects; +there is no public whole-object snapshot annotation. Other positional `Annotated` helpers are preserved as semantic constraints: @@ -1271,15 +1354,16 @@ reassociation and deallocation, using `resize`, `allocate_resize`, or `deallocate_resize` as appropriate. Other values keep those operations absent from the completed handle policy. -Pointer-array extraction policy is selected from the completed pointer policy -before wrapper lowering. A policy with `transfer="snapshot_copy"`, -`aliasing="independent_copy"`, or `mutability="copy"` selects `copy_only` only -when `contiguity="contiguous"` gives the backend a contiguous target path. -Other contiguous pointer policies select `contiguous_view`. Strided or otherwise -general pointer views select `descriptor_view`, which requires standard -descriptor interop support. When generated descriptor interop supplies decoded -descriptor fields as mappings or field-record objects, the shared runtime can -build NumPy views for positive or negative descriptor stride multipliers. +Pointer-array extraction policy is selected from completed descriptor and +layout facts before wrapper lowering. A contiguous target selects +`contiguous_view`. A strided or otherwise general target selects +`descriptor_view`, which requires standard descriptor interop support. Copy- +oriented `PointerPolicy(...)` values may retain unrelated call/ownership +meaning, but they do not request a copied `to_numpy()` result. If those facts +cannot support a live view, extraction is unsupported rather than copied. When +generated descriptor interop supplies decoded descriptor fields as mappings or +field-record objects, the shared runtime can build live NumPy views for positive +or negative descriptor stride multipliers. ```python from x2py.contracts import Annotated, Float64, Pointer, PointerPolicy @@ -1288,29 +1372,27 @@ value: Annotated[ Pointer[Float64[:]], PointerPolicy( nullable=True, - transfer="snapshot_copy", + transfer="call_local", target_owner="module", lifetime="module", deallocation="never", shape_source="pointer_bounds", contiguity="contiguous", - reassociation="snapshot_final", - aliasing="independent_copy", - mutability="copy", + reassociation="never", + aliasing="borrowed", + mutability="view", ), ] ``` For module and derived-field pointer-array handles, a completed contiguous -policy enables `contiguous_view` or `copy_only` extraction and checks the -target's current contiguity before reading it. General strided extraction still -requires the descriptor-view path. Pointer-array function results remain +policy enables `contiguous_view` extraction and checks the target's current +contiguity before reading it. General strided extraction still requires the +descriptor-view path. Pointer-array function results remain blocked until returned-handle owner storage, target lifetime, descriptor extraction, and destroy behavior are implemented. -Whole-object `Snapshot[T]` contracts are future-only. They are not accepted in -active semantic `.pyi` files and are not generated for derived module objects. -Use an explicit live-borrow policy such as `Annotated[T, Aliased]` when the -native object is addressable: +Derived module objects use the normal generated class in both plain and +`Aliased` declarations: ```python from x2py.contracts import Aliased, Allocatable, Annotated, Float64 @@ -1318,12 +1400,15 @@ from x2py.contracts import Aliased, Allocatable, Annotated, Float64 class box: values: Allocatable[Float64[:]] -current: Annotated[box, Aliased] +live_current: Annotated[box, Aliased] +plain_current: box ``` -Without a completed live-borrow policy, a plain derived module variable is a -readiness blocker. The backend must not silently fall back to a detached object -copy. +Both reads return live native-owned objects. `Aliased` remains a +language-neutral addressability fact: it permits an address-backed borrow, but +does not make a native-array handle copy. A plain derived module declaration +uses typed module-specific bridge operations and must not be lowered by +fabricating a native address. `Final[T]` is the only public constant spelling. Do not use `Annotated[T, Constant]` or `T[Constant]`. @@ -1487,6 +1572,15 @@ When the name matches an existing Python-visible argument, the argument remains an input and the return item represents replacement-style writable-reference behavior for immutable public values such as Python `str`. +With an explicit `@native_call`, a matching `Returns["name", T]` item +automatically assigns that visible `Arg(i)` its Python result position; it does +not require a duplicate `Return(...)` entry. The first ordinary return item is +the native function result. A native output dummy with no visible Python +argument must instead appear explicitly as `Return("name", position)` in the +native argument list. The list itself is exhaustive: once `@native_call` is +present, every native dummy position must have exactly one entry in native +order; native arguments are never inferred from leftovers. + For bare numeric scalar values, `Addr(Arg(i))` means x2py first converts the Python argument to its native scalar representation and then passes the address of that native slot. It does not mean the user passed a reference. @@ -1527,6 +1621,16 @@ default `Arg(i)` representation is already the native storage, handle, or raw address representation. Address projections of `Return(...)` and `Work(...)` are also rejected; native outputs and workspaces already name their storage. +`Value(Arg(i))` is the inverse override for an exact rank-zero monomorphic +wrapped derived object. Plain `Arg(i)` passes that object by reference; +`Value(Arg(i))` asks the typed bridge to pass the exact native object by value. +Primitive scalars already use value passing with plain `Arg(i)`, while arrays, +strings, raw addresses, and descriptor handles keep their normal storage ABI and +do not accept `Value(...)`. The foreign binding boundary still carries an opaque object +address: `Value(...)` records the native Fortran dummy contract, and the Fortran +compiler performs any required copy when the typed bridge makes the call. It +never asks the binding to pass aggregate bytes through the foreign ABI. + ```python from x2py.contracts import Returns, String @@ -1558,6 +1662,13 @@ to the native procedure by address. The optional `result=` keyword records a native scalar descriptor function result, for example `result=Allocatable(Return(0))` or `result=Pointer(Return(0))`. +For descriptor projections, `Pointer(Arg(i))` without a matching +`Returns["name", T]` creates a permissive call-local pointer adapter and +discards any native reassociation after the call. Adding the matching projected +return requests association writeback instead; scalar-derived values then +require persistent pointer storage. This choice belongs to the Python contract, +not native `intent`. + The same native routine can be edited into an identity call without projection: ```python @@ -1666,18 +1777,19 @@ by the selected concrete wrapper, but they do not distinguish overloads; overloads that differ only in those properties are rejected during generation. X2PY_C_DOCS_END --> -All specifics must have one compatible Python call shape. Parameter names and -keyword parsing use the first specific procedure's signature. A call that -matches no specific raises `TypeError`; duplicate dtype/rank signatures are a -deterministic generation error. +Each specific keeps its declared Python call shape. The generated dispatcher +normalizes positional and keyword arguments against each candidate, then uses +the candidate's exact typed predicates. A call that matches no specific raises +`TypeError`; duplicate runtime dtype/rank/class signatures are a deterministic +generation error. ## Defined Operators And Assignment @@ -1782,13 +1894,12 @@ exposed as `Allocatable[T[...]]` handles. The handle carries allocation state and descriptor operations. It is not a NumPy array, and unallocated state lives inside the handle. -`h.to_numpy()` returns `None` when the descriptor is unallocated. When completed -policy proves live aliasing is safe, it returns a borrowed NumPy view over -native storage. For derived-type fields, the extracted view retains the field +`h.to_numpy()` returns `None` when the descriptor is unallocated. Otherwise it +returns a live NumPy view over the current native storage and never an automatic +detached copy. For derived-type fields, the extracted view retains the field handle and the field handle retains the containing Python wrapper. For module -variables, the handle retains the generated module owner while the Fortran -module controls allocation. When live aliasing is unsafe but copying is -implemented, `to_numpy()` returns a read-only detached copy instead. +variables, the handle retains the generated module owner while the native +module controls allocation. Existing views are not invalidated, detached, or tracked. If a wrapped Fortran procedure reallocates or deallocates native storage while Python still holds an @@ -1796,7 +1907,7 @@ old view, that old view is stale; reading or writing it is unsupported and may crash the process. Users who need independent lifetime must copy explicitly: ```python -x = obj.values.to_numpy() # borrowed view, detached copy, or None +x = obj.values.to_numpy() # live view or None y = None if x is None else x.copy() obj.reset_values() # may invalidate x; y remains valid ``` @@ -1885,9 +1996,11 @@ class state: The generated keyword-only shape remains reserved: if undecorated `__init__` keeps the `self, *, ...` form and every keyword has a default, the loader treats it as the generated field constructor metadata. Constructor overload -declarations may still be used only when the generated field constructor is -present; overloaded `tp_init` runtime lowering is not implemented yet and code -generation reports an explicit blocker for that form. +declarations replace runtime field initialization with an exact constructor +overload set linked to concrete same-class targets. The direct wrapper allocates +one native owner, dispatches without candidate trial calls, and releases an +uncommitted owner on failure. Missing, incompatible, or indistinguishable +candidates are rejected before source emission. Module variables are declarations in the semantic contract. Allocatable array module variables expose handles; unallocated state is represented by the handle, @@ -1897,18 +2010,18 @@ not by making the module attribute `None`: from x2py.contracts import Aliased, Allocatable, Annotated, Float64 module_values: Annotated[Allocatable[Float64[:]], Aliased] -snapshot_values: Allocatable[Float64[:]] +plain_values: Allocatable[Float64[:]] ``` -`Aliased` says a native-owned borrowed view may be exposed through -`to_numpy()`. A plain allocatable module array remains wrappable as a handle; -if live aliasing is unsafe, `to_numpy()` uses a read-only detached copy when -that extraction policy is implemented. Fortran source declarations with -`target` are printed as `Aliased` because they prove that the current allocation -may be aliased by the wrapper. +Both declarations expose a stable native-owned handle whose `to_numpy()` call +returns a current live view or `None`. `Aliased` does not select view versus +copy extraction. It remains a language-neutral fact that native storage may be +externally aliased or addressed. Fortran source declarations with `target` are +printed as `Aliased` because they supply that native fact. -`Aliased` also controls borrowed access to an existing derived-type module -object: +`Aliased` also records addressability used by borrowed access to an existing +derived-type module object. Plain derived module objects remain live but use a +different bridge mechanism: ```python from x2py.contracts import Aliased, Allocatable, Annotated, Float64 @@ -1917,15 +2030,17 @@ class box: values: Allocatable[Float64[:]] live_current: Annotated[box, Aliased] +plain_current: box ``` The annotation belongs to the module variable, not to `box`. An x2py-created `box()` is addressable because its generated constructor allocates pointer-backed native storage. A native module declaration is a different object origin. `Annotated[box, Aliased]` lets the wrapper retain that object's native -address and return a live borrowed `box` wrapper. Without `Aliased` or another -completed policy, the derived module object blocks readiness because -whole-object snapshots are not part of the active contract. +address and return a live borrowed `box` wrapper. A plain `box` module variable +returns the same public wrapper type, backed by typed module-specific bridge +operations. Unsupported live lifetime or module-access policy blocks readiness; +the backend must not fall back to a detached object or an invented address. Public scalar Fortran module variables are emitted directly with their resolved semantic type: @@ -1951,10 +2066,10 @@ Wrapper generation may synthesize native getter and setter bridge functions to implement Python attribute reads and writes. Those functions are internal: they are absent from the `.pyi` and are not exported as Python-callable procedures. The post-IR policy stage separately decides the getter result policy, native -setter assignment mode, and Python setter exposure before `ir2ast.py`. A native -value-copy setter can therefore exist for ABI use while Python replacement is -explicitly rejected, as for allocatable or derived fields. Bridge and binding -generation only dispatch those completed accessor decisions. +setter assignment mode, and Python setter exposure before wrapper planning. A +native value-copy setter can therefore exist for ABI use while Python +replacement is explicitly rejected, as for allocatable or derived fields. +Bridge and binding generation only dispatch those completed accessor decisions. A mutable scalar module variable may include a literal default in an edited `.pyi` contract: @@ -2049,9 +2164,10 @@ The handle carries association state and descriptor operations: When descriptor-backed extraction is enabled, `to_numpy()` builds NumPy shape and strides from descriptor metadata and can expose strided pointer targets. If that -path is unavailable, the completed policy must choose contiguous-only views, -an explicit copy fallback, or a readiness diagnostic. Pointer handle ownership -is descriptor or association access by default, not target ownership. +path is unavailable, the completed policy must choose a contiguous live view or +an explicit readiness diagnostic. It must not fall back to a copy. Pointer +handle ownership is descriptor or association access by default, not target +ownership. The generated descriptor-view path establishes portable descriptor storage, associates an `intent(out)` pointer dummy with the live target, and decodes the descriptor synchronously. It does not inspect a compiler-private Fortran @@ -2083,14 +2199,14 @@ is a user contract applied to a declaration that was otherwise available to the wrapper, so the declaration remains printed and loadable as wrapper input. Names that are not valid Python identifiers are represented with `var[...]` for -data declarations, or with `Annotated[..., Name("native-name")]` for callable +data declarations, or with `Annotated[..., SourceName("native-name")]` for callable arguments: ```python -from x2py.contracts import Annotated, Int32, Name +from x2py.contracts import Annotated, Int32, SourceName var["class"]: Int32 -def f(class_: Annotated[Int32, Name("class")]) -> None: ... +def f(class_: Annotated[Int32, SourceName("class")]) -> None: ... ``` ## Projection Metadata @@ -2122,6 +2238,7 @@ Loaded projection entries: | --- | --- | | `Arg(i)` | native argument is Python argument `i`'s default native representation | | `Addr(Arg(i))` | native argument is the address of Python argument `i`'s call-local native scalar representation | +| `Value(Arg(i))` | exact rank-zero monomorphic wrapped derived object is passed to the native value dummy by the typed bridge | | `Allocatable(Arg(i))`, `Pointer(Arg(i))` | native argument is a nullable call-local scalar descriptor initialized from Python argument `i`; `None` means present but unallocated or unassociated | | `Return(i)` | native argument is supplied by projected return slot `i` as hidden writable storage passed by address | | `Return("name", i)` | named native argument is supplied by projected return slot `i` as hidden writable storage passed by address | @@ -2159,7 +2276,7 @@ Generated `.pyi` currently covers these exact-contract areas: | Hidden Fortran outputs | Python returns plus generated `@native_call` in native argument order | | Scalar address inputs | Python-visible `T` plus `Addr(Arg(...))` native-call projection | | Writable scalar storage | `T[()]`, or visible `T` plus projected replacement `Returns["name", T]` | -| Arrays | shaped storage with extents, strided axes, `ORDER_F` for multidimensional Fortran arrays | +| Arrays | shaped storage with extents and strided axes; multidimensional order defaults from the selected native language | | Module variables | direct module-level annotations; native accessors remain internal | | Native array descriptor handles | `Allocatable[T[...]]` and `Pointer[T[...]]` handles for module variables, supported fields, and descriptor arguments; owned allocatable result handles; unallocated or unassociated state remains inside the handle | | Constants | `Final[T]` module variables | @@ -2168,7 +2285,7 @@ Generated `.pyi` currently covers these exact-contract areas: | Fortran defined assignment | explicit mutating `assign(...)` overloads | | Opaque types | `Opaque` classes and owner-module dependency stubs | | Imports | retained contract dependencies with aliases; source kind modules are omitted after dtype resolution | -| Callbacks | complete `Callable` signatures when source interfaces resolve | +| Callbacks | named `@prototype` declarations when source interfaces resolve | - -Projection/runtime roadmap: - -1. Lower `@native_call` mappings into executable wrapper calls. -2. Add validation and coercion contracts for dtype, rank, shape, order, - strides, alignment, mutability and aliasing. -3. Add ownership and lifetime contracts for opaque handles, pointer returns, - allocatable/pointer reassociation, callbacks and work buffers. -4. Decide how to emit clean IDE/type-checker stubs from semantic `.pyi` files - without losing the native wrapper contract. diff --git a/docs/user/tutorials/basic-wrapper.md b/docs/user/tutorials/basic-wrapper.md index 6ad97cce1..6af6b6e63 100644 --- a/docs/user/tutorials/basic-wrapper.md +++ b/docs/user/tutorials/basic-wrapper.md @@ -72,7 +72,7 @@ Fortran sources -> parser facts -> semantic IR and readiness blockers -> generated Fortran bind(C) bridge - -> generated C/CPython binding and runtime support + -> generated C/CPython binding and native binding support -> compiled Python extension ``` X2PY_C_DOCS_END --> @@ -206,15 +206,14 @@ From the command line, a build looks like this: ```bash python3 -m x2py tests/data/fortran/wrapper/fruntime_abi_f90.f90 \ - --wrap \ --out-dir build/fruntime_abi \ --json ``` The command writes generated bridge, binding, runtime, object, and shared library artifacts under the output directory. The JSON output reports the -module name and generated files. The `--wrap` flag is optional when all inputs -are recognizable Fortran sources and no inspection stage is selected. +module name and generated files. Recognizable wrapper inputs select the wrapper +build stage automatically when no inspection stage is selected. ## Step 5: Import And Call The Extension diff --git a/pyproject.toml b/pyproject.toml index a9414c999..594cda9dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ where = ["."] include = ["x2py*"] [tool.setuptools.package-data] -"x2py.stdlib" = ["x2py_runtime/*"] +"x2py.binding_support" = ["*.h"] [project.scripts] x2py = "x2py.cli:main" diff --git a/tests/README.md b/tests/README.md index 439df126e..5e56cf15c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -21,11 +21,9 @@ user-visible feature that a contributor is changing. | `.pyi` AST-to-semantic conversion | `tests/semantics/conversion/pyi/` | `python3 -m pytest -q tests/semantics/conversion/pyi` | | Completed semantic policy | `tests/semantics/policy/` | `python3 -m pytest -q tests/semantics/policy` | | Wrap readiness and blockers | `tests/semantics/readiness/` | `python3 -m pytest -q tests/semantics/readiness` | -| Semantic IR-to-codegen lowering | `tests/lowering/` | `python3 -m pytest -q tests/lowering` | -| Bridge generation | `tests/codegen/bridges/` | `python3 -m pytest -q tests/codegen/bridges` | -| Python binding generation | `tests/codegen/bindings/` | `python3 -m pytest -q tests/codegen/bindings` | -| Source and `.pyi` printers | `tests/codegen/printers/` | `python3 -m pytest -q tests/codegen/printers` | +| Wrapper planning, bridge/binding generation, and source/`.pyi` printing | `tests/wrapper_codegen/` | `python3 -m pytest -q tests/wrapper_codegen` | | Naming policy | `tests/naming/` | `python3 -m pytest -q tests/naming` | +| Shared Python utilities | `tests/utilities/` | `python3 -m pytest -q tests/utilities` | | NumPy and semantic type mapping | `tests/types/` | `python3 -m pytest -q tests/types` | | Runtime handles | `tests/runtime/handles/` | `python3 -m pytest -q tests/runtime/handles` | | Documentation structure and examples | `tests/docs/` | `python3 -m pytest -q tests/docs` | @@ -40,20 +38,18 @@ empty directory merely to mirror this table. | Source package or surface | Primary test owner | | --- | --- | -| `x2py.c_parser` | `tests/parsing/c/` | -| `x2py.fortran_parser` | `tests/parsing/fortran/` | -| `x2py.pyi_parser` | `tests/parsing/pyi/` | +| `x2py.parsers.c` | `tests/parsing/c/` | +| `x2py.parsers.fortran` | `tests/parsing/fortran/` | +| `x2py.parsers.pyi` | `tests/parsing/pyi/` | | `x2py.probes` | `tests/probes/` | | `x2py.pipeline` | matching subject under `tests/pipeline/` | | `x2py.semantics.c2ir`, `fortran2ir`, `pyi2ir` | matching language under `tests/semantics/conversion/` | | semantic ownership and policy completion | `tests/semantics/policy/` | | `x2py.semantics.readiness` | `tests/semantics/readiness/` | -| `x2py.semantics.ir2ast` | `tests/lowering/` | -| `x2py.codegen.bridges` | `tests/codegen/bridges/` | -| `x2py.codegen.bindings` | `tests/codegen/bindings/` | -| `x2py.codegen.printers` | `tests/codegen/printers/` | +| `x2py.wrapper_codegen` | `tests/wrapper_codegen/` plus compiled behavior under `tests/wrapper/` | | `x2py.compiling` | compiled build and runtime feature evidence under `tests/wrapper/fortran/` | | `x2py.naming` | `tests/naming/` | +| `x2py.utilities` | `tests/utilities/` | | `x2py.types` | `tests/types/` | | `x2py.runtime.handles` | `tests/runtime/handles/` | | `x2py.cli` and parser CLIs | `tests/cli/` | @@ -73,8 +69,17 @@ know roadmap wording but not the feature module. Source-build, generated-`.pyi`, and modified-`.pyi` scenarios for one feature stay together; identical source/generated behavior uses one shared assertion body. -Do not run LAPACK wrapper runtime tests locally. Leave LAPACK coverage to GitHub -Actions unless the task explicitly requests it. +The legacy lowering AST and `x2py.codegen` implementation are removed. Their +handler maps and emitted-source details have no compatibility tests. Behavior +still required by users belongs either in `tests/wrapper_codegen/` against the +canonical plan/generator or in compiled `tests/wrapper/` coverage against the +public build APIs. + +During the wrapper-plan migration, do not run the full BLAS or LAPACK +real-library wrapper tests locally or in GitHub Actions. Exclude +`wrapper/fortran/real_libraries/test_real_blas_lapack.py`; keep the general +native-bundle tests active. Re-enable both corpora only after every other +migration row is complete. ## Adding a test or helper diff --git a/tests/_shared/fixture_outputs.py b/tests/_shared/fixture_outputs.py index edaf38605..90743ccf8 100644 --- a/tests/_shared/fixture_outputs.py +++ b/tests/_shared/fixture_outputs.py @@ -5,13 +5,13 @@ from pathlib import Path from tempfile import TemporaryDirectory -from x2py.c_parser import CParser -from x2py.c_parser.cli import attach_preprocessing_recipe +from x2py.parsers.c import CParser +from x2py.parsers.c.cli import attach_preprocessing_recipe from x2py import parse_fortran_file from x2py.pipeline.preprocessing import PreprocessingConfig, preprocess_source from x2py.semantics.c2ir import c_project_to_semantic_module from x2py.semantics.fortran2ir import fortran_file_to_semantic_modules, fortran_module_to_semantic_module -from x2py.codegen.printers.pyi_printer import emit_module +from x2py.wrapper_codegen.printers import emit_module from x2py.semantics.readiness import assess_semantic_wrap_readiness from x2py.cli import _fortran_contract_files, _semantic_report diff --git a/tests/_shared/ownership_policy_support.py b/tests/_shared/ownership_policy_support.py index c483473f2..5b6f8b2c7 100644 --- a/tests/_shared/ownership_policy_support.py +++ b/tests/_shared/ownership_policy_support.py @@ -1,5 +1,3 @@ -from dataclasses import replace - import pytest from x2py.contracts import CONTRACT_SYMBOLS @@ -12,47 +10,7 @@ SCALAR_STORAGE_CATEGORY, ) -from x2py.codegen.bind_c import ( - BindCArrayType, - BindCFunctionDef, - BindCNativeArrayDescriptorType, - BindCNativeArrayHandleProperty, - BindCNativeArrayHandleVariable, - BindCPointer, - BindCScalarModuleVariable, - native_array_descriptor_argument_type, -) - -from x2py.codegen.bindings.c_concepts import CFIDescriptorType - -from x2py.codegen.bindings.c_to_python import CPythonBindingGenerator - -from x2py.codegen.bindings.cpython_api import PythonObjectType - -from x2py.codegen.bridges.fortran_to_c import FortranToCBridgeGenerator - -from x2py.codegen.models.core import ( - Declare, - FunctionCall, - FunctionDef, - FunctionDefArgument, - FunctionDefResult, - IndexedElement, - Return, - Variable, -) - -from x2py.codegen.models.datatypes import NIL, NumpyFloat64Type, NumpyNDArrayType, convert_to_literal - -from x2py.codegen.printers.ccode import CCodePrinter - -from x2py.codegen.printers.cpythoncode import CPythonCodePrinter - -from x2py.codegen.printers.fcode import FCodePrinter - -from x2py.codegen.printers.pyi_printer import PyiPrinter - -from x2py.codegen.scope import Scope +from x2py.wrapper_codegen.printers import PyiPrinter from x2py.semantics.ownership import ( AssignmentMode, @@ -71,13 +29,10 @@ SetterAction, StorageMode, TransferMode, - codegen_action_for_variable, default_ownership_policy, set_ownership_metadata, ) -from x2py.semantics.ir2ast import semantic_ir_to_codegen_ast as _semantic_ir_to_codegen_ast - from x2py.semantics.models import ( POLICY_COMPLETION_PREPARED_METADATA, MODULE_VARIABLE_INITIALIZER_UNSUPPORTED_BLOCKER, @@ -89,7 +44,6 @@ RESOLVED_SETTER_OWNERSHIP_POLICY_METADATA, RESOLVED_OWNERSHIP_POLICY_METADATA, RESOLVED_RETURN_OWNERSHIP_POLICY_METADATA, - RESOLVED_SNAPSHOT_FIELD_ACTION_METADATA, ProjectionMapping, SemanticArgument, SemanticArrayContract, @@ -126,35 +80,10 @@ def parse_pyi_text(source: str, *args, **kwargs): return _parse_pyi_text(f"{CONTRACT_IMPORT}{source}", *args, **kwargs) -def _model_call_names(node, seen=None): - """Collect function-call names from a generated model subtree.""" - if seen is None: - seen = set() - node_id = id(node) - if node_id in seen: - return - seen.add(node_id) - if isinstance(node, FunctionCall): - yield str(node.func_name) - for attr in getattr(node, "_attribute_nodes", ()): - value = getattr(node, attr) - if isinstance(value, tuple | list): - for item in value: - yield from _model_call_names(item, seen) - elif value is not None: - yield from _model_call_names(value, seen) - - def _scalar_type(name: str = "Int32") -> SemanticType: return SemanticType(name=name, dtype=name) -def semantic_ir_to_codegen_ast(node, *args, **kwargs): - if isinstance(node, SemanticModule): - complete_semantic_policies(node) - return _semantic_ir_to_codegen_ast(node, *args, **kwargs) - - def _string_type() -> SemanticType: return SemanticType(name="String", dtype="String") @@ -275,7 +204,6 @@ def _native_array_policy( "ADDRESS_ROLE_PROJECTION", "ADDRESS_ROLE_RAW", "MODULE_VARIABLE_INITIALIZER_UNSUPPORTED_BLOCKER", - "NIL", "POLICY_COMPLETION_PREPARED_METADATA", "PROJECTED_OUTPUT_METADATA", "PYTHON_EXPORTS_METADATA", @@ -286,36 +214,15 @@ def _native_array_policy( "RESOLVED_OWNERSHIP_POLICY_METADATA", "RESOLVED_RETURN_OWNERSHIP_POLICY_METADATA", "RESOLVED_SETTER_OWNERSHIP_POLICY_METADATA", - "RESOLVED_SNAPSHOT_FIELD_ACTION_METADATA", "ArrayInteropPolicy", "ArrayInteropPolicyDispatcher", "AssignmentMode", - "BindCArrayType", - "BindCFunctionDef", - "BindCNativeArrayDescriptorType", - "BindCNativeArrayHandleProperty", - "BindCNativeArrayHandleVariable", - "BindCPointer", - "BindCScalarModuleVariable", - "CCodePrinter", - "CFIDescriptorType", - "CPythonBindingGenerator", - "CPythonCodePrinter", "CodegenAction", - "Declare", "DestructionPolicy", - "FCodePrinter", - "FortranToCBridgeGenerator", - "FunctionDef", - "FunctionDefArgument", - "FunctionDefResult", - "IndexedElement", "NativeArrayBuildRequirement", "NativeArrayHandlePolicyDispatcher", "NativeBarrierAction", "NativeBarrierDispatcher", - "NumpyFloat64Type", - "NumpyNDArrayType", "ObjectKind", "OwnershipContext", "OwnershipDecision", @@ -326,9 +233,6 @@ def _native_array_policy( "PyiPrinter", "PythonBarrierAction", "PythonBarrierDispatcher", - "PythonObjectType", - "Return", - "Scope", "SemanticArgument", "SemanticClass", "SemanticConstraint", @@ -340,30 +244,22 @@ def _native_array_policy( "SetterAction", "StorageMode", "TransferMode", - "Variable", "_address_type", "_array_type", "_derived_type", "_hidden_output_context", - "_model_call_names", "_native_array_policy", "_read_only_argument_context", "_scalar_storage_type", "_scalar_type", - "_semantic_ir_to_codegen_ast", "_string_storage_type", "_string_type", "_writable_argument_context", - "codegen_action_for_variable", "complete_semantic_policies", - "convert_to_literal", "default_ownership_policy", - "native_array_descriptor_argument_type", "native_array_descriptor_kind", "native_array_handle_build_requirements", "parse_pyi_text", "pytest", - "replace", - "semantic_ir_to_codegen_ast", "set_ownership_metadata", ) diff --git a/tests/_shared/parser_property_support.py b/tests/_shared/parser_property_support.py index 33bdbc91f..f6129db81 100644 --- a/tests/_shared/parser_property_support.py +++ b/tests/_shared/parser_property_support.py @@ -22,15 +22,15 @@ import x2py.pipeline.preprocessing as preprocessing -from x2py.c_parser import CParseError, parse_c_file +from x2py.parsers.c import CParseError, parse_c_file -from x2py.c_parser.lexer import split_top_level_c_source, top_level_split +from x2py.parsers.c.lexer import split_top_level_c_source, top_level_split from x2py.semantics.fortran2ir import fortran_file_to_semantic_modules from x2py.pipeline.pyi import pyi_text_to_semantic_module as parse_pyi_text -from x2py.codegen.printers.pyi_printer import emit_module_stubs +from x2py.wrapper_codegen.printers import emit_module_stubs from x2py import FortranParseError, parse_fortran_file diff --git a/tests/_shared/pyi_conversion_support.py b/tests/_shared/pyi_conversion_support.py index 9502fd839..edb7e795b 100644 --- a/tests/_shared/pyi_conversion_support.py +++ b/tests/_shared/pyi_conversion_support.py @@ -34,7 +34,6 @@ ) from x2py.semantics.models import ( - CALLBACK_DECLARATION_ACCESS_METADATA, ProjectionMapping, PYTHON_VALUE_IMMUTABLE, PYTHON_VALUE_MUTABILITY_METADATA, @@ -57,9 +56,7 @@ from x2py.pipeline.pyi import pyi_file_to_semantic_module, pyi_paths_to_semantic_modules, pyi_text_to_semantic_module -from x2py.pyi_parser import parse_pyi_text as parse_pyi_ast_text - -from x2py.semantics.ir2ast import semantic_ir_to_codegen_ast as _semantic_ir_to_codegen_ast +from x2py.parsers.pyi import parse_pyi_text as parse_pyi_ast_text from x2py.semantics.native_contract import native_contract_issues @@ -67,11 +64,7 @@ from x2py.semantics.readiness import assess_semantic_wrap_readiness -from x2py.codegen.bindings.c_to_python import CPythonBindingGenerator - -from x2py.codegen.printers.pyi_printer import emit_module - -from x2py.codegen.scope import Scope +from x2py.wrapper_codegen.printers import emit_module from tests._shared.fixture_outputs import FORTRAN_DATA_DIR, FORTRAN_SUFFIXES @@ -87,12 +80,6 @@ ) -def semantic_ir_to_codegen_ast(node, *args, **kwargs): - if isinstance(node, SemanticModule): - complete_semantic_policies(node) - return _semantic_ir_to_codegen_ast(node, *args, **kwargs) - - def _sample_pyi_compare_fixtures(paths: list[Path]) -> list[Path]: by_dir: dict[str, list[Path]] = {} for path in paths: @@ -139,7 +126,6 @@ def _semantic_modules_for_source(path: Path): "ADDRESS_ROLE_PROJECTION", "ADDRESS_ROLE_RAW", "BIND_TARGET_METADATA", - "CALLBACK_DECLARATION_ACCESS_METADATA", "CONTRACT_IMPORT", "CONTRACT_SYMBOLS", "FORTRAN_PYI_COMPARE_FIXTURES", @@ -150,10 +136,8 @@ def _semantic_modules_for_source(path: Path): "PYTHON_VALUE_MUTABILITY_METADATA", "SUPPRESS_DEFAULT_CONSTRUCTOR_METADATA", "USER_PRIVATE_METADATA", - "CPythonBindingGenerator", "Path", "ProjectionMapping", - "Scope", "SemanticArgument", "SemanticConstraint", "SemanticField", @@ -187,5 +171,4 @@ def _semantic_modules_for_source(path: Path): "pyi_text_to_semantic_module", "pytest", "re", - "semantic_ir_to_codegen_ast", ) diff --git a/tests/architecture/test_dependency_boundaries.py b/tests/architecture/test_dependency_boundaries.py index 72f653b86..3bb297de8 100644 --- a/tests/architecture/test_dependency_boundaries.py +++ b/tests/architecture/test_dependency_boundaries.py @@ -1,4 +1,4 @@ -"""Structural contracts for navigable codegen classes.""" +"""Structural contracts for navigable wrapper-generation boundaries.""" from __future__ import annotations @@ -6,32 +6,25 @@ from tests.wrapper.fortran._support import REPO_ROOT -CODEGEN_ROOT = REPO_ROOT / "x2py" / "codegen" -BOUNDARY_DIRS = ("bridges", "bindings", "printers") +WRAPPER_CODEGEN_ROOT = REPO_ROOT / "x2py" / "wrapper_codegen" +BOUNDARY_MODULES = ( + ("c", WRAPPER_CODEGEN_ROOT / "c" / "binding.py"), + ("fortran", WRAPPER_CODEGEN_ROOT / "fortran" / "bridge.py"), + ("printers", WRAPPER_CODEGEN_ROOT / "printers" / "pyi_printer.py"), + ("printers", WRAPPER_CODEGEN_ROOT / "printers" / "source_printers.py"), +) PUBLIC_MODULE_FUNCTIONS = { - ("bindings", "cpython_api.py", "C_to_Python"), - ("bindings", "numpy_cpython_api.py", "get_numpy_max_acceptable_version_file"), ("printers", "pyi_printer.py", "emit_module"), ("printers", "pyi_printer.py", "emit_module_stubs"), ("printers", "pyi_printer.py", "opaque_dependency_modules"), } -SHARED_PRIVATE_FUNCTIONS = { - ("bindings", "c_concepts.py", "_is_string_literal"), -} - - -def _boundary_modules(): - """Yield each Python module in the codegen boundaries under review.""" - for directory in BOUNDARY_DIRS: - for path in sorted((CODEGEN_ROOT / directory).glob("*.py")): - yield directory, path -def test_codegen_boundary_callables_are_documented(): - """Require every boundary function and method to state its contract.""" +def test_wrapper_codegen_boundary_entrypoints_and_visitors_are_documented(): + """Require public entrypoints and dispatched model visitors to state their contract.""" missing = [] - for _, path in _boundary_modules(): - tree = ast.parse(path.read_text(), filename=str(path)) + for _, path in BOUNDARY_MODULES: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) for node in tree.body: if isinstance(node, ast.FunctionDef) and ast.get_docstring(node) is None: missing.append(f"{path.name}:{node.lineno}:{node.name}") @@ -39,17 +32,19 @@ def test_codegen_boundary_callables_are_documented(): missing.extend( f"{path.name}:{method.lineno}:{node.name}.{method.name}" for method in node.body - if isinstance(method, ast.FunctionDef) and ast.get_docstring(method) is None + if isinstance(method, ast.FunctionDef) + and (not method.name.startswith("_") or method.name.startswith("_visit_")) + and ast.get_docstring(method) is None ) - assert not missing, "Undocumented codegen callables:\n" + "\n".join(missing) + assert not missing, "Undocumented wrapper-codegen callables:\n" + "\n".join(missing) -def test_codegen_uses_one_model_visitor_protocol(): - """Prevent legacy printer and extractor dispatch protocols from returning.""" +def test_wrapper_codegen_uses_one_model_visitor_protocol(): + """Prevent alternate printer and extractor dispatch protocols from appearing.""" invalid = [] lowercase_model_names = {"int", "str", "tuple"} - for _, path in _boundary_modules(): - tree = ast.parse(path.read_text(), filename=str(path)) + for _, path in BOUNDARY_MODULES: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) for node in ast.walk(tree): if isinstance(node, ast.FunctionDef) and node.name.startswith(("_print_", "_extract_")): invalid.append(f"{path.name}:{node.lineno}:{node.name}") @@ -60,31 +55,13 @@ def test_codegen_uses_one_model_visitor_protocol(): assert not invalid, "Use _visit_* handlers or named helpers:\n" + "\n".join(invalid) -def test_public_methods_precede_internal_methods(): - """Keep each class's real public API above its visitors and helpers.""" - misplaced = [] - for _, path in _boundary_modules(): - tree = ast.parse(path.read_text(), filename=str(path)) - for class_node in (node for node in tree.body if isinstance(node, ast.ClassDef)): - private_seen = False - for method in (node for node in class_node.body if isinstance(node, ast.FunctionDef)): - is_public = not method.name.startswith("_") - if is_public and private_seen: - misplaced.append(f"{path.name}:{method.lineno}:{class_node.name}.{method.name}") - is_dunder = method.name.startswith("__") and method.name.endswith("__") - if method.name.startswith("_") and not is_dunder: - private_seen = True - assert not misplaced, "Public methods below internal methods:\n" + "\n".join(misplaced) - - -def test_module_functions_are_deliberate_boundary_apis_or_shared_utilities(): +def test_module_functions_are_deliberate_boundary_apis(): """Keep stateful generation logic on its owning class.""" unexpected = [] - allowed = PUBLIC_MODULE_FUNCTIONS | SHARED_PRIVATE_FUNCTIONS - for directory, path in _boundary_modules(): - tree = ast.parse(path.read_text(), filename=str(path)) + for area, path in BOUNDARY_MODULES: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) for node in (node for node in tree.body if isinstance(node, ast.FunctionDef)): - key = (directory, path.name, node.name) - if key not in allowed: + key = (area, path.name, node.name) + if key not in PUBLIC_MODULE_FUNCTIONS: unexpected.append(f"{path.name}:{node.lineno}:{node.name}") - assert not unexpected, "Unexpected module-level codegen functions:\n" + "\n".join(unexpected) + assert not unexpected, "Unexpected module-level wrapper-codegen functions:\n" + "\n".join(unexpected) diff --git a/tests/architecture/test_package_structure.py b/tests/architecture/test_package_structure.py index 0036cebaf..2a829350d 100644 --- a/tests/architecture/test_package_structure.py +++ b/tests/architecture/test_package_structure.py @@ -4,7 +4,7 @@ PACKAGE_ROOT = Path(__file__).parents[2] / "x2py" -ROOT_PYTHON_MODULES = {"__init__.py", "__main__.py", "cli.py"} +ROOT_PYTHON_MODULES = {"__init__.py", "__main__.py", "cli.py", "stage_values.py"} def test_x2py_root_contains_only_public_entrypoint_modules(): diff --git a/tests/architecture/test_test_suite_layout.py b/tests/architecture/test_test_suite_layout.py index f7123844c..913441405 100644 --- a/tests/architecture/test_test_suite_layout.py +++ b/tests/architecture/test_test_suite_layout.py @@ -2,22 +2,22 @@ from __future__ import annotations -from pathlib import Path import re +from pathlib import Path REPO_ROOT = Path(__file__).parents[2] TEST_ROOT = REPO_ROOT / "tests" TEST_INDEX = TEST_ROOT / "README.md" MIGRATION_CHECKLIST = REPO_ROOT / "docs/maintainer/roadmap/test-suite-organization-checklist.md" +QUALITY_WORKFLOW = REPO_ROOT / ".github/workflows/quality.yml" +FULL_REAL_LIBRARY_TEST = "tests/wrapper/fortran/real_libraries/test_real_blas_lapack.py" STAGE_DIRECTORIES = { "architecture", "benchmarks", "cli", - "codegen", "docs", - "lowering", "naming", "parsing", "pipeline", @@ -26,19 +26,19 @@ "semantics", "tools", "types", + "utilities", + "wrapper_codegen", } NON_STAGE_DIRECTORIES = {"wrapper"} DOCUMENTED_SOURCE_OWNERS = { - "x2py.c_parser": "tests/parsing/c/", - "x2py.fortran_parser": "tests/parsing/fortran/", - "x2py.pyi_parser": "tests/parsing/pyi/", + "x2py.parsers.c": "tests/parsing/c/", + "x2py.parsers.fortran": "tests/parsing/fortran/", + "x2py.parsers.pyi": "tests/parsing/pyi/", "x2py.probes": "tests/probes/", "x2py.pipeline": "tests/pipeline/", - "x2py.semantics.ir2ast": "tests/lowering/", - "x2py.codegen.bridges": "tests/codegen/bridges/", - "x2py.codegen.bindings": "tests/codegen/bindings/", - "x2py.codegen.printers": "tests/codegen/printers/", + "x2py.wrapper_codegen": "tests/wrapper_codegen/", "x2py.naming": "tests/naming/", + "x2py.utilities": "tests/utilities/", "x2py.types": "tests/types/", "x2py.runtime.handles": "tests/runtime/handles/", "x2py.cli": "tests/cli/", @@ -55,6 +55,7 @@ "test_dependency_boundaries.py", "test_package_structure.py", "test_test_suite_layout.py", + "test_visitor_protocol.py", } DEPRECATED_PYTEST_ROOTS = { TEST_ROOT / "parser", @@ -161,3 +162,21 @@ def test_maintained_docs_do_not_name_deprecated_pytest_locations() -> None: if pattern in text: stale.append(f"{path.relative_to(REPO_ROOT)}: {pattern}") assert stale == [] + + +def test_full_real_library_nodes_have_one_dedicated_quality_job() -> None: + text = QUALITY_WORKFLOW.read_text(encoding="utf-8") + ordinary_jobs, dedicated_and_later = text.split(" real-library-wrappers:", maxsplit=1) + dedicated_job, _later_jobs = dedicated_and_later.split("\n coverage-report:", maxsplit=1) + + assert f"--ignore={FULL_REAL_LIBRARY_TEST}" in ordinary_jobs + assert ( + f'"{FULL_REAL_LIBRARY_TEST}::test_full_library_wrapper_imports_every_root_procedure_from_cached_shared_library[blas]"' + in dedicated_job + ) + assert ( + f'"{FULL_REAL_LIBRARY_TEST}::test_full_library_wrapper_imports_every_root_procedure_from_cached_shared_library[lapack]"' + in dedicated_job + ) + assert "ignore-real-library-wrappers" in dedicated_job + assert "matrix.library" not in dedicated_job diff --git a/tests/architecture/test_visitor_protocol.py b/tests/architecture/test_visitor_protocol.py new file mode 100644 index 000000000..8a37fcee2 --- /dev/null +++ b/tests/architecture/test_visitor_protocol.py @@ -0,0 +1,130 @@ +"""Structural contract for the active parser, semantic, and wrapper visitors.""" + +from __future__ import annotations + +import ast +import inspect + +from tests.wrapper.fortran._support import REPO_ROOT +from x2py.parsers.c.parser import CParser +from x2py.parsers.fortran.parser import FortranParser, SourceUnit, _SOURCE_UNIT_TYPES +from x2py.semantics.c2ir import CToIRConverter +from x2py.semantics.fortran2ir import FortranToIRConverter, _FortranVariableContextVisitor +from x2py.semantics.pyi2ir import _ClassBodyVisitor, _ModuleVisitor +from x2py.utilities.visitor import ClassVisitor as SemanticClassVisitor +from x2py.wrapper_codegen.c.binding import CBindingGenerator +from x2py.wrapper_codegen.fortran.bridge import FortranBridgeGenerator +from x2py.wrapper_codegen.planner import WrapperPlanner +from x2py.wrapper_codegen.printers import PyiPrinter +from x2py.wrapper_codegen.support import WrapperPlanSupportAnalyzer +from x2py.wrapper_codegen.visitor import ClassVisitor as WrapperClassVisitor + + +SEMANTIC_VISITORS = ( + FortranParser, + CToIRConverter, + FortranToIRConverter, + _FortranVariableContextVisitor, + _ClassBodyVisitor, + _ModuleVisitor, + PyiPrinter, +) +WRAPPER_VISITORS = ( + WrapperPlanSupportAnalyzer, + WrapperPlanner, + CBindingGenerator, + FortranBridgeGenerator, +) +VISITOR_IMPLEMENTATION_PATHS = ( + REPO_ROOT / "x2py" / "parsers" / "fortran" / "parser.py", + REPO_ROOT / "x2py" / "semantics" / "c2ir.py", + REPO_ROOT / "x2py" / "semantics" / "fortran2ir.py", + REPO_ROOT / "x2py" / "semantics" / "pyi2ir.py", + REPO_ROOT / "x2py" / "wrapper_codegen" / "planner.py", + REPO_ROOT / "x2py" / "wrapper_codegen" / "support.py", + REPO_ROOT / "x2py" / "wrapper_codegen" / "c" / "binding.py", + REPO_ROOT / "x2py" / "wrapper_codegen" / "fortran" / "bridge.py", + REPO_ROOT / "x2py" / "wrapper_codegen" / "printers" / "pyi_printer.py", +) + + +def test_active_model_visitors_use_their_owned_dispatch_protocol(): + assert all(issubclass(visitor, SemanticClassVisitor) for visitor in SEMANTIC_VISITORS) + assert all(issubclass(visitor, WrapperClassVisitor) for visitor in WRAPPER_VISITORS) + + +def test_semantic_class_visitor_supports_configured_handler_prefix(): + class Node: + pass + + class SpecificNode(Node): + pass + + class ParserVisitor(SemanticClassVisitor): + visitor_method_prefix = "_parse" + + @staticmethod + def _parse_Node(node): + return type(node).__name__ + + assert ParserVisitor()._visit(SpecificNode()) == "SpecificNode" + + +def test_wrapper_class_visitor_supports_configured_handler_prefix(): + class Node: + pass + + class SpecificNode(Node): + pass + + class PlanVisitor(WrapperClassVisitor): + @staticmethod + def _plan_Node(node): + return type(node).__name__ + + assert PlanVisitor(method_prefix="_plan").visit(SpecificNode()) == "SpecificNode" + + +def test_active_visitor_handlers_use_configured_class_names(): + invalid = [] + lowercase_model_names = {"int", "str", "tuple"} + for visitor in (*SEMANTIC_VISITORS, *WRAPPER_VISITORS): + handler_prefix = f"{visitor.visitor_method_prefix}_" + for name, _method in inspect.getmembers(visitor, predicate=inspect.isfunction): + if name.startswith("visit_"): + invalid.append(f"{visitor.__name__}.{name}") + if name == "_visit_not_supported" or not name.startswith(handler_prefix): + continue + model_name = name.removeprefix(handler_prefix) + if model_name[:1].islower() and model_name not in lowercase_model_names: + invalid.append(f"{visitor.__name__}.{name}") + assert not invalid, "Use configured _ handlers:\n" + "\n".join(invalid) + + +def test_parser_entrypoints_are_not_misnamed_as_visitors(): + invalid = [ + f"{parser.__name__}.{name}" + for parser in (FortranParser, CParser) + for name in vars(parser) + if name.startswith("visit_") + ] + assert not invalid, "Source entrypoints must use parse_* names:\n" + "\n".join(invalid) + + +def test_fortran_source_unit_classes_have_matching_handlers(): + assert all(issubclass(unit_type, SourceUnit) for unit_type in _SOURCE_UNIT_TYPES.values()) + assert { + kind: f"_visit_{unit_type.__name__}" + for kind, unit_type in _SOURCE_UNIT_TYPES.items() + if not hasattr(FortranParser, f"_visit_{unit_type.__name__}") + } == {} + + +def test_visitors_do_not_reimplement_mro_dispatch(): + invalid = [] + for path in VISITOR_IMPLEMENTATION_PATHS: + tree = ast.parse(path.read_text(), filename=str(path)) + for node in ast.walk(tree): + if isinstance(node, ast.Attribute) and node.attr in {"__mro__", "mro"}: + invalid.append(f"{path.relative_to(REPO_ROOT)}:{node.lineno}") + assert not invalid, "Use the owned ClassVisitor instead of local MRO dispatch:\n" + "\n".join(invalid) diff --git a/tests/benchmarks/test_parser_benchmarks.py b/tests/benchmarks/test_parser_benchmarks.py index c4cf209ea..986ff9b79 100644 --- a/tests/benchmarks/test_parser_benchmarks.py +++ b/tests/benchmarks/test_parser_benchmarks.py @@ -6,9 +6,9 @@ import pytest -from x2py.c_parser import parse_c_file +from x2py.parsers.c import parse_c_file from x2py.semantics.fortran2ir import fortran_file_to_semantic_modules -from x2py.codegen.printers.pyi_printer import emit_module_stubs +from x2py.wrapper_codegen.printers import emit_module_stubs from x2py import parse_fortran_file pytestmark = pytest.mark.skip(reason="Benchmarks are parked until benchmark adoption resumes.") diff --git a/tests/cli/_cli_support.py b/tests/cli/_cli_support.py index b15a835ba..143cb0a11 100644 --- a/tests/cli/_cli_support.py +++ b/tests/cli/_cli_support.py @@ -18,7 +18,7 @@ import pytest -from x2py.fortran_parser import cli as fortran_parser_cli +from x2py.parsers.fortran import cli as fortran_parser_cli from x2py import FortranParseError diff --git a/tests/cli/test_argument_contract.py b/tests/cli/test_argument_contract.py index a4ab979db..2ff144332 100644 --- a/tests/cli/test_argument_contract.py +++ b/tests/cli/test_argument_contract.py @@ -132,15 +132,11 @@ class extra(Opaque): "--out for wrapper builds expects a valid Python module name", ), ( - {"wrap": True, "out": "module", "makefile": True}, + {"out": "module", "makefile": True}, "--out names a compiled wrapper extension and cannot be combined with --makefile", ), ( - {"makefile": True}, - "--makefile requires --wrap", - ), - ( - {"wrap": True, "makefile": True, "verbose": True}, + {"makefile": True, "verbose": True}, "--makefile cannot be combined with --verbose", ), ( @@ -210,11 +206,8 @@ class extra(Opaque): ), ( {"paths": ["input.pyi"]}, - "Select at least one stage flag: --parse, --semantics, --pyi, --wrap-readiness, or --wrap", - ), - ( - {"paths": [], "build_manifest": "build/x2py-build.json"}, - "--build-manifest requires --wrap", + "A .pyi wrapper build requires --native-fortran-sources, --native-objects, " + "--native-library, or --native-link-item", ), ], ) @@ -252,7 +245,6 @@ def test_x2py_main_collects_many_native_inputs_from_one_option_group( [ "x2py", str(contract), - "--wrap", "--native-fortran-sources", "source_one.f90", "source_two.f90", @@ -311,6 +303,23 @@ def test_x2py_main_collects_many_native_inputs_from_one_option_group( assert payload["module_name"] == "module" +@pytest.mark.parametrize( + "overrides", + [ + {"paths": ["input.f90"]}, + {"paths": ["contract.pyi"], "native_objects": ["native.o"]}, + {"paths": ["input.f90"], "makefile": True}, + {"paths": [], "build_manifest": "build/x2py-build.json"}, + ], +) +def test_wrapper_inputs_select_the_default_build_stage(overrides): + assert x2py_cli._stage_defaults_to_wrap(_main_args(**overrides)) + + +def test_explicit_inspection_stage_prevents_default_wrapper_selection(): + assert not x2py_cli._stage_defaults_to_wrap(_main_args(parse=True)) + + def test_cli_native_fortran_flags_split_grouped_shell_words(): assert x2py_cli._cli_native_fortran_flags(["-O2 -g0", "-DNAME='value with spaces'"]) == ( "-O2", @@ -791,19 +800,13 @@ def test_cli_fortran_rejects_embedded_c_declaration_outside_execution_body(tmp_p assert "Unknown or unsupported datatype declaration" in result.stderr -@pytest.mark.parametrize( - ("extra_args", "message"), - [ - ([], "Select at least one stage flag"), - ], -) -def test_x2py_cli_rejects_pyi_without_stage(extra_args, message, tmp_path: Path): +def test_x2py_cli_defaults_pyi_to_wrapper_and_requires_native_implementation(tmp_path: Path): pyi = tmp_path / "module.pyi" pyi.write_text("def f() -> None: ...\n", encoding="utf-8") - cmd = [sys.executable, "-m", "x2py", str(pyi), *extra_args] + cmd = [sys.executable, "-m", "x2py", str(pyi)] res = subprocess.run(cmd, capture_output=True, text=True) assert res.returncode == 2 - assert message in res.stderr + assert "A .pyi wrapper build requires --native-fortran-sources" in res.stderr @pytest.mark.parametrize("macro_flag", ["-D", "-U"]) diff --git a/tests/cli/test_output_contract.py b/tests/cli/test_output_contract.py index 92ef03d1b..119a511a0 100644 --- a/tests/cli/test_output_contract.py +++ b/tests/cli/test_output_contract.py @@ -816,13 +816,13 @@ def test_fortran_parser_cli_json_and_parse_errors(tmp_path: Path): good = tmp_path / "good.f90" good.write_text("subroutine work(n)\n integer, intent(in) :: n\nend subroutine work\n", encoding="utf-8") - json_cmd = [sys.executable, "-m", "x2py.fortran_parser", str(good), "--json"] + json_cmd = [sys.executable, "-m", "x2py.parsers.fortran", str(good), "--json"] json_res = subprocess.run(json_cmd, capture_output=True, text=True, check=True) assert str(good) in json.loads(json_res.stdout) bad = tmp_path / "bad.f90" bad.write_text("subroutine bad(x)\n weirdtype :: x\nend subroutine bad\n", encoding="utf-8") - bad_cmd = [sys.executable, "-m", "x2py.fortran_parser", str(bad), "--no-color"] + bad_cmd = [sys.executable, "-m", "x2py.parsers.fortran", str(bad), "--no-color"] bad_res = subprocess.run(bad_cmd, capture_output=True, text=True) assert bad_res.returncode == 1 assert bad_res.stdout == "" diff --git a/tests/cli/test_readiness_reports.py b/tests/cli/test_readiness_reports.py index 5597ef11c..6c1cb47a8 100644 --- a/tests/cli/test_readiness_reports.py +++ b/tests/cli/test_readiness_reports.py @@ -116,7 +116,7 @@ def test_x2py_readiness_formatting_and_compiler_without_requirements(): "callback_signature_incomplete", {"owner": "handler", "needs": ["arguments", "return type"]}, ) - == "handler needs Callable[[...], ...] metadata (arguments, return type)" + == "handler needs a complete named @prototype (arguments, return type)" ) assert x2py_cli._format_semantic_blocker_item("c_unknown_type", {"owner": "api", "type": "widget"}) == "api: widget" assert x2py_cli._format_semantic_blocker_item("c_unknown_type", {"type": "widget"}) == ": widget" @@ -319,7 +319,7 @@ def serialize(received): monkeypatch.setattr(x2py_cli, "_parse_c_project", parse_project) monkeypatch.setattr(x2py_cli, "c_project_to_semantic_modules", convert) monkeypatch.setattr(x2py_cli, "expand_c_paths", expand) - monkeypatch.setattr("x2py.codegen.printers.pyi_printer.emit_module_stubs", emit) + monkeypatch.setattr("x2py.wrapper_codegen.printers.emit_module_stubs", emit) monkeypatch.setattr(x2py_cli, "asdict", serialize) assert x2py_cli._semantic_report( @@ -437,7 +437,7 @@ def serialize(module): monkeypatch.setattr(x2py_cli, "_fortran_wrapped_derived_types", wrapped) monkeypatch.setattr(x2py_cli, "_fortran_compile_time_values", compile_values) monkeypatch.setattr(x2py_cli, "fortran_file_to_semantic_modules", convert) - monkeypatch.setattr("x2py.codegen.printers.pyi_printer.emit_module_stubs", emit) + monkeypatch.setattr("x2py.wrapper_codegen.printers.emit_module_stubs", emit) monkeypatch.setattr(x2py_cli, "asdict", serialize) assert x2py_cli._semantic_report(["api"], config) == { @@ -484,8 +484,9 @@ def assess(modules, *, source): assert source == str(path) return readiness - def pyi(paths): + def pyi(paths, *, native_language): assert paths == ["api"] + assert native_language == "c" return pyi_report monkeypatch.setattr(x2py_cli, "expand_c_paths", expand) @@ -556,8 +557,9 @@ def assess(modules, *, source): assert source == str(path) return readiness - def pyi(paths): + def pyi(paths, *, native_language): assert paths == ["api"] + assert native_language == "fortran" return pyi_report expected_compile_time_values = compile_time_values @@ -611,9 +613,10 @@ def expand(paths): calls.append(("expand", paths)) return [stub] - def load(paths): + def load(paths, *, native_language): assert paths == [str(package), str(stub)] - calls.append(("load", paths)) + assert native_language == "c" + calls.append(("load", paths, native_language)) return [module] def serialize(received): @@ -633,7 +636,7 @@ def assess(modules, *, source, require_native_contract): monkeypatch.setattr(x2py_cli, "asdict", serialize) monkeypatch.setattr(x2py_cli, "assess_semantic_wrap_readiness", assess) - assert x2py_cli._pyi_readiness_report([str(package), str(stub), str(ignored)]) == { + assert x2py_cli._pyi_readiness_report([str(package), str(stub), str(ignored)], native_language="c") == { str(stub): { "source_kind": "pyi", "semantic_modules": [{"name": "api"}], @@ -642,7 +645,7 @@ def assess(modules, *, source, require_native_contract): } assert calls == [ ("expand", [str(package), str(stub), str(ignored)]), - ("load", [str(package), str(stub)]), + ("load", [str(package), str(stub)], "c"), ("asdict", module), ("assess", [module], str(stub)), ] @@ -707,7 +710,7 @@ def test_x2py_format_semantic_readiness_reports_wrappable_and_blocked_sources(): - unresolved_semantic_types: unresolved external type * api_mod.solve uses unresolved type external_t - callback_signature_incomplete: callback metadata incomplete - * api_mod.apply needs Callable[[...], ...] metadata (arguments) + * api_mod.apply needs a complete named @prototype (arguments) File: interface.pyi Source: pyi @@ -738,7 +741,7 @@ def test_x2py_format_semantic_readiness_reports_wrappable_and_blocked_sources(): assert " Why not wrappable:" in text assert " - unresolved_semantic_types: unresolved external type" in text assert " * api_mod.solve uses unresolved type external_t" in text - assert " * api_mod.apply needs Callable[[...], ...] metadata (arguments)" in text + assert " * api_mod.apply needs a complete named @prototype (arguments)" in text assert "File: interface.pyi" in text assert " Source: pyi" in text assert " Semantic modules: " in text diff --git a/tests/cli/test_stage_dispatch.py b/tests/cli/test_stage_dispatch.py index b36f31b74..4ccdba5df 100644 --- a/tests/cli/test_stage_dispatch.py +++ b/tests/cli/test_stage_dispatch.py @@ -149,7 +149,7 @@ def test_fortran_parser_cli_reports_full_source_tree_from_inline_code(tmp_path: encoding="utf-8", ) - cmd = [sys.executable, "-m", "x2py.fortran_parser", str(f90)] + cmd = [sys.executable, "-m", "x2py.parsers.fortran", str(f90)] res = subprocess.run(cmd, capture_output=True, text=True, check=True) assert f"File: {f90}" in res.stdout @@ -196,7 +196,7 @@ def test_fortran_parser_cli_semantics_pyi_and_empty_module_report_from_inline_co semantics_cmd = [ sys.executable, "-m", - "x2py.fortran_parser", + "x2py.parsers.fortran", str(module_source), "--semantics", "--json-out", @@ -209,13 +209,13 @@ def test_fortran_parser_cli_semantics_pyi_and_empty_module_report_from_inline_co assert str(module_source) in payload assert payload[str(module_source)]["semantic_modules"][0]["functions"][0]["name"] == "solve" - pyi_cmd = [sys.executable, "-m", "x2py.fortran_parser", str(module_source), "--pyi"] + pyi_cmd = [sys.executable, "-m", "x2py.parsers.fortran", str(module_source), "--pyi"] pyi_res = subprocess.run(pyi_cmd, capture_output=True, text=True, check=True) assert "@native_call([Addr(Arg(0)), Return('x', 0), Addr(Arg(1))])" in pyi_res.stdout assert "x: Addr(Float64)" not in pyi_res.stdout assert "def solve(" in pyi_res.stdout - empty_pyi_cmd = [sys.executable, "-m", "x2py.fortran_parser", str(program_source), "--pyi"] + empty_pyi_cmd = [sys.executable, "-m", "x2py.parsers.fortran", str(program_source), "--pyi"] empty_pyi_res = subprocess.run(empty_pyi_cmd, capture_output=True, text=True, check=True) assert "" in empty_pyi_res.stdout @@ -729,7 +729,7 @@ def test_x2py_and_fortran_module_entrypoints_and_debug_errors(monkeypatch, capsy monkeypatch.setattr(fortran_parser_cli, "main", lambda: 0) with pytest.raises(SystemExit) as fortran_exit: - runpy.run_module("x2py.fortran_parser.__main__", run_name="__main__") + runpy.run_module("x2py.parsers.fortran.__main__", run_name="__main__") assert fortran_exit.value.code == 0 monkeypatch.setattr(fortran_parser_cli, "main", original_fortran_main) @@ -737,7 +737,7 @@ def fail_parse(_paths): raise FortranParseError("bad", filename="bad.f90", line_number=1, source_line="bad") monkeypatch.setattr(fortran_parser_cli, "_parse_paths", fail_parse) - monkeypatch.setattr(sys, "argv", ["x2py.fortran_parser", "bad.f90", "--no-color"]) + monkeypatch.setattr(sys, "argv", ["x2py.parsers.fortran", "bad.f90", "--no-color"]) assert fortran_parser_cli.main() == 1 assert "bad.f90:1:1: error[PARSE_ERROR]: bad" in capsys.readouterr().err monkeypatch.setenv("FORTRAN_PARSER_DEBUG", "1") @@ -788,7 +788,7 @@ def test_fortran_parser_cli_debug_flag_reraises_parse_errors(tmp_path: Path): encoding="utf-8", ) - cmd = [sys.executable, "-m", "x2py.fortran_parser", str(f90), "--debug"] + cmd = [sys.executable, "-m", "x2py.parsers.fortran", str(f90), "--debug"] res = subprocess.run(cmd, capture_output=True, text=True) assert res.returncode == 1 @@ -806,7 +806,7 @@ def test_fortran_parser_cli_debug_traceback_env_reraises_parse_errors(tmp_path: encoding="utf-8", ) - cmd = [sys.executable, "-m", "x2py.fortran_parser", str(f90)] + cmd = [sys.executable, "-m", "x2py.parsers.fortran", str(f90)] res = subprocess.run( cmd, capture_output=True, @@ -833,19 +833,19 @@ def test_fortran_parser_main_public_api_modes_from_inline_source(tmp_path: Path, ) json_out = tmp_path / "report.json" - monkeypatch.setattr(sys, "argv", ["x2py.fortran_parser", str(f90), "--json-out", str(json_out), "--json"]) + monkeypatch.setattr(sys, "argv", ["x2py.parsers.fortran", str(f90), "--json-out", str(json_out), "--json"]) assert fortran_parser_cli.main() == 0 stdout_payload = json.loads(capsys.readouterr().out) assert str(f90) in stdout_payload assert json_out.exists() - monkeypatch.setattr(sys, "argv", ["x2py.fortran_parser", str(f90), "--pyi"]) + monkeypatch.setattr(sys, "argv", ["x2py.parsers.fortran", str(f90), "--pyi"]) assert fortran_parser_cli.main() == 0 pyi_out = capsys.readouterr().out assert "File:" in pyi_out assert "def work(" in pyi_out - monkeypatch.setattr(sys, "argv", ["x2py.fortran_parser", str(f90)]) + monkeypatch.setattr(sys, "argv", ["x2py.parsers.fortran", str(f90)]) assert fortran_parser_cli.main() == 0 readable = capsys.readouterr().out assert "module m" in readable diff --git a/tests/cli/test_wrap_readiness.py b/tests/cli/test_wrap_readiness.py index c6cd07700..a5a835946 100644 --- a/tests/cli/test_wrap_readiness.py +++ b/tests/cli/test_wrap_readiness.py @@ -145,7 +145,7 @@ def test_x2py_main_semantic_readiness_blocker_formatting(): assert "step uses unresolved type sim_state" in text assert "fill shape 'n' uses unresolved symbol n" in text assert "fill needs literal value for Final constant n" in text - assert "integrate.objective needs Callable[[...], ...] metadata (callback argument types)" in text + assert "integrate.objective needs a complete named @prototype (callback argument types)" in text assert "empty needs public functions" in text assert "{'payload': 1}" in text @@ -171,4 +171,4 @@ def test_x2py_main_argument_validation_errors(tmp_path: Path, monkeypatch, capsy with pytest.raises(SystemExit) as stage_error: x2py_cli.main() assert stage_error.value.code == 2 - assert "Select at least one stage flag" in capsys.readouterr().err + assert "A .pyi wrapper build requires --native-fortran-sources" in capsys.readouterr().err diff --git a/tests/codegen/bindings/test_binding_handle_policy_dispatch.py b/tests/codegen/bindings/test_binding_handle_policy_dispatch.py deleted file mode 100644 index 33d6878a1..000000000 --- a/tests/codegen/bindings/test_binding_handle_policy_dispatch.py +++ /dev/null @@ -1,545 +0,0 @@ -"""Tests split by stable ownership concept from `test_handle_policy_dispatch.py`.""" - -from tests._shared.ownership_policy_support import ( - BindCArrayType, - BindCFunctionDef, - BindCNativeArrayDescriptorType, - BindCNativeArrayHandleVariable, - BindCPointer, - CCodePrinter, - CFIDescriptorType, - CPythonBindingGenerator, - CPythonCodePrinter, - CodegenAction, - DestructionPolicy, - FortranToCBridgeGenerator, - FunctionDef, - FunctionDefArgument, - FunctionDefResult, - IndexedElement, - NIL, - NativeBarrierAction, - NumpyFloat64Type, - NumpyNDArrayType, - ObjectKind, - PythonBarrierAction, - PythonObjectType, - Scope, - SetterAction, - Variable, - _model_call_names, - _native_array_policy, - _semantic_ir_to_codegen_ast, - complete_semantic_policies, - convert_to_literal, - parse_pyi_text, - pytest, -) - - -def test_bridge_and_binding_generators_expose_ownership_action_maps(): - assert ( - CPythonBindingGenerator._RESULT_DETAIL_DISPATCHER.handlers[ - (ObjectKind.NUMPY_ARRAY, CodegenAction.SNAPSHOT_COPY) - ] - == "_snapshot_copy_result_detail_lines" - ) - assert ( - CPythonBindingGenerator._RESULT_POLICY_DISPATCHER.handlers[(ObjectKind.SCALAR, CodegenAction.SNAPSHOT_COPY)] - == "_convert_snapshot_policy_scalar_result" - ) - assert ( - CPythonBindingGenerator._PYTHON_BARRIER_DISPATCHER.handlers[PythonBarrierAction.SCALAR_STORAGE] - == "_convert_python_scalar_storage_argument" - ) - assert ( - CPythonBindingGenerator._PYTHON_BARRIER_DISPATCHER.handlers[PythonBarrierAction.STRING_VALUE] - == "_convert_python_string_value_argument" - ) - assert ( - CPythonBindingGenerator._PYTHON_BARRIER_DISPATCHER.handlers[PythonBarrierAction.STRING_STORAGE] - == "_convert_python_string_storage_argument" - ) - assert ( - FortranToCBridgeGenerator._NATIVE_BARRIER_DISPATCHER.handlers[NativeBarrierAction.PASS_CALL_LOCAL_ADDRESS] - == "_convert_native_call_local_address_argument" - ) - assert ( - FortranToCBridgeGenerator._NATIVE_BARRIER_DISPATCHER.handlers[NativeBarrierAction.PASS_RAW_ADDRESS] - == "_convert_native_raw_address_argument" - ) - assert ( - CPythonBindingGenerator._ARGUMENT_CAST_GUARD_DISPATCHER.handlers[ - (ObjectKind.SCALAR, CodegenAction.IDENTITY_OUTPUT) - ] - == "_append_unchecked_argument_cast" - ) - assert ( - CPythonBindingGenerator._ARGUMENT_CAST_GUARD_DISPATCHER.handlers[ - (ObjectKind.NUMPY_ARRAY, CodegenAction.COPY_IN_OUT) - ] - == "_append_replacement_argument_cast" - ) - assert ( - CPythonBindingGenerator._RESULT_NOTE_DISPATCHER.handlers[(ObjectKind.NUMPY_ARRAY, CodegenAction.COPY_IN_OUT)] - == "_copy_return_result_notes" - ) - assert FortranToCBridgeGenerator._NDARRAY_RESULT_DISPATCHER.handlers == { - (ObjectKind.NUMPY_ARRAY, CodegenAction.SNAPSHOT_COPY): "_build_snapshot_copy_array_result", - (ObjectKind.NUMPY_ARRAY, CodegenAction.BORROWED_VIEW): "_build_borrowed_array_result", - (ObjectKind.NUMPY_ARRAY, CodegenAction.COPY_OUT): "_build_copy_return_array_result", - (ObjectKind.NUMPY_ARRAY, CodegenAction.COPY_IN_OUT): "_build_copy_return_array_result", - (ObjectKind.NUMPY_ARRAY, CodegenAction.HIDDEN_OUTPUT): "_build_copy_return_array_result", - } - assert ( - FortranToCBridgeGenerator._ALLOCATABLE_RESULT_HELPER_DISPATCHER.handlers[ - (ObjectKind.NUMPY_ARRAY, CodegenAction.COPY_OUT) - ] - == "_uses_heap_allocatable_result_helper" - ) - native_array_handle_keys = { - ("allocatable", "argument_descriptor"), - ("allocatable", "borrowed_field_descriptor"), - ("allocatable", "borrowed_module_descriptor"), - ("allocatable", "optional_absent_handle"), - ("allocatable", "owned_result_descriptor"), - ("pointer", "argument_descriptor"), - ("pointer", "borrowed_field_descriptor"), - ("pointer", "borrowed_module_descriptor"), - ("pointer", "optional_absent_handle"), - } - assert set(FortranToCBridgeGenerator._NATIVE_ARRAY_HANDLE_DISPATCHER.handlers) == native_array_handle_keys - assert set(CPythonBindingGenerator._NATIVE_ARRAY_HANDLE_DISPATCHER.handlers) == native_array_handle_keys - assert FortranToCBridgeGenerator._ARRAY_INTEROP_POLICY_DISPATCHER.handlers == { - ("argument", "data_buffer"): "_bridge_data_buffer_argument", - ("argument", "descriptor"): "_bridge_descriptor_argument", - ("module_variable", "data_buffer"): "_bridge_data_buffer_module_variable", - ("module_variable", "descriptor"): "_bridge_descriptor_module_variable", - ("result", "data_buffer"): "_bridge_data_buffer_result", - ("result", "descriptor"): "_bridge_descriptor_result", - } - assert CPythonBindingGenerator._ARRAY_INTEROP_POLICY_DISPATCHER.handlers == { - ("argument", "data_buffer"): "_bind_data_buffer_argument", - ("argument", "descriptor"): "_bind_descriptor_argument", - ("result", "data_buffer"): "_bind_data_buffer_result", - ("result", "descriptor"): "_bind_descriptor_result", - } - assert ( - FortranToCBridgeGenerator._NATIVE_ARRAY_HANDLE_DISPATCHER.handlers[ - ("allocatable", "borrowed_module_descriptor") - ] - == "_bridge_borrowed_native_array_module_handle" - ) - assert ( - CPythonBindingGenerator._NATIVE_ARRAY_HANDLE_DISPATCHER.handlers[("pointer", "borrowed_module_descriptor")] - == "_bind_borrowed_native_array_module_handle" - ) - dispatchers = ( - (FortranToCBridgeGenerator, "_NATIVE_BARRIER_DISPATCHER"), - (FortranToCBridgeGenerator, "_FUNCTION_ARGUMENT_POLICY_DISPATCHER"), - (FortranToCBridgeGenerator, "_RESULT_POLICY_DISPATCHER"), - (FortranToCBridgeGenerator, "_REPLACEMENT_RESULT_DISPATCHER"), - (FortranToCBridgeGenerator, "_NDARRAY_RESULT_DISPATCHER"), - (FortranToCBridgeGenerator, "_ALLOCATABLE_RESULT_HELPER_DISPATCHER"), - (FortranToCBridgeGenerator, "_FIELD_SETTER_POLICY_DISPATCHER"), - (FortranToCBridgeGenerator, "_FIELD_GETTER_POLICY_DISPATCHER"), - (FortranToCBridgeGenerator, "_MODULE_VARIABLE_POLICY_DISPATCHER"), - (FortranToCBridgeGenerator, "_MODULE_ARRAY_GETTER_POLICY_DISPATCHER"), - (FortranToCBridgeGenerator, "_NATIVE_ARRAY_HANDLE_DISPATCHER"), - (FortranToCBridgeGenerator, "_CALLBACK_ARGUMENT_POLICY_DISPATCHER"), - (FortranToCBridgeGenerator, "_CALLBACK_RESULT_POLICY_DISPATCHER"), - (CPythonBindingGenerator, "_PYTHON_BARRIER_DISPATCHER"), - (CPythonBindingGenerator, "_ARGUMENT_DETAIL_DISPATCHER"), - (CPythonBindingGenerator, "_ARGUMENT_CAST_GUARD_DISPATCHER"), - (CPythonBindingGenerator, "_RESULT_POLICY_DISPATCHER"), - (CPythonBindingGenerator, "_RESULT_DETAIL_DISPATCHER"), - (CPythonBindingGenerator, "_RESULT_NOTE_DISPATCHER"), - (CPythonBindingGenerator, "_PROPERTY_SETTER_POLICY_DISPATCHER"), - (CPythonBindingGenerator, "_BORROWED_GETTER_POLICY_DISPATCHER"), - (CPythonBindingGenerator, "_NATIVE_ARRAY_HANDLE_DISPATCHER"), - (CPythonBindingGenerator, "_NATIVE_ARRAY_DESCRIPTOR_ARGUMENT_DISPATCHER"), - (CPythonBindingGenerator, "_ARGUMENT_RETURN_PROJECTION_DISPATCHER"), - (CPythonBindingGenerator, "_PROJECTED_ARGUMENT_OBJECT_DISPATCHER"), - (CPythonBindingGenerator, "_ARRAY_ACCESS_VALIDATION_DISPATCHER"), - (CPythonBindingGenerator, "_ARRAY_RELEASE_POLICY_DISPATCHER"), - ) - for generator, dispatcher_name in dispatchers: - dispatcher = getattr(generator, dispatcher_name) - assert dispatcher.handlers - assert all(hasattr(generator, handler_name) for handler_name in dispatcher.handlers.values()) - - assert set(FortranToCBridgeGenerator._NATIVE_BARRIER_DISPATCHER.handlers) == { - NativeBarrierAction.PASS_VALUE, - NativeBarrierAction.PASS_CALL_LOCAL_ADDRESS, - NativeBarrierAction.PASS_STORAGE_ADDRESS, - NativeBarrierAction.PASS_RAW_ADDRESS, - NativeBarrierAction.PASS_ARRAY_DESCRIPTOR, - NativeBarrierAction.PASS_WRAPPER_ADDRESS, - } - assert set(CPythonBindingGenerator._PYTHON_BARRIER_DISPATCHER.handlers) == { - PythonBarrierAction.SCALAR_VALUE, - PythonBarrierAction.SCALAR_STORAGE, - PythonBarrierAction.ARRAY_STORAGE, - PythonBarrierAction.STRING_VALUE, - PythonBarrierAction.STRING_STORAGE, - PythonBarrierAction.RAW_ADDRESS, - PythonBarrierAction.WRAPPER_INSTANCE, - } - assert ( - FortranToCBridgeGenerator._RESULT_POLICY_DISPATCHER.handlers.keys() - == CPythonBindingGenerator._RESULT_POLICY_DISPATCHER.handlers.keys() - ) - assert ( - CPythonBindingGenerator._ARGUMENT_RETURN_PROJECTION_DISPATCHER.handlers[ - (ObjectKind.NUMPY_ARRAY, CodegenAction.COPY_IN_OUT, True) - ] - == "_project_native_argument_return" - ) - assert ( - CPythonBindingGenerator._PROJECTED_ARGUMENT_OBJECT_DISPATCHER.handlers[ - (ObjectKind.NUMPY_ARRAY, CodegenAction.IN_PLACE_ARGUMENT, True) - ] - == "_record_projected_argument_object" - ) - assert ( - CPythonBindingGenerator._ARGUMENT_RETURN_PROJECTION_DISPATCHER.handlers[ - (ObjectKind.NUMPY_ARRAY, CodegenAction.CALL_LOCAL_INPUT, True) - ] - == "_project_visible_argument_return" - ) - assert ( - CPythonBindingGenerator._PROJECTED_ARGUMENT_OBJECT_DISPATCHER.handlers[ - (ObjectKind.NUMPY_ARRAY, CodegenAction.CALL_LOCAL_INPUT, True) - ] - == "_record_projected_argument_object" - ) - assert ( - CPythonBindingGenerator._ARRAY_ACCESS_VALIDATION_DISPATCHER.handlers[ - (ObjectKind.NUMPY_ARRAY, CodegenAction.IDENTITY_OUTPUT) - ] - == "_writable_array_access_validation" - ) - assert ( - FortranToCBridgeGenerator._FIELD_SETTER_POLICY_DISPATCHER.handlers[SetterAction.WRITE_THROUGH] - == "_build_field_setter" - ) - assert ( - FortranToCBridgeGenerator._FIELD_SETTER_POLICY_DISPATCHER.handlers[SetterAction.REJECT_REPLACEMENT] - == "_skip_field_setter" - ) - assert ( - FortranToCBridgeGenerator._FIELD_GETTER_POLICY_DISPATCHER.handlers[ - (ObjectKind.NUMPY_ARRAY, CodegenAction.BORROWED_VIEW) - ] - == "_append_borrowed_array_field_getter" - ) - assert ( - FortranToCBridgeGenerator._FIELD_GETTER_POLICY_DISPATCHER.handlers[ - (ObjectKind.SCALAR, CodegenAction.SNAPSHOT_COPY) - ] - == "_append_nullable_scalar_field_getter" - ) - assert ( - FortranToCBridgeGenerator._MODULE_VARIABLE_POLICY_DISPATCHER.handlers[ - (ObjectKind.SCALAR, CodegenAction.SNAPSHOT_COPY) - ] - == "_scalar_module_variable" - ) - assert ( - FortranToCBridgeGenerator._MODULE_VARIABLE_POLICY_DISPATCHER.handlers[ - (ObjectKind.DERIVED_TYPE, CodegenAction.BORROWED_VIEW) - ] - == "_derived_module_variable" - ) - assert ( - FortranToCBridgeGenerator._MODULE_VARIABLE_POLICY_DISPATCHER.handlers[ - (ObjectKind.DERIVED_TYPE, CodegenAction.SNAPSHOT_COPY) - ] - == "_snapshot_derived_module_variable" - ) - assert ( - CPythonBindingGenerator._ARRAY_RELEASE_POLICY_DISPATCHER.handlers[DestructionPolicy.PYTHON_REFCOUNT] - == "_release_python_owned_array_memory" - ) - assert ( - CPythonBindingGenerator._ARRAY_RELEASE_POLICY_DISPATCHER.handlers[DestructionPolicy.BLOCKED] - == "_blocked_array_release_policy" - ) - - -def test_native_array_handle_binding_builds_runtime_handle_from_named_generated_ops(): - module = parse_pyi_text( - """ -values: Allocatable[Float64[:]] -""", - module_name="native_handle_binding_substrate", - ) - complete_semantic_policies(module) - lowered = _semantic_ir_to_codegen_ast(module, Scope(name=module.name, scope_type="module")) - variable = lowered.variables[0] - op_original = FunctionDef( - "__x2py_values_shape", - (), - (), - FunctionDefResult(NIL), - scope=lowered.scope, - ) - op_wrapper = BindCFunctionDef( - "bind_c___x2py_values_shape", - (), - (), - FunctionDefResult(NIL), - original_function=op_original, - scope=lowered.scope, - ) - handle_variable = variable.clone( - variable.name, - new_class=BindCNativeArrayHandleVariable, - operation_functions={"shape": op_wrapper}, - original_variable=variable, - ) - - binding = CPythonBindingGenerator("", 0) - binding.scope = lowered.scope - binding._native_array_handle_owner_module = Variable(PythonObjectType(), "mod", memory_handling="alias") - body = binding._visit_BindCNativeArrayHandleVariable(handle_variable) - - call_names = {name for node in body for name in _model_call_names(node)} - assert { - "PyDict_New", - "PyDict_SetItem", - "PyImport_ImportModule", - "PyObject_CallObject", - "PyObject_GetAttrString", - } <= call_names - assert binding._python_object_map[handle_variable].name - assert handle_variable.operation_functions == {"shape": op_wrapper} - - -def test_pointer_descriptor_view_operation_wrapper_decodes_generated_cfi_descriptor_pointer(): - scope = Scope(name="descriptor_view_operation", scope_type="module") - policy = _native_array_policy( - descriptor_kind="pointer", - handle_kind="borrowed_module_descriptor", - to_numpy="descriptor_view", - descriptor_interop="pointer_c_descriptor", - operations=("associated", "nullify", "to_numpy"), - ) - original = FunctionDef( - "__x2py_target_to_numpy", - (), - (), - FunctionDefResult(NIL), - scope=scope, - ) - descriptor_arg = Variable(BindCPointer(), "descriptor", is_argument=True, memory_handling="alias") - operation = BindCFunctionDef( - "bind_c___x2py_target_to_numpy", - (FunctionDefArgument(descriptor_arg),), - (), - FunctionDefResult(NIL), - original_function=original, - scope=scope, - ) - source_variable = Variable( - NumpyNDArrayType.get_new(NumpyFloat64Type(), 1, "F"), - "target", - native_array_handle_policy=policy, - ) - handle_variable = source_variable.clone( - source_variable.name, - new_class=BindCNativeArrayHandleVariable, - operation_functions={"to_numpy": operation}, - original_variable=source_variable, - ) - binding = CPythonBindingGenerator("", 0) - binding.scope = scope - - wrapped = binding._native_array_descriptor_view_operation_wrapper(handle_variable, operation) - call_names = set(_model_call_names(wrapped.body)) - code = CPythonCodePrinter("test.c", verbose=0)._visit(wrapped) - - assert "bind_c___x2py_target_to_numpy" in call_names - assert { - "PyDict_New", - "PyDict_SetItem", - "PyLong_FromLongLong", - "PyLong_FromVoidPtr", - } <= call_names - assert "CFI_CDESC_T(1) target_descriptor_storage" in code - assert "CFI_establish(target_descriptor, NULL, CFI_attribute_pointer, CFI_type_double" in code - assert "bind_c___x2py_target_to_numpy(target_descriptor)" in code - assert "((CFI_cdesc_t*)target_descriptor)->base_addr" in code - assert "((CFI_cdesc_t*)target_descriptor)->dim[INT64_C(0)].sm" in code - - -def test_native_array_handle_operation_wrapper_uses_descriptor_reader_only_for_pointer_descriptor_view(): - policy = _native_array_policy( - descriptor_kind="pointer", - handle_kind="borrowed_module_descriptor", - to_numpy="descriptor_view", - descriptor_interop="pointer_c_descriptor", - operations=("associated", "nullify", "to_numpy"), - ) - variable = Variable( - NumpyNDArrayType.get_new(NumpyFloat64Type(), 1, "F"), - "target", - native_array_handle_policy=policy, - ) - binding = CPythonBindingGenerator("", 0) - - assert binding._uses_native_array_descriptor_view_operation_wrapper(variable, "to_numpy") is True - assert binding._uses_native_array_descriptor_view_operation_wrapper(variable, "associated") is False - assert ( - binding._uses_native_array_descriptor_view_operation_wrapper( - variable.clone( - variable.name, - native_array_handle_policy=_native_array_policy( - descriptor_kind="pointer", - handle_kind="borrowed_module_descriptor", - to_numpy="unsupported", - operations=("associated", "nullify", "to_numpy"), - ), - ), - "to_numpy", - ) - is False - ) - - -def test_cfi_descriptor_type_printing_is_local_to_descriptor_reader_path(): - descriptor = Variable(CFIDescriptorType(), "descriptor", memory_handling="alias") - printer = CCodePrinter("test.c", verbose=0) - - assert printer._get_declare_type(descriptor) == "CFI_cdesc_t*" - assert "ISO_Fortran_binding" in printer.get_additional_imports() - - -@pytest.mark.parametrize( - ("descriptor_kind", "annotation"), - [ - ("allocatable", "Allocatable[Float64[:]]"), - ("pointer", "Pointer[Float64[:]]"), - ], -) -def test_native_array_optional_handle_argument_binding_uses_presence_tuple( - descriptor_kind, - annotation, -): - module = parse_pyi_text( - f""" -def maybe(values: {annotation} | None = ...) -> None: ... -""", - module_name=f"{descriptor_kind}_optional_handle_argument_binding", - ) - complete_semantic_policies(module) - lowered = _semantic_ir_to_codegen_ast(module, Scope(name=module.name, scope_type="module")) - argument = lowered.funcs[0].arguments[0] - policy = argument.var.native_array_handle_policy - - assert policy.descriptor_kind == descriptor_kind - assert policy.handle_kind == "optional_absent_handle" - assert policy.optional_absent is True - - bridge = FortranToCBridgeGenerator("", 0) - bridge.scope = lowered.funcs[0].scope - bridged = bridge._convert_argument(argument, lowered.funcs[0]) - - descriptor_arg = bridged["c_arg"].var - descriptor_dummy = lowered.funcs[0].scope.collect_tuple_element( - IndexedElement(descriptor_arg.new_var, convert_to_literal(0)), - ) - presence_var = lowered.funcs[0].scope.collect_tuple_element( - IndexedElement(descriptor_arg.new_var, convert_to_literal(1)), - ) - assert descriptor_arg.class_type is BindCNativeArrayDescriptorType.get_new(has_presence=True) - assert descriptor_dummy.native_array_handle_policy is policy - assert descriptor_dummy.is_optional is True - assert descriptor_dummy.memory_handling == ("alias" if descriptor_kind == "pointer" else "heap") - assert bridged["optional_presence_var"] is presence_var - assert presence_var.is_argument is True - assert bridged["body"] == [] - - binding = CPythonBindingGenerator("", 0) - binding.scope = lowered.funcs[0].scope - collect_arg = Variable(PythonObjectType(), "py_values", memory_handling="alias") - converted = binding._convert_argument( - argument.var, - collect_arg, - bound_argument=False, - is_bind_c_argument=False, - ) - - assert converted["args"][0].class_type is BindCNativeArrayDescriptorType.get_new(has_presence=True) - assert converted["owns_type_check"] is True - assert len(converted["default_init"]) == 2 - assert len(converted["body"]) > 0 - - -def test_native_array_descriptor_argument_binding_forwards_fixed_rank_one_extent(): - module = parse_pyi_text( - """ -def fill(values: Allocatable[Float64[2]]) -> None: ... -""", - module_name="allocatable_handle_fixed_extent_binding", - ) - complete_semantic_policies(module) - lowered = _semantic_ir_to_codegen_ast(module, Scope(name=module.name, scope_type="module")) - argument = lowered.funcs[0].arguments[0] - binding = CPythonBindingGenerator("", 0) - - assert binding._rank_one_fixed_extent(argument.var) == 2 - - -def test_normal_array_bind_c_argument_binding_uses_native_handle_fallback(): - module = parse_pyi_text( - """ -def fill(values: Float64[:]) -> None: ... -""", - module_name="normal_array_handle_fallback_binding", - ) - complete_semantic_policies(module) - lowered = _semantic_ir_to_codegen_ast(module, Scope(name=module.name, scope_type="module")) - argument = lowered.funcs[0].arguments[0] - binding = CPythonBindingGenerator("", 0) - binding.scope = lowered.funcs[0].scope - collect_arg = Variable(PythonObjectType(), "py_values", memory_handling="alias") - - converted = binding._convert_argument( - argument.var, - collect_arg, - bound_argument=False, - is_bind_c_argument=True, - ) - - descriptor_type = converted["args"][0].class_type - assert isinstance(descriptor_type, BindCArrayType) - assert descriptor_type.has_rank is False - assert descriptor_type.has_itemsize is False - assert descriptor_type.has_strides is False - assert converted["owns_type_check"] is True - assert len(converted["body"]) == 1 - assert len(converted["body"][0].blocks) == 2 - assert "array_actual_helper" in str(converted["body"][0]) - - -@pytest.mark.parametrize( - "annotation", - [ - "Float64[...]", - "String[8][:]", - ], -) -def test_normal_array_bind_c_argument_binding_keeps_specialized_array_paths(annotation): - module = parse_pyi_text( - f""" -def fill(values: {annotation}) -> None: ... -""", - module_name="specialized_array_binding", - ) - complete_semantic_policies(module) - lowered = _semantic_ir_to_codegen_ast(module, Scope(name=module.name, scope_type="module")) - argument = lowered.funcs[0].arguments[0] - binding = CPythonBindingGenerator("", 0) - - assert binding._bind_c_array_argument_uses_native_handle_fallback(argument.var) is False diff --git a/tests/codegen/bridges/test_bridge_handle_policy_dispatch.py b/tests/codegen/bridges/test_bridge_handle_policy_dispatch.py deleted file mode 100644 index 5d2652ee9..000000000 --- a/tests/codegen/bridges/test_bridge_handle_policy_dispatch.py +++ /dev/null @@ -1,331 +0,0 @@ -"""Tests split by stable ownership concept from `test_handle_policy_dispatch.py`.""" - -from tests._shared.ownership_policy_support import ( - BindCNativeArrayDescriptorType, - BindCNativeArrayHandleProperty, - BindCNativeArrayHandleVariable, - BindCPointer, - CCodePrinter, - CPythonBindingGenerator, - CPythonCodePrinter, - CodegenAction, - Declare, - FCodePrinter, - FortranToCBridgeGenerator, - FunctionDef, - FunctionDefArgument, - FunctionDefResult, - IndexedElement, - PythonObjectType, - Return, - Scope, - Variable, - _model_call_names, - _semantic_ir_to_codegen_ast, - complete_semantic_policies, - convert_to_literal, - parse_pyi_text, - pytest, -) - - -@pytest.mark.parametrize( - ("descriptor_kind", "annotation", "expected_operations"), - [ - ( - "allocatable", - "Allocatable[Float64[:]]", - { - "aligned", - "allocated", - "array_actual", - "deallocate", - "descriptor", - "native_byte_order", - "resize", - "shape", - "to_numpy", - "writeable", - }, - ), - ( - "pointer", - "Pointer[Float64[:]]", - { - "aligned", - "array_actual", - "associated", - "contiguous", - "descriptor", - "native_byte_order", - "nullify", - "shape", - "writeable", - }, - ), - ], -) -def test_native_array_handle_module_variable_bridge_uses_completed_handle_policy_dispatch( - descriptor_kind, - annotation, - expected_operations, -): - module = parse_pyi_text( - f""" -values: {annotation} -""", - module_name=f"{descriptor_kind}_native_handle_bridge_dispatch", - ) - complete_semantic_policies(module) - lowered = _semantic_ir_to_codegen_ast(module, Scope(name=module.name, scope_type="module")) - generator = FortranToCBridgeGenerator("", 0) - generator.scope = lowered.scope - - handle = generator._visit_Variable(lowered.variables[0]) - - assert isinstance(handle, BindCNativeArrayHandleVariable) - assert handle.native_array_handle_policy.handle_kind == "borrowed_module_descriptor" - assert handle.native_array_handle_policy.descriptor_kind == descriptor_kind - assert handle.original_variable is lowered.variables[0] - assert set(handle.operation_functions) == expected_operations - - -def test_native_array_handle_module_operations_print_and_wrap_pointer_handoff_results(): - module = parse_pyi_text( - """ -values: Allocatable[Float64[:]] -""", - module_name="native_handle_module_operations", - ) - complete_semantic_policies(module) - lowered = _semantic_ir_to_codegen_ast(module, Scope(name=module.name, scope_type="module")) - bridged = FortranToCBridgeGenerator("", 0)._visit_Module(lowered) - - fortran_code = FCodePrinter("native_handle_module_operations.f90", verbose=0)._visit(bridged) - assert "function bind_c_private__x2py_values_shape()" in fortran_code - assert "function bind_c_private__x2py_values_array_actual()" in fortran_code - assert "function bind_c_private__x2py_values_descriptor()" in fortran_code - assert "values_descriptor = c_null_ptr" in fortran_code - - cpython_module = CPythonBindingGenerator("", 0)._visit_Module(bridged) - c_code = CPythonCodePrinter("native_handle_module_operations.c", verbose=0)._visit(cpython_module) - assert "_native_array_handle_from_generated_ops" in c_code - assert "PyLong_FromVoidPtr" in c_code - assert "private__x2py_values_array_actual" in c_code - assert "private__x2py_values_descriptor" in c_code - assert "static PyObject* bind_c_private__x2py_values_array_actual(void)" not in c_code - assert "static PyObject* bind_c_private__x2py_values_descriptor(void)" not in c_code - assert "static PyObject* bind_c_private__x2py_values_array_actual_wrapper" in c_code - assert "static PyObject* bind_c_private__x2py_values_descriptor_wrapper" in c_code - - -def test_native_array_descriptor_view_reader_builds_runtime_mapping_from_cfi_descriptor_fields(): - binding = CPythonBindingGenerator("", 0) - binding.scope = Scope(name="descriptor_reader", scope_type="function") - descriptor_pointer = Variable(BindCPointer(), "descriptor", memory_handling="alias") - descriptor_result = binding._new_python_object("descriptor_view") - - body = binding._native_array_descriptor_view_body(descriptor_pointer, descriptor_result, rank=2) - - call_names = {name for node in body for name in _model_call_names(node)} - assert { - "PyDict_New", - "PyDict_SetItem", - "PyList_Append", - "PyList_New", - "PyLong_FromLongLong", - "PyLong_FromVoidPtr", - "PyUnicode_FromString", - } <= call_names - - -def test_native_array_descriptor_view_reader_prints_cfi_descriptor_access_without_global_requirement(): - binding = CPythonBindingGenerator("", 0) - scope = Scope(name="descriptor_reader", scope_type="function") - binding.scope = scope - descriptor_pointer = Variable(BindCPointer(), "descriptor", memory_handling="alias", is_argument=True) - descriptor_result = binding._new_python_object("descriptor_view") - body = binding._native_array_descriptor_view_body(descriptor_pointer, descriptor_result, rank=2) - body.append(Return(descriptor_result)) - function = FunctionDef( - "decode_descriptor", - (FunctionDefArgument(descriptor_pointer),), - body, - FunctionDefResult(descriptor_result), - scope=scope, - ) - - printer = CPythonCodePrinter("test.c", verbose=0) - code = printer._visit(function) - - assert "((CFI_cdesc_t*)descriptor)->base_addr" in code - assert "((CFI_cdesc_t*)descriptor)->elem_len" in code - assert "((CFI_cdesc_t*)descriptor)->rank" in code - assert "((CFI_cdesc_t*)descriptor)->dim[INT64_C(0)].lower_bound" in code - assert "((CFI_cdesc_t*)descriptor)->dim[INT64_C(1)].extent" in code - assert "((CFI_cdesc_t*)descriptor)->dim[INT64_C(1)].sm" in code - assert "PyLong_FromVoidPtr" in code - assert "PyLong_FromLongLong" in code - assert "ISO_Fortran_binding" in printer.get_additional_imports() - - -@pytest.mark.parametrize( - ("descriptor_kind", "annotation"), - [ - ("allocatable", "Allocatable[Float64[:]]"), - ("pointer", "Pointer[Float64[:]]"), - ], -) -def test_native_array_handle_field_generation_uses_completed_handle_policy_dispatch( - descriptor_kind, - annotation, -): - module = parse_pyi_text( - f""" -class box: - values: {annotation} -""", - module_name=f"{descriptor_kind}_handle_field_dispatch", - ) - complete_semantic_policies(module) - lowered = _semantic_ir_to_codegen_ast(module, Scope(name=module.name, scope_type="module")) - field = lowered.classes[0].attributes[0] - policy = field.native_array_handle_policy - - assert policy.descriptor_kind == descriptor_kind - assert policy.handle_kind == "borrowed_field_descriptor" - assert policy.origin == "derived_field" - assert policy.owner_retention == "parent_wrapper" - assert policy.blocker is None - - bridged = FortranToCBridgeGenerator("", 0)._visit_Module(lowered) - wrapped_field = bridged.classes[0].attributes[0] - - assert isinstance(wrapped_field, BindCNativeArrayHandleProperty) - assert wrapped_field.owner_class is lowered.classes[0] - assert wrapped_field.native_array_handle_policy is policy - expected_operations = { - "aligned", - "array_actual", - "descriptor", - "native_byte_order", - "shape", - "writeable", - "allocated" if descriptor_kind == "allocatable" else "associated", - "deallocate" if descriptor_kind == "allocatable" else "nullify", - "resize" if descriptor_kind == "allocatable" else "descriptor", - } - if descriptor_kind == "pointer": - expected_operations.add("contiguous") - assert expected_operations <= set(wrapped_field.operation_functions) - - fortran_code = FCodePrinter("field_handle.f90", verbose=0)._visit(bridged) - assert "self%values" in fortran_code - if descriptor_kind == "allocatable": - assert "bound_values = c_loc(values(lbound(values," in fortran_code - assert "kind=i64)))" in fortran_code - - cpython_module = CPythonBindingGenerator("", 0)._visit_Module(bridged) - c_code = CPythonCodePrinter("field_handle.c", verbose=0)._visit(cpython_module) - assert "_native_array_handle_from_generated_ops" in c_code - assert "values_handle_getter" in c_code - - -def test_native_array_handle_result_generation_uses_completed_handle_policy_dispatch(): - module = parse_pyi_text( - """ -def make_values() -> Allocatable[Float64[:]]: ... -""", - module_name="native_handle_result_dispatch", - ) - complete_semantic_policies(module) - lowered = _semantic_ir_to_codegen_ast(module, Scope(name=module.name, scope_type="module")) - bridged = FortranToCBridgeGenerator("", 0)._visit_Module(lowered) - fortran_code = FCodePrinter("owned_result.f90", verbose=0)._visit(bridged) - assert "if (allocated(make_values" in fortran_code - assert "deallocate(make_values" in fortran_code - - cpython_module = CPythonBindingGenerator("", 0)._visit_Module(bridged) - c_code = CPythonCodePrinter("owned_result.c", verbose=0)._visit(cpython_module) - assert "sizeof(CFI_CDESC_T(1))" in c_code - assert "CFI_attribute_allocatable" in c_code - assert "CFI_allocate(" in c_code - assert "CFI_deallocate(" in c_code - assert 'PyUnicode_FromString("owned")' in c_code - assert "_native_array_handle_from_generated_ops" in c_code - - -@pytest.mark.parametrize( - ("descriptor_kind", "annotation"), - [ - ("allocatable", "Allocatable[Float64[:]]"), - ( - "pointer", - "Annotated[Pointer[Float64[:]], PointerPolicy(nullable=True, transfer='call_local', " - "target_owner='caller', lifetime='call', deallocation='deallocate_resize', " - "shape_source='pointer_bounds', contiguity='contiguous', reassociation='allocate_resize', " - "aliasing='descriptor', mutability='mutable')]", - ), - ], -) -def test_native_array_handle_argument_generation_uses_completed_handle_policy_dispatch( - descriptor_kind, - annotation, -): - module = parse_pyi_text( - f""" -def fill(values: {annotation}) -> Returns["values", {annotation}]: ... -""", - module_name=f"{descriptor_kind}_handle_argument_dispatch", - ) - complete_semantic_policies(module) - lowered = _semantic_ir_to_codegen_ast(module, Scope(name=module.name, scope_type="module")) - argument = lowered.funcs[0].arguments[0] - decision = argument.var.ownership_decision - policy = argument.var.native_array_handle_policy - - assert decision.codegen_action is CodegenAction.IN_PLACE_ARGUMENT - assert decision.mutates_native is True - assert decision.projects_result is True - assert policy.descriptor_kind == descriptor_kind - assert policy.handle_kind == "argument_descriptor" - assert policy.output_projection == "projected_handle" - assert policy.blocker is None - - bridge = FortranToCBridgeGenerator("", 0) - bridge.scope = lowered.funcs[0].scope - bridged = bridge._convert_argument(argument, lowered.funcs[0]) - - descriptor_arg = bridged["c_arg"].var - descriptor_dummy = lowered.funcs[0].scope.collect_tuple_element( - IndexedElement(descriptor_arg.new_var, convert_to_literal(0)), - ) - f_printer = FCodePrinter("test.f90", verbose=0) - f_printer.set_scope(lowered.funcs[0].scope) - f_printer._kind = lambda expr: "f64" - fortran_declaration = f_printer._visit(Declare(descriptor_dummy)) - assert descriptor_arg.class_type is BindCNativeArrayDescriptorType.get_new(has_presence=False) - assert descriptor_dummy.native_array_handle_policy is policy - assert descriptor_dummy.memory_handling == ("alias" if descriptor_kind == "pointer" else "heap") - assert descriptor_dummy.is_argument is True - assert bridged["body"] == [] - assert bridged["optional_presence_var"] is None - assert CCodePrinter("test.c", verbose=0)._get_declare_type(descriptor_dummy) == "void*" - assert (", pointer" if descriptor_kind == "pointer" else ", allocatable") in fortran_declaration - assert f_printer._fortran_argument_access(descriptor_dummy) == "readwrite" - assert "values" in str(bridged["f_arg"]) - - binding = CPythonBindingGenerator("", 0) - binding.scope = lowered.funcs[0].scope - collect_arg = Variable(PythonObjectType(), "py_values", memory_handling="alias") - converted = binding._convert_argument( - argument.var, - collect_arg, - bound_argument=False, - is_bind_c_argument=False, - ) - - assert converted["args"][0].class_type is BindCNativeArrayDescriptorType.get_new(has_presence=False) - assert converted["owns_type_check"] is True - assert len(converted["body"]) > 0 - assert len(converted["default_init"]) == 1 diff --git a/tests/data/fortran/wrapper/fscalar_derived_actual_dummy_matrix_f90.f90 b/tests/data/fortran/wrapper/fscalar_derived_actual_dummy_matrix_f90.f90 new file mode 100644 index 000000000..60c5f544d --- /dev/null +++ b/tests/data/fortran/wrapper/fscalar_derived_actual_dummy_matrix_f90.f90 @@ -0,0 +1,350 @@ +module phase8_left_types + use iso_c_binding, only: c_int32_t + implicit none + + type :: item + integer(c_int32_t) :: value = 3_c_int32_t + end type item + + type(item) :: state + +contains + + function make_item(initial) result(value) + integer(c_int32_t), intent(in) :: initial + type(item) :: value + value%value = initial + end function make_item + +end module phase8_left_types + +module phase8_right_types + use iso_c_binding, only: c_int32_t + implicit none + + type :: item + integer(c_int32_t) :: value = 7_c_int32_t + end type item + + type(item) :: state + +contains + + function make_item(initial) result(value) + integer(c_int32_t), intent(in) :: initial + type(item) :: value + value%value = initial + end function make_item + +end module phase8_right_types + +module fscalar_derived_actual_dummy_matrix_f90 + use iso_c_binding, only: c_bool, c_int, c_int32_t + use phase8_left_types, only: left_item => item + use phase8_right_types, only: right_item => item + implicit none + + type :: item + integer(c_int32_t) :: value = 0_c_int32_t + end type item + + type :: sequence_item + sequence + integer(c_int32_t) :: value = 0_c_int32_t + end type sequence_item + + ! The five rank-zero module actual declaration forms. + type(item) :: ordinary_module + type(item) :: ordinary_module_two + type(item), target :: target_module + type(item), allocatable :: allocatable_module + type(item), allocatable, target :: allocatable_target_module + type(item), pointer :: pointer_module => null() + + ! Extra distinct origins let one native call exercise several transactions. + type(item), target :: pointer_target_one + type(item), target :: pointer_target_two + type(item), pointer :: pointer_module_two => null() + type(item), pointer :: allocation_follower => null() + integer(c_int32_t) :: writable_call_count = 0_c_int32_t + +contains + + subroutine reset_state() + ordinary_module%value = 10_c_int32_t + ordinary_module_two%value = 12_c_int32_t + target_module%value = 20_c_int32_t + pointer_target_one%value = 50_c_int32_t + pointer_target_two%value = 60_c_int32_t + if (allocated(allocatable_module)) deallocate(allocatable_module) + if (allocated(allocatable_target_module)) deallocate(allocatable_target_module) + allocate(allocatable_module) + allocate(allocatable_target_module) + allocatable_module%value = 30_c_int32_t + allocatable_target_module%value = 40_c_int32_t + pointer_module => pointer_target_one + pointer_module_two => pointer_target_two + nullify(allocation_follower) + writable_call_count = 0_c_int32_t + end subroutine reset_state + + subroutine clear_allocatable_module() + if (allocated(allocatable_module)) deallocate(allocatable_module) + end subroutine clear_allocatable_module + + subroutine clear_allocatable_target_module() + nullify(allocation_follower) + if (allocated(allocatable_target_module)) deallocate(allocatable_target_module) + end subroutine clear_allocatable_target_module + + subroutine clear_pointer_module() + nullify(pointer_module) + end subroutine clear_pointer_module + + subroutine associate_allocation_follower() + if (allocated(allocatable_target_module)) then + allocation_follower => allocatable_target_module + else + nullify(allocation_follower) + end if + end subroutine associate_allocation_follower + + function allocation_follower_value() result(value) + integer(c_int32_t) :: value + value = -1_c_int32_t + if (associated(allocation_follower)) value = allocation_follower%value + end function allocation_follower_value + + function get_writable_call_count() result(value) + integer(c_int32_t) :: value + value = writable_call_count + end function get_writable_call_count + + function make_item(initial) result(value) + integer(c_int32_t), intent(in) :: initial + type(item) :: value + value%value = initial + end function make_item + + function make_sequence_item(initial) result(value) + integer(c_int32_t), intent(in) :: initial + type(sequence_item) :: value + value%value = initial + end function make_sequence_item + + subroutine make_target_item(initial, value) + integer(c_int32_t), intent(in) :: initial + type(item), target, intent(out) :: value + value%value = initial + end subroutine make_target_item + + subroutine make_allocatable_item(initial, make_present, value) + integer(c_int32_t), intent(in) :: initial + logical(c_bool), intent(in) :: make_present + type(item), allocatable, intent(out) :: value + if (make_present) then + allocate(value) + value%value = initial + end if + end subroutine make_allocatable_item + + subroutine make_allocatable_target_item(initial, make_present, value) + integer(c_int32_t), intent(in) :: initial + logical(c_bool), intent(in) :: make_present + type(item), allocatable, target, intent(out) :: value + if (make_present) then + allocate(value) + value%value = initial + end if + end subroutine make_allocatable_target_item + + function make_pointer_item(selector) result(value) + integer(c_int32_t), intent(in) :: selector + type(item), pointer :: value + select case (selector) + case (1_c_int32_t) + value => pointer_target_one + case (2_c_int32_t) + value => pointer_target_two + case default + nullify(value) + end select + end function make_pointer_item + + ! The six exact native dummy forms. + function read_object(value) result(observed) + type(item), intent(in) :: value + integer(c_int32_t) :: observed + observed = value%value + end function read_object + + function read_target(value) result(observed) + type(item), target, intent(in) :: value + integer(c_int32_t) :: observed + observed = value%value + end function read_target + + function read_allocatable(value) result(observed) + type(item), allocatable, intent(in) :: value + integer(c_int32_t) :: observed + observed = -1_c_int32_t + if (allocated(value)) observed = value%value + end function read_allocatable + + function read_allocatable_target(value) result(observed) + type(item), allocatable, target, intent(in) :: value + integer(c_int32_t) :: observed + observed = -1_c_int32_t + if (allocated(value)) observed = value%value + end function read_allocatable_target + + function read_pointer_input(value) result(observed) + type(item), pointer, intent(in) :: value + integer(c_int32_t) :: observed + observed = -1_c_int32_t + if (associated(value)) observed = value%value + end function read_pointer_input + + function read_value(value) result(observed) + type(item), value :: value + integer(c_int32_t) :: observed + observed = value%value + end function read_value + + function read_sequence_value(value) result(observed) + type(sequence_item), value :: value + integer(c_int32_t) :: observed + observed = value%value + end function read_sequence_value + + subroutine increment_object(value, amount) + type(item), intent(inout) :: value + integer(c_int32_t), intent(in) :: amount + value%value = value%value + amount + end subroutine increment_object + + subroutine set_allocatable(value, new_value) + type(item), allocatable, intent(inout) :: value + integer(c_int32_t), intent(in) :: new_value + if (new_value < 0_c_int32_t) then + if (allocated(value)) deallocate(value) + return + end if + if (.not. allocated(value)) allocate(value) + value%value = new_value + end subroutine set_allocatable + + subroutine set_allocatable_target(value, new_value) + type(item), allocatable, target, intent(inout) :: value + integer(c_int32_t), intent(in) :: new_value + if (new_value < 0_c_int32_t) then + if (allocated(value)) deallocate(value) + return + end if + if (.not. allocated(value)) allocate(value) + value%value = new_value + end subroutine set_allocatable_target + + subroutine set_pointer(value, selector) + type(item), pointer, intent(inout) :: value + integer(c_int32_t), intent(in) :: selector + select case (selector) + case (1_c_int32_t) + value => pointer_target_one + case (2_c_int32_t) + value => pointer_target_two + case (3_c_int32_t) + nullify(value) + allocate(value) + value%value = 70_c_int32_t + case (4_c_int32_t) + if (associated(value)) deallocate(value) + nullify(value) + case default + nullify(value) + end select + end subroutine set_pointer + + function read_six_forms(object_value, target_value, allocatable_value, & + allocatable_target_value, pointer_value, value_value) result(total) + type(item), intent(in) :: object_value + type(item), target, intent(in) :: target_value + type(item), allocatable, intent(in) :: allocatable_value + type(item), allocatable, target, intent(in) :: allocatable_target_value + type(item), pointer, intent(in) :: pointer_value + type(item), value :: value_value + integer(c_int32_t) :: total + total = object_value%value + target_value%value + value_value%value + if (allocated(allocatable_value)) total = total + allocatable_value%value + if (allocated(allocatable_target_value)) total = total + allocatable_target_value%value + if (associated(pointer_value)) total = total + pointer_value%value + end function read_six_forms + + function read_qualified(left, right) result(total) + type(left_item), intent(in) :: left + type(right_item), intent(in) :: right + integer(c_int32_t) :: total + total = left%value * 100_c_int32_t + right%value + end function read_qualified + + subroutine mutate_three_descriptors(first, second, third, amount) + type(item), allocatable, intent(inout) :: first + type(item), allocatable, target, intent(inout) :: second + type(item), pointer, intent(inout) :: third + integer(c_int32_t), intent(in) :: amount + if (.not. allocated(first)) allocate(first) + if (.not. allocated(second)) allocate(second) + first%value = first%value + amount + second%value = second%value + amount + third => pointer_target_two + end subroutine mutate_three_descriptors + + function read_duplicate(first, second) result(total) + type(item), intent(in) :: first + type(item), intent(in) :: second + integer(c_int32_t) :: total + total = first%value + second%value + end function read_duplicate + + function read_optional(first, second) result(total) + type(item), intent(in), optional :: first + type(item), intent(in), optional :: second + integer(c_int32_t) :: total + total = 0_c_int32_t + if (present(first)) total = total + first%value + if (present(second)) total = total + second%value + end function read_optional + + subroutine mutate_duplicate(first, second) + type(item), intent(inout) :: first + type(item), intent(inout) :: second + writable_call_count = writable_call_count + 1_c_int32_t + first%value = first%value + 1_c_int32_t + second%value = second%value + 1_c_int32_t + end subroutine mutate_duplicate + + subroutine hold_allocatable(value, milliseconds) + type(item), allocatable, intent(inout) :: value + integer(c_int32_t), intent(in) :: milliseconds + integer(c_int) :: start_count, current_count, count_rate + call system_clock(start_count, count_rate) + do + call system_clock(current_count) + if ((current_count - start_count) * 1000_c_int / count_rate >= milliseconds) exit + end do + if (allocated(value)) value%value = value%value + 1_c_int32_t + end subroutine hold_allocatable + + subroutine hold_object(value, milliseconds) + type(item), intent(inout) :: value + integer(c_int32_t), intent(in) :: milliseconds + integer(c_int) :: start_count, current_count, count_rate + call system_clock(start_count, count_rate) + do + call system_clock(current_count) + if ((current_count - start_count) * 1000_c_int / count_rate >= milliseconds) exit + end do + value%value = value%value + 1_c_int32_t + end subroutine hold_object + +end module fscalar_derived_actual_dummy_matrix_f90 diff --git a/tests/docs/test_examples.py b/tests/docs/test_examples.py index 49e0b14b3..6cd20a596 100644 --- a/tests/docs/test_examples.py +++ b/tests/docs/test_examples.py @@ -2,6 +2,7 @@ from __future__ import annotations +import ast from dataclasses import dataclass import os import platform @@ -13,12 +14,19 @@ import pytest +from x2py import pyi_text_to_semantic_module + ROOT = Path(__file__).parents[2] DOC_PATHS = [ ROOT / "README.md", *sorted(path for path in (ROOT / "docs").rglob("*.md") if "old_docs" not in path.parts), ] +AUDITED_PYTHON_DOC_PATHS = [ + ROOT / "README.md", + *sorted((ROOT / "docs/user/getting-started").glob("*.md")), + *sorted((ROOT / "docs/user/guide").glob("*.md")), +] TEST_MARKER = re.compile(r"^\s*\s*$") OUTPUT_MARKER = re.compile(r"^\s*\s*$") SOURCE_MARKER = re.compile(r"^\s*\s*$") @@ -63,6 +71,17 @@ def test_id(self) -> str: return f"{self.path.relative_to(ROOT)}:{self.line}" +@dataclass(frozen=True) +class DocumentedPythonBlock: + path: Path + line: int + source: str + + @property + def test_id(self) -> str: + return f"{self.path.relative_to(ROOT)}:{self.line}" + + def _platform_id() -> str: machine = platform.machine().lower() machine = {"amd64": "x86_64", "arm64": "aarch64"}.get(machine, machine) @@ -198,6 +217,24 @@ def _documented_content_from_path(path: Path) -> tuple[list[DocumentationExample DOCUMENTED_SOURCES = [source for _examples, sources in DOCUMENTATION_CONTENT for source in sources] +def _documented_python_blocks(path: Path) -> list[DocumentedPythonBlock]: + """Collect every visible Python fence for syntax and contract validation.""" + lines = _visible_documentation_lines(path) + blocks = [] + index = 0 + while index < len(lines): + if lines[index].strip() != "```python": + index += 1 + continue + source, after_block, _language = _fenced_block(lines, index, language="python") + blocks.append(DocumentedPythonBlock(path=path, line=index + 1, source=source)) + index = after_block + return blocks + + +DOCUMENTED_PYTHON_BLOCKS = [block for path in AUDITED_PYTHON_DOC_PATHS for block in _documented_python_blocks(path)] + + def _command_argv(example: DocumentationExample) -> list[str]: if example.language == "python": return [sys.executable, "-c", example.command] @@ -229,6 +266,14 @@ def test_documented_source_input(source: DocumentedSource): assert source.source_text.rstrip("\n") == source.source_path.read_text(encoding="utf-8").rstrip("\n") +@pytest.mark.parametrize("block", DOCUMENTED_PYTHON_BLOCKS, ids=lambda block: block.test_id) +def test_documented_python_block_is_valid(block: DocumentedPythonBlock): + """Keep Python examples parseable and semantic contract examples loadable.""" + ast.parse(block.source, filename=block.test_id) + if "from x2py.contracts import" in block.source: + pyi_text_to_semantic_module(block.source, module_name="documentation_example") + + @pytest.mark.parametrize("path", DOC_PATHS, ids=lambda path: str(path.relative_to(ROOT))) def test_documented_expected_output_labels_are_automatically_verified(path: Path): lines = _visible_documentation_lines(path) diff --git a/tests/docs/test_structure.py b/tests/docs/test_structure.py index 292c35e18..f819da1cc 100644 --- a/tests/docs/test_structure.py +++ b/tests/docs/test_structure.py @@ -36,6 +36,16 @@ C_DOCS_DISABLED = "