From c53d3260aaa9130046088523d73f445d4b7ac540 Mon Sep 17 00:00:00 2001 From: Leilei Zhang Date: Tue, 29 Sep 2026 17:47:28 +0800 Subject: [PATCH 01/15] Reject native null before mutating stock JSON collections Validate the receiver's stock runtime-class contract before JSON value writes, preflight bulk mutations, and keep custom generic collections nullable. Narrow verified stock class stubs and test real native behavior and strict consumers. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- bindings/py/README.md | 15 +- bindings/py/dynwinrt.pyi | 4 + bindings/py/src/lib.rs | 36 ++ bindings/py/src/runtime.rs | 51 +- tools/dynwinrt-codegen/python/README.md | 12 +- .../src/codegen/winrt/python/collections.rs | 138 +++++- .../codegen/winrt/python/generator/class.rs | 27 +- .../codegen/winrt/python/generator/types.rs | 13 +- .../src/codegen/winrt/python/method.rs | 29 +- .../src/codegen/winrt/python/stub_helpers.rs | 40 +- .../src/codegen/winrt/python/stubs.rs | 107 ++++- .../src/codegen/winrt/python/type_helpers.rs | 21 + tools/dynwinrt-codegen/src/meta.rs | 3 + .../tests/python_consumer_typing_test.rs | 21 +- .../tests/python_json_native_null_test.rs | 437 ++++++++++++++++++ .../tests/python_stub_nullability_test.rs | 16 +- 16 files changed, 912 insertions(+), 58 deletions(-) create mode 100644 tools/dynwinrt-codegen/tests/python_json_native_null_test.rs diff --git a/bindings/py/README.md b/bindings/py/README.md index 01e4f74d..cee268dc 100644 --- a/bindings/py/README.md +++ b/bindings/py/README.md @@ -54,8 +54,8 @@ the array value itself remains non-null. String, GUID, scalar, enum, and struct array elements and keys remain non-null. A view or iterator obtained from a mutable collection can expose a null slot, and WinRT collection interfaces do not retain enough provenance for the stubs to distinguish that case. For -example, a `JsonArray` holds -`IJsonValue | None`, and `get_files_async()` returns +example, a custom `IVector` can hold `IJsonValue | None`, and +`get_files_async()` returns `WinRTCoroutine[Sequence[StorageFile | None]]`. Value-type elements remain non-null. @@ -65,6 +65,17 @@ type is a WinRT reference type and store a real null WinRT value. This includes `setdefault()`. String, GUID, scalar, enum, and struct keys and value-type elements reject `None` with `TypeError`. +Stock `Windows.Data.Json.JsonArray` and `JsonObject` are exceptions: their +native APIs reject a null `IJsonValue` pointer. Generated wrappers check the +receiver's native class before mutation, including generic `IVector` +and `IMap` views, raw `DynWinRTArray` inputs, `replace_all`, +slice assignment, `extend` and `update`. A rejected null leaves the JSON +collection unchanged. Use `JsonValue.create_null_value()` to store JSON +**semantic** null; it is a non-null `IJsonValue` object. Their class stubs type +elements as non-null, while generic interface stubs retain `| None` because +custom implementations can store a native null and a view's origin is only +known at runtime. + Other arguments keep accepting `None` where they did before. The stubs are optimistic, like the generated TypeScript declarations: the runtime still returns `None` when a WinRT API returns null, so check the API documentation diff --git a/bindings/py/dynwinrt.pyi b/bindings/py/dynwinrt.pyi index 973f5ac5..70500be2 100644 --- a/bindings/py/dynwinrt.pyi +++ b/bindings/py/dynwinrt.pyi @@ -508,6 +508,10 @@ class DynWinRTValue: def to_guid(self) -> WinGUID: ... def to_bytes(self) -> bytes: ... def is_null(self) -> bool: ... + def _matches_runtime_class(self, iid: WinGUID, name: str) -> bool: ... + def _validate_non_null_collection_input( + self, value: DynWinRTValue, iid: WinGUID, name: str + ) -> DynWinRTValue: ... def _try_query_interface(self, iid: WinGUID) -> bool: ... # True after release(), release_projected(), or a closing # projected_lifetime_scope(); a WinRT null reference is not released. diff --git a/bindings/py/src/lib.rs b/bindings/py/src/lib.rs index b1e91d93..54070e51 100644 --- a/bindings/py/src/lib.rs +++ b/bindings/py/src/lib.rs @@ -422,6 +422,22 @@ def _dynwinrt_normalize_index(value, length): raise IndexError('collection index out of range') return value +def _dynwinrt_preflight_non_null_collection(self, values): + contract = getattr(type(self), '_dynwinrt_non_null_collection_contract', None) + if contract is None: + return None + native = getattr(self, '_collection_obj', self._obj) + if not native._matches_runtime_class(*contract): + return None + items = list(values) + for item in items: + raw = getattr(item, '_obj', item) + if raw is None: + raw = DynWinRTValue.null_value() + if isinstance(raw, DynWinRTValue): + native._validate_non_null_collection_input(raw, *contract) + return items + class _WinRTSequenceMixin(_Sequence): def __len__(self): return self.size @@ -465,6 +481,10 @@ class _WinRTMutableSequenceMixin(_MutableSequence): index = min(index, length) self.insert_at(index, value) + def extend(self, values): + checked = _dynwinrt_preflight_non_null_collection(self, values) + return super().extend(values if checked is None else checked) + class _WinRTIterableMixin(_Iterable): def __iter__(self): return iter(self.first()) @@ -514,6 +534,22 @@ class _WinRTMutableMappingMixin(_MutableMapping): raise KeyError(key) self.remove(key) + def update(self, other=(), /, **kwargs): + contract = getattr(type(self), '_dynwinrt_non_null_collection_contract', None) + native = getattr(self, '_collection_obj', self._obj) + if contract is None or not native._matches_runtime_class(*contract): + return super().update(other, **kwargs) + if hasattr(other, 'keys'): + entries = [(key, other[key]) for key in other.keys()] + else: + entries = list(other) + entries.extend(kwargs.items()) + for key, _ in entries: + if key is None: + raise TypeError('map key cannot be None') + _dynwinrt_preflight_non_null_collection(self, (value for _, value in entries)) + return super().update(entries) + async def _dynwinrt_convert_future(future, converter): try: completed = await future diff --git a/bindings/py/src/runtime.rs b/bindings/py/src/runtime.rs index e03532bf..0b8526d9 100644 --- a/bindings/py/src/runtime.rs +++ b/bindings/py/src/runtime.rs @@ -10,7 +10,7 @@ use pyo3::types::PyDict; use windows::Win32::System::WinRT::{ RO_INIT_MULTITHREADED, RO_INIT_SINGLETHREADED, RO_INIT_TYPE, RoInitialize, }; -use windows::core::{GUID, HSTRING, IUnknown, Interface}; +use windows::core::{GUID, HSTRING, IInspectable, IUnknown, Interface}; use crate::errors::{ InputSlot, map_dynwinrt_error, map_dynwinrt_error_with_context, map_windows_error, @@ -1146,6 +1146,27 @@ impl DynWinRTValue { } } + /// Match a native runtime class only after confirming its class interface. + /// Custom collections with the same generic IID need not provide a class + /// name (or accept the stock class's element contract). + fn matches_runtime_class(&self, iid: &GUID, name: &str) -> PyResult { + let receiver = self.receiver("collection runtime-class check")?; + let mut raw = std::ptr::null_mut(); + match unsafe { receiver.query(iid, &mut raw) }.ok() { + Ok(()) => { + let class_interface = unsafe { IUnknown::from_raw(raw) }; + let inspectable: IInspectable = + class_interface.cast().map_err(map_windows_error)?; + let actual = inspectable + .GetRuntimeClassName() + .map_err(map_windows_error)?; + Ok(actual == name) + } + Err(error) if error.code() == windows::Win32::Foundation::E_NOINTERFACE => Ok(false), + Err(error) => Err(map_windows_error(error)), + } + } + /// Reject this value if released; `slot` names where `operation` received it. pub(crate) fn check_input(&self, operation: &str, slot: InputSlot) -> PyResult<()> { match self.1 { @@ -1626,6 +1647,34 @@ impl DynWinRTValue { self.0.is_null_object() } + fn _matches_runtime_class(&self, iid: &WinGUID, name: &str) -> PyResult { + self.matches_runtime_class(&iid.0, name) + } + + /// Validate the receiver-specific native collection contract before any + /// method call. An array may already contain nulls when supplied as a + /// DynWinRTArray or a raw DynWinRTValue. + fn _validate_non_null_collection_input( + &self, + value: DynWinRTValue, + iid: &WinGUID, + name: &str, + ) -> PyResult { + value.check_input("collection input", InputSlot::Argument(0))?; + let contains_null = match &value.0 { + dynwinrt::WinRTValue::Array(data) => { + (0..data.len()).any(|index| data.get(index).is_null_object()) + } + other => other.is_null_object(), + }; + if contains_null && self.matches_runtime_class(&iid.0, name)? { + return Err(PyTypeError::new_err(format!( + "{name} requires a non-null IJsonValue; use JsonValue.create_null_value() for JSON null" + ))); + } + Ok(value) + } + /// Guard-only QueryInterface probe; never treats a native failure as a non-match. fn _try_query_interface(&self, iid: &WinGUID) -> PyResult { self.ensure_live()?; diff --git a/tools/dynwinrt-codegen/python/README.md b/tools/dynwinrt-codegen/python/README.md index f9dda50b..140706f1 100644 --- a/tools/dynwinrt-codegen/python/README.md +++ b/tools/dynwinrt-codegen/python/README.md @@ -150,9 +150,15 @@ Collection subscripts use the input contract for keys and values: for example, `properties["uri"] = uri` accepts a generated `Uri`, while reading the item still returns `DynWinRTValue | None`. Sequence item assignment, slice assignment, and `insert` likewise accept projected inputs without changing their read types; -integer indices take one item and slices take an iterable of items. Existing -nullable `collections.abc` contracts remain unchanged. To pass a native null -reference, use `DynWinRTValue.null_value()`, not implicit `None` boxing. +integer indices take one item and slices take an iterable of items. Generic +nullable `collections.abc` contracts remain unchanged. The stock `JsonArray` +and `JsonObject` classes instead have non-null `IJsonValue` +element/value contracts in their stubs. Their native implementations reject +`None` (including through generic interface views) before mutation; use +`JsonValue.create_null_value()` for JSON semantic null. A custom +`IVector` or `IMap` may still store a native null. +For other nullable WinRT positions, pass `DynWinRTValue.null_value()` rather +than implicit `None` boxing. The output directory belongs to codegen; do not store handwritten files in it. After changing metadata files, SDK versions, or reference inputs, regenerate the diff --git a/tools/dynwinrt-codegen/src/codegen/winrt/python/collections.rs b/tools/dynwinrt-codegen/src/codegen/winrt/python/collections.rs index c4df0121..43bfab51 100644 --- a/tools/dynwinrt-codegen/src/codegen/winrt/python/collections.rs +++ b/tools/dynwinrt-codegen/src/codegen/winrt/python/collections.rs @@ -1,7 +1,9 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use crate::meta::{ClassMeta, InterfaceMeta, WINDOWS_FOUNDATION_COLLECTIONS_NAMESPACE}; +use crate::meta::{ + ClassMeta, CollectionInputRole, InterfaceMeta, WINDOWS_FOUNDATION_COLLECTIONS_NAMESPACE, +}; use crate::types::{TypeIdentity, TypeIdentityKind, TypeMeta}; pub(crate) const IITERABLE_PIID: &str = "faa585ea-6214-4217-afda-7f46de5869b3"; @@ -14,6 +16,71 @@ pub(crate) const IOBSERVABLE_MAP_PIID: &str = "65df2bf5-bf39-41b5-aebc-5a9d865e4 pub(crate) const IMAP_VIEW_PIID: &str = "e480ce40-a338-4ada-adcf-272272e48cb9"; pub(crate) const IKEY_VALUE_PAIR_PIID: &str = "02b51929-c1c4-4a7e-8940-0312b5c18500"; +#[derive(Clone, Copy)] +pub(crate) struct NonNullJsonCollection { + pub(crate) class_name: &'static str, + pub(crate) class_iid: &'static str, +} + +const JSON_ARRAY: NonNullJsonCollection = NonNullJsonCollection { + class_name: "Windows.Data.Json.JsonArray", + class_iid: "08c1ddb6-0cbd-4a9a-b5d3-2f852dc37e81", +}; +const JSON_OBJECT: NonNullJsonCollection = NonNullJsonCollection { + class_name: "Windows.Data.Json.JsonObject", + class_iid: "064e24dd-29c2-4f83-9ac1-9ee11578beb3", +}; + +fn is_json_value(typ: &TypeMeta) -> bool { + matches!( + typ, + TypeMeta::Interface { + namespace, + name, + iid, + } if namespace == "Windows.Data.Json" + && name == "IJsonValue" + && iid.eq_ignore_ascii_case("a3219ecb-f0b3-4dcd-beee-19d48cd3ed1e") + ) +} + +pub(crate) fn non_null_json_input( + role: CollectionInputRole, + typ: &TypeMeta, +) -> Option { + let element = match typ { + TypeMeta::Array(element) => element.as_ref(), + element => element, + }; + if !is_json_value(element) { + return None; + } + match role { + CollectionInputRole::Element => Some(JSON_ARRAY), + CollectionInputRole::Value => Some(JSON_OBJECT), + CollectionInputRole::Key => None, + } +} + +pub(crate) fn non_null_json_collection( + kind: CollectionKind, + args: &[TypeMeta], +) -> Option { + match (kind, args) { + (CollectionKind::MutableSequence, [value]) if is_json_value(value) => Some(JSON_ARRAY), + (CollectionKind::MutableMapping, [TypeMeta::String, value]) if is_json_value(value) => { + Some(JSON_OBJECT) + } + _ => None, + } +} + +pub(crate) fn stock_json_class_contract(class: &ClassMeta) -> Option { + let iface = class_interface(class)?; + let contract = non_null_json_collection(interface_kind(iface)?, &iface.generic_args)?; + (class.full_name == contract.class_name).then_some(contract) +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum CollectionKind { Iterable, @@ -169,6 +236,75 @@ pub(crate) fn is_mapping_input(kind: CollectionKind, args: &[TypeMeta]) -> bool mod tests { use super::*; + #[test] + fn json_null_contract_needs_the_exact_element_iid_and_native_class() { + let json_value = TypeMeta::Interface { + namespace: "Windows.Data.Json".into(), + name: "IJsonValue".into(), + iid: "a3219ecb-f0b3-4dcd-beee-19d48cd3ed1e".into(), + }; + assert_eq!( + non_null_json_input(CollectionInputRole::Element, &json_value) + .unwrap() + .class_name, + "Windows.Data.Json.JsonArray" + ); + assert_eq!( + non_null_json_input( + CollectionInputRole::Value, + &TypeMeta::Array(Box::new(json_value.clone())) + ) + .unwrap() + .class_name, + "Windows.Data.Json.JsonObject" + ); + assert!(non_null_json_input(CollectionInputRole::Key, &json_value).is_none()); + assert!( + non_null_json_collection( + CollectionKind::MutableMapping, + &[TypeMeta::String, json_value.clone()] + ) + .is_some() + ); + + let iface = InterfaceMeta { + generic_piid: Some(IVECTOR_PIID.into()), + generic_args: vec![json_value.clone()], + ..Default::default() + }; + let custom = ClassMeta { + full_name: "Contoso.CustomJsonVector".into(), + default_interface: Some(iface.clone()), + ..Default::default() + }; + assert!(stock_json_class_contract(&custom).is_none()); + assert!( + stock_json_class_contract(&ClassMeta { + full_name: JSON_ARRAY.class_name.into(), + default_interface: Some(iface), + ..Default::default() + }) + .is_some() + ); + + let TypeMeta::Interface { + namespace, + name, + iid: _, + } = json_value + else { + unreachable!() + }; + let wrong_iid = TypeMeta::Interface { + namespace, + name, + iid: "00000000-0000-0000-0000-000000000000".into(), + }; + assert!(non_null_json_input(CollectionInputRole::Element, &wrong_iid).is_none()); + assert!(non_null_json_collection(CollectionKind::MutableSequence, &[wrong_iid]).is_none()); + assert!(non_null_json_input(CollectionInputRole::Value, &TypeMeta::Object).is_none()); + } + #[test] fn map_piids_project_to_python_mapping_protocols() { assert_eq!( diff --git a/tools/dynwinrt-codegen/src/codegen/winrt/python/generator/class.rs b/tools/dynwinrt-codegen/src/codegen/winrt/python/generator/class.rs index 2053061d..bd2a725b 100644 --- a/tools/dynwinrt-codegen/src/codegen/winrt/python/generator/class.rs +++ b/tools/dynwinrt-codegen/src/codegen/winrt/python/generator/class.rs @@ -8,7 +8,8 @@ use super::structs::{generate_struct_helpers, generate_struct_imports}; use super::*; use crate::codegen::winrt::extensions::winui::{self, WinUiAbiType}; use crate::codegen::winrt::python::collections::{ - CollectionKind, class_interface, interface_kind, map_iterable_identity, runtime_mixin, + CollectionKind, class_interface, interface_kind, map_iterable_identity, + non_null_json_collection, runtime_mixin, stock_json_class_contract, }; use crate::codegen::winrt::python::member_plan::{ ClassMemberPlan, PlannedMember, ScopePlan, class_instance_interfaces, interface_member_plan, @@ -41,6 +42,9 @@ pub fn generate_class<'a>( let context = context.as_ref(); let collection_iface = class_interface(class); let collection_kind = collection_iface.and_then(interface_kind); + let non_null_json = collection_iface + .zip(collection_kind) + .and_then(|(iface, kind)| non_null_json_collection(kind, &iface.generic_args)); let known_full_names = context.known_full_names(); let winui_bootstrap = winui::resolve_application_bootstrap(class, &known_full_names); let has_public_composition = class @@ -329,6 +333,12 @@ pub fn generate_class<'a>( } else if native_projectable { out.push_str(" _dynwinrt_projectable_class_type = True\n"); } + if let Some(contract) = non_null_json { + out.push_str(&format!( + " _dynwinrt_non_null_collection_contract = (WinGUID.parse('{}'), '{}')\n", + contract.class_iid, contract.class_name + )); + } if matches!( collection_kind, Some(CollectionKind::Mapping | CollectionKind::MutableMapping) @@ -831,6 +841,14 @@ pub fn generate_class<'a>( } out.push_str(" _dynwinrt_interface_type = True\n"); out.push_str(&format!(" _dynwinrt_interface_iid = IID_{symbol}\n")); + if let Some(contract) = interface_kind(req_iface) + .and_then(|kind| non_null_json_collection(kind, &req_iface.generic_args)) + { + out.push_str(&format!( + " _dynwinrt_non_null_collection_contract = (WinGUID.parse('{}'), '{}')\n", + contract.class_iid, contract.class_name + )); + } if matches!( interface_kind(req_iface), Some(CollectionKind::Mapping | CollectionKind::MutableMapping) @@ -1335,6 +1353,13 @@ fn generate_python_constructor( context.class_name(class) )); } + if let Some(contract) = stock_json_class_contract(class) { + out.push_str(&format!( + " if not obj._matches_runtime_class(WinGUID.parse('{}'), '{}'):\n\ + \x20 raise TypeError('Expected a native {}')\n", + contract.class_iid, contract.class_name, contract.class_name + )); + } if let Some(default_iface) = &class.default_interface { if default_iface.iid.is_empty() { out.push_str(" self._obj = obj\n"); diff --git a/tools/dynwinrt-codegen/src/codegen/winrt/python/generator/types.rs b/tools/dynwinrt-codegen/src/codegen/winrt/python/generator/types.rs index 45f407c2..0938cfb1 100644 --- a/tools/dynwinrt-codegen/src/codegen/winrt/python/generator/types.rs +++ b/tools/dynwinrt-codegen/src/codegen/winrt/python/generator/types.rs @@ -7,8 +7,9 @@ use super::imports::{emit_type_checking_imports, format_py_type_import}; use super::structs::{generate_struct_helpers, generate_struct_imports}; use super::*; use crate::codegen::winrt::python::collections::{ - CollectionKind, interface_kind, map_iterable_identity, observable_collection_identity, - observable_map_identity, observable_vector_identity, runtime_mixin, + CollectionKind, interface_kind, map_iterable_identity, non_null_json_collection, + observable_collection_identity, observable_map_identity, observable_vector_identity, + runtime_mixin, }; use crate::codegen::winrt::python::member_plan::{PlannedMember, interface_member_plan}; use crate::meta::CollectionInputRole; @@ -309,6 +310,14 @@ pub fn generate_interface(context: &PythonProjectionContext, iface: &InterfaceMe "_obj", ); } + if let Some(contract) = + collection_kind.and_then(|kind| non_null_json_collection(kind, &iface.generic_args)) + { + out.push_str(&format!( + " _dynwinrt_non_null_collection_contract = (WinGUID.parse('{}'), '{}')\n", + contract.class_iid, contract.class_name + )); + } out.push_str(" def __new__(cls, *args, **kwargs):\n"); out.push_str( " if len(args) == 1 and not kwargs and isinstance(args[0], DynWinRTValue):\n\ diff --git a/tools/dynwinrt-codegen/src/codegen/winrt/python/method.rs b/tools/dynwinrt-codegen/src/codegen/winrt/python/method.rs index 0250b173..e05a9d7c 100644 --- a/tools/dynwinrt-codegen/src/codegen/winrt/python/method.rs +++ b/tools/dynwinrt-codegen/src/codegen/winrt/python/method.rs @@ -9,6 +9,7 @@ use crate::codegen::winrt::shared::imports::{ fill_array_output_index, fill_array_uses_retval_count, get_in_params, }; +use super::collections::non_null_json_input; use super::delegates::{ py_delegate_input_arg, py_event_handler_arg, py_once_callback_check, py_runtime_delegate_callable_type, py_runtime_delegate_param_type, @@ -56,7 +57,11 @@ pub(crate) fn py_wrap_method_arg( py_wrap_arg(name, typ, context) } -fn py_build_method_args_expr(method: &MethodMeta, context: &PythonProjectionContext) -> String { +fn py_build_method_args_expr( + method: &MethodMeta, + context: &PythonProjectionContext, + receiver: Option<&str>, +) -> String { method .params .iter() @@ -69,13 +74,23 @@ fn py_build_method_args_expr(method: &MethodMeta, context: &PythonProjectionCont }) .map(|(index, param)| { let name = to_snake_case(¶m.name); - match method + let role = method .collection_inputs .iter() - .find_map(|(parameter, role)| (*parameter == index).then_some(*role)) - { + .find_map(|(parameter, role)| (*parameter == index).then_some(*role)); + let wrapped = match role { Some(role) => py_wrap_collection_input(&name, ¶m.typ, role, context), None => py_wrap_method_arg(&name, ¶m.typ, context), + }; + if let (Some(receiver), Some(role)) = (receiver, role) + && let Some(contract) = non_null_json_input(role, ¶m.typ) + { + format!( + "{receiver}._validate_non_null_collection_input({wrapped}, WinGUID.parse('{}'), '{}')", + contract.class_iid, contract.class_name + ) + } else { + wrapped } }) .collect::>() @@ -445,7 +460,7 @@ fn generate_factory_method_invoke_named( } out.push_str(&method_pydoc(method, &in_params)); - let args_expr = py_build_method_args_expr(method, context); + let args_expr = py_build_method_args_expr(method, context, None); let iface_symbol = context.reference_name(&iface.type_identity()); let call_expr = method_call_expr( &context.registration_symbol(iface), @@ -545,7 +560,7 @@ fn generate_static_method_invoke_named( )); } out.push_str(&method_pydoc(method, &in_params)); - let args_expr = py_build_method_args_expr(method, context); + let args_expr = py_build_method_args_expr(method, context, None); let call_expr = method_call_expr( &context.registration_symbol(iface), method, @@ -952,7 +967,7 @@ pub(crate) fn generate_method_body( )); out.push_str(&method_pydoc(method, &in_params)); - let args_expr = py_build_method_args_expr(method, context); + let args_expr = py_build_method_args_expr(method, context, Some(obj_expr)); let call_expr = method_call_expr(iface_var, method, obj_expr, &args_expr, context); emit_method_result(&mut out, &call_expr, method, context); } diff --git a/tools/dynwinrt-codegen/src/codegen/winrt/python/stub_helpers.rs b/tools/dynwinrt-codegen/src/codegen/winrt/python/stub_helpers.rs index 793afadd..5f9d7190 100644 --- a/tools/dynwinrt-codegen/src/codegen/winrt/python/stub_helpers.rs +++ b/tools/dynwinrt-codegen/src/codegen/winrt/python/stub_helpers.rs @@ -4,7 +4,7 @@ //! Rendering helpers for Python type stubs. use crate::codegen::winrt::shared::imports::get_in_params; -use crate::meta::MethodMeta; +use crate::meta::{CollectionInputRole, MethodMeta}; use crate::types::{FieldMeta, TypeMeta}; use super::delegates::{py_delegate_callable_type, py_delegate_param_type}; @@ -14,7 +14,8 @@ use super::nullability::{AnnotationSurface, ElementContainer}; use super::structs::{py_struct_field_read_type, py_struct_field_type}; use super::type_helpers::{ method_pydoc_with_indent, py_collection_item_type, py_factory_return_type, - py_method_param_list, py_method_return_type, py_param_type_safe, py_property_type, + py_method_param_list, py_method_param_list_for_receiver, py_method_return_type, + py_param_type_safe, py_property_type, }; use crate::codegen::winrt::shared::imports::ireference_inner_type; @@ -185,6 +186,7 @@ pub(super) fn emit_method_stub( event_has_remove: bool, property_has_getter: bool, overrides_mutable_sequence: bool, + stock_json_receiver: bool, ) -> String { emit_method_stub_named( method, @@ -194,6 +196,7 @@ pub(super) fn emit_method_stub( event_has_remove, property_has_getter, overrides_mutable_sequence, + stock_json_receiver, ) } @@ -215,6 +218,7 @@ pub(super) fn emit_method_stub_named( event_has_remove: bool, property_has_getter: bool, overrides_mutable_sequence: bool, + stock_json_receiver: bool, ) -> String { let indent = " ".repeat(indent_spaces); let in_params = get_in_params(method); @@ -324,8 +328,22 @@ pub(super) fn emit_method_stub_named( ); } } else { - let py_params = py_method_param_list(method, context); - let py_return = py_method_return_type(method, AnnotationSurface::Stub, context); + let py_params = py_method_param_list_for_receiver(method, context, stock_json_receiver); + let py_return = match (stock_json_receiver, method.raw_name.as_str(), return_type) { + (true, "GetAt", Some(typ)) + if super::collections::non_null_json_input(CollectionInputRole::Element, typ) + .is_some() => + { + py_param_type_safe(typ, context) + } + (true, "Lookup", Some(typ)) + if super::collections::non_null_json_input(CollectionInputRole::Value, typ) + .is_some() => + { + py_param_type_safe(typ, context) + } + _ => py_method_return_type(method, AnnotationSurface::Stub, context), + }; let method_name = name_override .map(str::to_string) .unwrap_or_else(|| to_snake_case(&method.name)); @@ -340,6 +358,7 @@ pub(super) fn emit_method_stub_named( // them. Empty structural protocols can make mypy consider the // override compatible. let override_ignore = if overrides_mutable_sequence + && !stock_json_receiver && method_name == "append" && in_params.first().is_some_and(|param| { py_param_type_safe(¶m.typ, context) @@ -463,6 +482,7 @@ mod tests { true, true, false, + false, ); assert!(code.contains("def on_changed(")); assert!(code.contains("-> 'DynWinRTValue': ...")); @@ -479,6 +499,7 @@ mod tests { false, true, false, + false, ); assert!(code.contains("def on_changed(")); assert!(!code.contains("subscribe_changed")); @@ -504,7 +525,15 @@ mod tests { }; let context = PythonProjectionContext::standalone([reference_type.type_identity()]).unwrap(); - let reference = emit_method_stub(&append(reference_type), &context, 4, false, true, true); + let reference = emit_method_stub( + &append(reference_type), + &context, + 4, + false, + true, + true, + false, + ); let scalar = emit_method_stub( &append(TypeMeta::I32), &PythonProjectionContext::default(), @@ -512,6 +541,7 @@ mod tests { false, true, true, + false, ); assert!(reference.contains("type: ignore[override, unused-ignore]")); diff --git a/tools/dynwinrt-codegen/src/codegen/winrt/python/stubs.rs b/tools/dynwinrt-codegen/src/codegen/winrt/python/stubs.rs index 55984c62..95524302 100644 --- a/tools/dynwinrt-codegen/src/codegen/winrt/python/stubs.rs +++ b/tools/dynwinrt-codegen/src/codegen/winrt/python/stubs.rs @@ -26,7 +26,7 @@ use crate::codegen::winrt::shared::structs::{ use super::collections::{ CollectionKind, class_interface, interface_kind, observable_vector_identity, - projected_interface_kind, + projected_interface_kind, stock_json_class_contract, }; use super::member_plan::{ ClassMemberPlan, MethodGroup, PlannedMember, ScopePlan, class_instance_interfaces, @@ -488,7 +488,7 @@ pub fn generate_interface_stub(context: &PythonProjectionContext, iface: &Interf if !is_protocol { out.push_str(" def __init__(self, obj: DynWinRTValue) -> None: ...\n"); } - out.push_str(&collection_protocol_stubs(iface, context, 4)); + out.push_str(&collection_protocol_stubs(iface, context, 4, false)); if has_projection { out.push('\n'); if !is_protocol { @@ -595,6 +595,7 @@ pub fn generate_interface_stub(context: &PythonProjectionContext, iface: &Interf event_has_remove, property_has_getter, collection_kind == Some(CollectionKind::MutableSequence), + false, ) } PlannedMember::Group(group) => emit_instance_stub_group( @@ -602,6 +603,7 @@ pub fn generate_interface_stub(context: &PythonProjectionContext, iface: &Interf context, 4, collection_kind == Some(CollectionKind::MutableSequence), + false, ), }); } @@ -610,6 +612,7 @@ pub fn generate_interface_stub(context: &PythonProjectionContext, iface: &Interf context, 4, collection_kind == Some(CollectionKind::MutableSequence), + false, )); out @@ -626,6 +629,7 @@ pub fn generate_class_stub<'a>( let context = context.as_ref(); let collection_iface = class_interface(class); let collection_kind = collection_iface.and_then(interface_kind); + let stock_json_receiver = stock_json_class_contract(class).is_some(); let known_full_names = context.known_full_names(); let winui_bootstrap = winui::resolve_application_bootstrap(class, &known_full_names); let has_public_composition = class @@ -841,18 +845,35 @@ pub fn generate_class_stub<'a>( let collection_base = collection_iface .zip(collection_kind) .and_then(|(iface, kind)| { - super::type_helpers::py_collection_base_type( + let base = super::type_helpers::py_collection_base_type( kind, &iface.generic_args, AnnotationSurface::Stub, context, - ) + ); + if stock_json_receiver { + match (kind, iface.generic_args.as_slice()) { + (CollectionKind::MutableSequence, [value]) => Some(format!( + "MutableSequence[{}]", + super::type_helpers::py_param_type_safe(value, context) + )), + (CollectionKind::MutableMapping, [key, value]) => Some(format!( + "MutableMapping[{}, {}]", + super::type_helpers::py_param_type_safe(key, context), + super::type_helpers::py_param_type_safe(value, context) + )), + _ => base, + } + } else { + base + } }); let mut instance_stub_body = emit_class_instance_stubs( class, context, &plan.instance, collection_iface, + stock_json_receiver, false, has_closable, ); @@ -928,6 +949,7 @@ pub fn generate_class_stub<'a>( context, &plan.instance, collection_iface, + stock_json_receiver, collection_kind == Some(CollectionKind::MutableSequence), has_closable, )); @@ -1051,7 +1073,7 @@ pub fn generate_class_stub<'a>( } out.push_str(" def __init__(self, obj: DynWinRTValue) -> None: ...\n"); out.push_str(NATIVE_OBJECT_STUB); - out.push_str(&collection_protocol_stubs(req_iface, context, 4)); + out.push_str(&collection_protocol_stubs(req_iface, context, 4, false)); out.push('\n'); out.push_str(" @classmethod\n"); out.push_str(" def from_value(cls, obj: DynWinRTValue) -> Self: ...\n"); @@ -1086,6 +1108,7 @@ pub fn generate_class_stub<'a>( event_has_remove, property_has_getter, interface_kind(req_iface) == Some(CollectionKind::MutableSequence), + false, ) } PlannedMember::Group(group) => emit_instance_stub_group( @@ -1093,6 +1116,7 @@ pub fn generate_class_stub<'a>( context, 4, interface_kind(req_iface) == Some(CollectionKind::MutableSequence), + false, ), }); } @@ -1101,6 +1125,7 @@ pub fn generate_class_stub<'a>( context, 4, interface_kind(req_iface) == Some(CollectionKind::MutableSequence), + false, )); } @@ -1113,6 +1138,7 @@ fn emit_class_instance_stubs<'a>( context: &PythonProjectionContext, plan: &ScopePlan<'a>, collection_iface: Option<&InterfaceMeta>, + stock_json_receiver: bool, mutable_sequence_override: bool, has_closable: bool, ) -> String { @@ -1121,7 +1147,12 @@ fn emit_class_instance_stubs<'a>( out.push_str(NATIVE_OBJECT_STUB); } if let Some(collection_iface) = collection_iface { - out.push_str(&collection_protocol_stubs(collection_iface, context, 4)); + out.push_str(&collection_protocol_stubs( + collection_iface, + context, + 4, + stock_json_receiver, + )); } let instance_ifaces = class_instance_interfaces(class).collect::>(); @@ -1207,11 +1238,16 @@ fn emit_class_instance_stubs<'a>( event_has_remove, property_has_getter, mutable_sequence_override, + stock_json_receiver, ) } - PlannedMember::Group(group) => { - emit_instance_stub_group(group, context, 4, mutable_sequence_override) - } + PlannedMember::Group(group) => emit_instance_stub_group( + group, + context, + 4, + mutable_sequence_override, + stock_json_receiver, + ), }); } out.push_str(&emit_instance_compatibility_alias_stubs( @@ -1219,6 +1255,7 @@ fn emit_class_instance_stubs<'a>( context, 4, mutable_sequence_override, + stock_json_receiver, )); if has_closable { out.push('\n'); @@ -1268,6 +1305,7 @@ fn collection_protocol_stubs( iface: &InterfaceMeta, context: &PythonProjectionContext, indent_spaces: usize, + stock_json_receiver: bool, ) -> String { let Some(kind) = projected_interface_kind(iface) else { return String::new(); @@ -1279,7 +1317,9 @@ fn collection_protocol_stubs( .generic_args .first() .map(|typ| { - if matches!( + if stock_json_receiver && kind == super::collections::CollectionKind::MutableSequence { + super::type_helpers::py_param_type_safe(typ, context) + } else if matches!( kind, super::collections::CollectionKind::Mapping | super::collections::CollectionKind::MutableMapping @@ -1298,7 +1338,13 @@ fn collection_protocol_stubs( let item_input = iface .generic_args .first() - .map(|typ| super::type_helpers::py_collection_input_type(typ, context)) + .map(|typ| { + if stock_json_receiver && kind == super::collections::CollectionKind::MutableSequence { + super::type_helpers::py_param_type_safe(typ, context) + } else { + super::type_helpers::py_collection_input_type(typ, context) + } + }) .unwrap_or_else(|| "object".to_string()); match kind { super::collections::CollectionKind::Iterable => { @@ -1327,6 +1373,12 @@ fn collection_protocol_stubs( {indent}def __delitem__(self, index: int | slice) -> None: ...\n\ {indent}def insert(self, index: int, value: {item_input}) -> None: ...\n" )); + if stock_json_receiver { + result.push_str(&format!( + "{indent}def extend(self, values: Iterable[{item_input}]) -> None: ...\n\ + {indent}def __iadd__(self, values: Iterable[{item_input}]) -> Self: ...\n" + )); + } } result } @@ -1337,12 +1389,16 @@ fn collection_protocol_stubs( .generic_args .get(1) .map(|typ| { - super::type_helpers::py_collection_item_type( - typ, - container, - AnnotationSurface::Stub, - context, - ) + if stock_json_receiver { + super::type_helpers::py_param_type_safe(typ, context) + } else { + super::type_helpers::py_collection_item_type( + typ, + container, + AnnotationSurface::Stub, + context, + ) + } }) .unwrap_or_else(|| "object".to_string()); let mut result = format!( @@ -1354,12 +1410,23 @@ fn collection_protocol_stubs( let value_input = iface .generic_args .get(1) - .map(|typ| super::type_helpers::py_collection_input_type(typ, context)) + .map(|typ| { + if stock_json_receiver { + super::type_helpers::py_param_type_safe(typ, context) + } else { + super::type_helpers::py_collection_input_type(typ, context) + } + }) .unwrap_or_else(|| "object".to_string()); result.push_str(&format!( "{indent}def __setitem__(self, key: {item_input}, value: {value_input}) -> None: ...\n\ {indent}def __delitem__(self, key: {item_input}) -> None: ...\n" )); + if stock_json_receiver { + result.push_str(&format!( + "{indent}def setdefault(self, key: {item_input}, default: {value_input}) -> {value_type}: ...\n" + )); + } } result } @@ -1584,6 +1651,7 @@ fn emit_instance_stub_group( context: &PythonProjectionContext, indent_spaces: usize, overrides_mutable_sequence: bool, + stock_json_receiver: bool, ) -> String { let indent = " ".repeat(indent_spaces); let methods = typed_signatures( @@ -1605,6 +1673,7 @@ fn emit_instance_stub_group( false, true, overrides_mutable_sequence, + stock_json_receiver, ); if duplicate { stub = ignore_unreachable_overload(stub); @@ -1623,6 +1692,7 @@ fn emit_instance_compatibility_alias_stubs( context: &PythonProjectionContext, indent_spaces: usize, overrides_mutable_sequence: bool, + stock_json_receiver: bool, ) -> String { let indent = " ".repeat(indent_spaces); let mut out = String::new(); @@ -1646,6 +1716,7 @@ fn emit_instance_compatibility_alias_stubs( false, true, overrides_mutable_sequence, + stock_json_receiver, ); if *duplicate { stub = ignore_unreachable_overload(stub); diff --git a/tools/dynwinrt-codegen/src/codegen/winrt/python/type_helpers.rs b/tools/dynwinrt-codegen/src/codegen/winrt/python/type_helpers.rs index bac53d56..f607aa11 100644 --- a/tools/dynwinrt-codegen/src/codegen/winrt/python/type_helpers.rs +++ b/tools/dynwinrt-codegen/src/codegen/winrt/python/type_helpers.rs @@ -754,6 +754,14 @@ fn py_collection_param_type(typ: &TypeMeta, context: &PythonProjectionContext) - pub(super) fn py_method_param_list( method: &MethodMeta, context: &PythonProjectionContext, +) -> String { + py_method_param_list_for_receiver(method, context, false) +} + +pub(super) fn py_method_param_list_for_receiver( + method: &MethodMeta, + context: &PythonProjectionContext, + stock_json_receiver: bool, ) -> String { method .params @@ -771,6 +779,19 @@ pub(super) fn py_method_param_list( .iter() .find_map(|(parameter, role)| (*parameter == index).then_some(*role)); let param_type = match role { + Some(role @ (CollectionInputRole::Element | CollectionInputRole::Value)) + if stock_json_receiver + && super::collections::non_null_json_input(role, ¶m.typ).is_some() => + { + if let TypeMeta::Array(element) = ¶m.typ { + format!( + "DynWinRTArray | Sequence[{}]", + py_param_type_safe(element, context) + ) + } else { + py_param_type_safe(¶m.typ, context) + } + } Some(CollectionInputRole::Element | CollectionInputRole::Value) => { py_collection_contract_input_type(¶m.typ, context) } diff --git a/tools/dynwinrt-codegen/src/meta.rs b/tools/dynwinrt-codegen/src/meta.rs index 126f5039..bb4706f5 100644 --- a/tools/dynwinrt-codegen/src/meta.rs +++ b/tools/dynwinrt-codegen/src/meta.rs @@ -123,6 +123,9 @@ fn collection_input_roles( definition: &str, member: &str, ) -> Vec<(usize, CollectionInputRole)> { + if (namespace, definition, member) == ("Windows.Data.Json", "IJsonObject", "SetNamedValue") { + return vec![(1, CollectionInputRole::Value)]; + } if namespace != WINDOWS_FOUNDATION_COLLECTIONS_NAMESPACE { return Vec::new(); } diff --git a/tools/dynwinrt-codegen/tests/python_consumer_typing_test.rs b/tools/dynwinrt-codegen/tests/python_consumer_typing_test.rs index 837e732c..de6270cb 100644 --- a/tools/dynwinrt-codegen/tests/python_consumer_typing_test.rs +++ b/tools/dynwinrt-codegen/tests/python_consumer_typing_test.rs @@ -1425,8 +1425,8 @@ def json_demo() -> list[str]: parsed = JsonObject.parse('{{"tags": ["a", "b"]}}') assert_type(JsonObject.try_parse("{{}}"), tuple[JsonObject | None, bool]) tags = parsed.get_named_array("tags") - assert_type(tags[0], IJsonValue | None) - return [value.get_string() for value in tags if value is not None] + assert_type(tags[0], IJsonValue) + return [value.get_string() for value in tags] def sensor_demo() -> float | None: accelerometer = Accelerometer.get_default() @@ -1631,15 +1631,14 @@ fn mutable_collection_mutators_accept_none() { .unwrap(); assert!(pair_stub.contains("def key(self) -> DynWinRTValue | None: ...")); assert!(pair_stub.contains("def value(self) -> DynWinRTValue | None: ...")); - // Inherited MutableSequence and MutableMapping mutators take the element - // type of the collection base, which keeps `| None` for mutable - // collections, like the generated item setters. + // Generic mutable collections accept native null. The stock JsonObject + // has a validated, non-null JSON value contract instead. typecheck( &fixture, &["sdk"], r#"from typing import assert_type from dynwinrt import DynWinRTValue -from sdk.windows.data.json import IJsonValue, JsonObject +from sdk.windows.data.json import IJsonValue, JsonObject, JsonValue from sdk.windows.foundation import IStringable, Uri from sdk.windows.foundation.collections import ( IMap_Object_Object, @@ -1660,11 +1659,11 @@ def vector(folders: IObservableVector_StorageFolder) -> None: folders[0] = None assert_type(folders[0], StorageFolder | None) -def mapping(values: JsonObject) -> None: - values.update({"k": None}) - values.setdefault("k", None) - values["k"] = None - assert_type(values["k"], IJsonValue | None) +def mapping(values: JsonObject, json_null: JsonValue) -> None: + values.update({"k": json_null}) + values.setdefault("k", json_null) + values["k"] = json_null + assert_type(values["k"], IJsonValue) def object_values(values: IMap_String_Object, uri: Uri) -> None: values["none"] = None diff --git a/tools/dynwinrt-codegen/tests/python_json_native_null_test.rs b/tools/dynwinrt-codegen/tests/python_json_native_null_test.rs new file mode 100644 index 00000000..a1d493eb --- /dev/null +++ b/tools/dynwinrt-codegen/tests/python_json_native_null_test.rs @@ -0,0 +1,437 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; +use std::sync::atomic::{AtomicU64, Ordering}; + +const WINDOWS_WINMD: &str = + r"C:\Program Files (x86)\Windows Kits\10\UnionMetadata\10.0.26100.0\Windows.winmd"; +const JSON_CLASSES: &str = + "Windows.Data.Json.JsonArray,Windows.Data.Json.JsonObject,Windows.Data.Json.JsonValue"; +static NEXT: AtomicU64 = AtomicU64::new(0); + +fn repo_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .to_path_buf() +} + +fn python() -> PathBuf { + std::env::var_os("DYNWINRT_TEST_PYTHON") + .map(PathBuf::from) + .unwrap_or_else(|| { + let venv = repo_root().join(r"bindings\py\.venv\Scripts\python.exe"); + if venv.is_file() { + venv + } else { + PathBuf::from("python") + } + }) +} + +struct Generated { + root: PathBuf, + package: String, +} + +impl Generated { + fn new() -> Option { + if !Path::new(WINDOWS_WINMD).is_file() { + eprintln!("Skipping JSON SDK regression: Windows.winmd not found"); + return None; + } + let package = format!( + "json_native_null_{}_{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + ); + let root = repo_root().join("target").join(&package); + let output = Command::new(env!("CARGO_BIN_EXE_dynwinrt-codegen")) + .args([ + "generate", + "--winmd", + WINDOWS_WINMD, + "--class-name", + JSON_CLASSES, + "--lang", + "py", + "--output", + ]) + .arg(&root) + .output() + .expect("generate stock JSON bindings"); + assert_success(output); + Some(Self { root, package }) + } + + fn module(&self, name: &str) -> String { + fs::read_to_string(self.root.join(name)).expect(name) + } + + fn python(&self, script: &str) -> Output { + Command::new(python()) + .args(["-B", "-c", &script.replace("JSON_PACKAGE", &self.package)]) + .env("PYTHONPATH", self.root.parent().unwrap()) + .output() + .expect("execute isolated Python consumer") + } + + fn typing_environment(&self, command: &mut Command) { + command + .current_dir(self.root.parent().unwrap()) + .env( + "MYPYPATH", + std::env::join_paths([ + repo_root().join(r"bindings\py"), + self.root.parent().unwrap().to_path_buf(), + ]) + .expect("MYPYPATH"), + ) + .env("PYTHONPATH", self.root.parent().unwrap()); + } +} + +impl Drop for Generated { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.root); + } +} + +fn assert_success(output: Output) { + assert!( + output.status.success(), + "exit {:?}\n{}\n{}", + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn stock_json_generation_preserves_receiver_dependent_contract() { + let Some(generated) = Generated::new() else { + return; + }; + let array = generated.module("windows__data__json__json_array.py"); + let object = generated.module("windows__data__json__json_object.py"); + let array_stub = generated.module("windows__data__json__json_array.pyi"); + let object_stub = generated.module("windows__data__json__json_object.pyi"); + + for code in [&array, &object] { + assert!( + code.contains("_dynwinrt_non_null_collection_contract ="), + "{code}" + ); + assert!( + code.contains("if not obj._matches_runtime_class("), + "{code}" + ); + assert!( + code.contains("._validate_non_null_collection_input("), + "{code}" + ); + } + assert!( + array.contains("def replace_all(") + && array.contains( + "self._collection_obj._validate_non_null_collection_input(_dynwinrt_array(" + ) + ); + assert!( + object.contains("def set_named_value(") + && object.contains("self._obj._validate_non_null_collection_input(") + ); + assert!( + array_stub.contains("class JsonArray(_JsonArrayIdentity, MutableSequence['IJsonValue']") + && array_stub.contains("def get_at(self, index: int) -> 'IJsonValue': ...") + && array_stub + .contains("def replace_all(self, items: DynWinRTArray | Sequence['IJsonValue'])") + && array_stub.contains("class IVector_IJsonValue(MutableSequence[IJsonValue | None])"), + "{array_stub}" + ); + assert!( + object_stub + .contains("class JsonObject(_JsonObjectIdentity, MutableMapping[str, 'IJsonValue']") + && object_stub.contains("def lookup(self, key: str) -> 'IJsonValue': ...") + && object_stub.contains("def insert(self, key: str, value: 'IJsonValue')") + && object_stub + .contains("class IMap_String_IJsonValue(MutableMapping[str, IJsonValue | None])"), + "{object_stub}" + ); +} + +#[test] +fn stock_json_mutators_fail_before_native_mutation_but_custom_generics_keep_null() { + let Some(generated) = Generated::new() else { + return; + }; + let available = Command::new(python()) + .args([ + "-c", + "from dynwinrt import DynWinRTInterfacePlan, DynWinRTValue; assert hasattr(DynWinRTValue, '_validate_non_null_collection_input')", + ]) + .output() + .is_ok_and(|output| output.status.success()); + assert!( + available || std::env::var("DYNWINRT_REQUIRE_IMPLEMENTATION_RUNTIME").as_deref() != Ok("1"), + "the JSON native regression requires the matching Python binding" + ); + if !available { + eprintln!("Skipping JSON native regression: matching Python binding not installed"); + return; + } + let script = r#" +import operator +from dynwinrt import DynWinRTArray, DynWinRTType, DynWinRTValue, RoApartment, release_projected +from JSON_PACKAGE.windows__data__json__json_array import JsonArray, IVector_IJsonValue, IID_IJsonValue +from JSON_PACKAGE.windows__data__json__json_object import JsonObject, IMap_String_IJsonValue +from JSON_PACKAGE.windows__data__json__json_value import JsonValue + +def rejected_without_mutation(receiver, mutation): + before = receiver.stringify() + try: + mutation() + except TypeError as error: + assert ('requires a non-null IJsonValue' in str(error) + or 'map key cannot be None' in str(error)), error + else: + raise AssertionError('native null was accepted by a stock JSON collection') + assert receiver.stringify() == before, (before, receiver.stringify()) + +with RoApartment(): + element = DynWinRTType.interface(IID_IJsonValue) + native_null = DynWinRTValue.null_value() + array_of_null = DynWinRTArray.from_values([native_null], element) + json_null = JsonValue.create_null_value() + assert not json_null._obj.is_null() + assert json_null.stringify() == 'null' + + array = JsonArray.parse('[1]') + view = array.as_interface(IVector_IJsonValue) + for mutation in ( + lambda: array.append(None), + lambda: array.append(native_null), + lambda: array.insert(0, None), + lambda: array.insert_at(0, None), + lambda: array.set_at(0, None), + lambda: operator.setitem(array, 0, None), + lambda: operator.setitem(array, slice(None), [json_null, None]), + lambda: array.replace_all([None]), + lambda: array.replace_all([json_null, None]), + lambda: array.replace_all(array_of_null), + lambda: array.replace_all(array_of_null.to_value()), + lambda: array.extend([json_null, None]), + lambda: operator.iadd(array, [json_null, None]), + lambda: view.append(None), + lambda: view.set_at(0, None), + lambda: view.replace_all(array_of_null), + lambda: view.extend([json_null, None]), + ): + rejected_without_mutation(array, mutation) + + obj = JsonObject.parse('{"base":1}') + map_view = obj.as_interface(IMap_String_IJsonValue) + for mutation in ( + lambda: obj.insert('bad', None), + lambda: operator.setitem(obj, 'bad', native_null), + lambda: obj.set_named_value('bad', native_null), + lambda: obj.update({'good': json_null, 'bad': None}), + lambda: obj.update([('good', json_null), ('bad', native_null)]), + lambda: obj.update([('good', json_null), (None, json_null)]), + lambda: obj.setdefault('bad'), + lambda: map_view.insert('bad', None), + lambda: operator.setitem(map_view, 'bad', None), + lambda: map_view.update({'good': json_null, 'bad': None}), + ): + rejected_without_mutation(obj, mutation) + + array.append(json_null) + obj['valid'] = json_null + assert array[-1].stringify() == 'null' + assert obj['valid'].stringify() == 'null' + activated_array = JsonArray.create() + activated_object = JsonObject.create() + activated_array.append(json_null) + activated_object['valid'] = json_null + assert activated_array[0].stringify() == activated_object['valid'].stringify() == 'null' + + generic_vector = IVector_IJsonValue.from_value( + DynWinRTValue.create_vector([native_null], element) + ) + assert generic_vector[0] is None + generic_vector.append(None) + generic_vector.replace_all(array_of_null) + generic_vector.extend([None]) + assert list(generic_vector) == [None, None] + try: + JsonArray(generic_vector._obj) + except TypeError as error: + assert 'Expected a native Windows.Data.Json.JsonArray' in str(error) + else: + raise AssertionError('custom vector projected as stock JsonArray') + + generic_map = IMap_String_IJsonValue.from_value( + DynWinRTValue.create_map( + [DynWinRTValue.from_hstring('original')], + [native_null], + DynWinRTType.hstring(), + element, + ) + ) + generic_map.update({'next': None}) + assert generic_map['original'] is None and generic_map['next'] is None + try: + JsonObject(generic_map._obj) + except TypeError as error: + assert 'Expected a native Windows.Data.Json.JsonObject' in str(error) + else: + raise AssertionError('custom map projected as stock JsonObject') + + for wrapper in ( + generic_map, generic_vector, activated_object, activated_array, + map_view, obj, view, array, json_null, + ): + release_projected(wrapper) + native_null.release() +"#; + assert_success(generated.python(script)); +} + +const VALID_CONSUMER: &str = r#" +from JSON_PACKAGE.windows.data.json import JsonArray, JsonObject, JsonValue +from JSON_PACKAGE.windows__data__json__json_array import IVector_IJsonValue +from JSON_PACKAGE.windows__data__json__json_object import IMap_String_IJsonValue + +array: JsonArray = JsonArray.parse('[]') +obj: JsonObject = JsonObject.parse('{}') +value: JsonValue = JsonValue.create_null_value() +array.append(value) +array.extend([value]) +array[0] = value +array[:] = [value] +array.replace_all([value]) +obj['valid'] = value +obj.update({'valid': value}) +obj.setdefault('valid', value) +vector: IVector_IJsonValue = array.as_interface(IVector_IJsonValue) +vector.append(None) +map_view: IMap_String_IJsonValue = obj.as_interface(IMap_String_IJsonValue) +map_view.insert('native-null', None) +"#; + +const INVALID_CONSUMER: &str = r#" +from JSON_PACKAGE.windows.data.json import JsonArray, JsonObject, JsonValue +array: JsonArray = JsonArray.parse('[]') +obj: JsonObject = JsonObject.parse('{}') +value: JsonValue = JsonValue.create_null_value() +array.append(None) +array.insert(0, None) +array.replace_all([None]) +array[:] = [None] +array.extend([value, None]) +obj.insert('bad', None) +obj['bad'] = None +obj.update({'good': value, 'bad': None}) +obj.setdefault('bad') +"#; + +#[test] +fn stock_json_stubs_reject_null_with_strict_typecheckers() { + let Some(generated) = Generated::new() else { + return; + }; + let available = Command::new(python()) + .args(["-m", "mypy", "--version"]) + .output() + .is_ok_and(|output| output.status.success()); + assert!( + available || std::env::var("DYNWINRT_REQUIRE_MYPY").as_deref() != Ok("1"), + "strict JSON typing test requires mypy" + ); + if !available { + eprintln!("Skipping JSON typing test: mypy not installed"); + return; + } + for (source, expected_errors) in [(VALID_CONSUMER, 0), (INVALID_CONSUMER, 9)] { + let mut command = Command::new(python()); + command.args([ + "-m", + "mypy", + "--strict", + "--no-incremental", + "--no-pretty", + "--cache-dir", + ]); + command.arg(generated.root.join("mypy-cache")); + command.args(["-c", &source.replace("JSON_PACKAGE", &generated.package)]); + generated.typing_environment(&mut command); + let output = command.output().expect("run mypy"); + let diagnostics = format!( + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + diagnostics.matches(": error:").count(), + expected_errors, + "{diagnostics}" + ); + assert_eq!( + output.status.success(), + expected_errors == 0, + "{diagnostics}" + ); + } + + if let Some(pyright) = std::env::var_os("DYNWINRT_PYRIGHT") { + for (file, source, expected_errors) in [ + ("valid.py", VALID_CONSUMER, 0), + ("invalid.py", INVALID_CONSUMER, 9), + ] { + let path = generated.root.join(file); + fs::write( + &path, + format!( + "# pyright: strict\n{}", + source.replace("JSON_PACKAGE", &generated.package) + ), + ) + .unwrap(); + let mut command = Command::new(&pyright); + command.args(["--pythonpath"]).arg(python()).arg(&path); + generated.typing_environment(&mut command); + let output = command.output().expect("run pyright"); + let diagnostics = format!( + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let error_lines = diagnostics + .lines() + .filter(|line| line.contains(" - error: ")) + .collect::>(); + if expected_errors == 0 { + assert!(error_lines.is_empty(), "{diagnostics}"); + } else { + let mut locations = error_lines + .iter() + .filter_map(|line| line.split("invalid.py:").nth(1)) + .filter_map(|location| location.split(':').next()?.parse::().ok()) + .collect::>(); + locations.sort_unstable(); + locations.dedup(); + assert_eq!(locations, (7..=15).collect::>(), "{diagnostics}"); + } + assert_eq!( + output.status.success(), + expected_errors == 0, + "{diagnostics}" + ); + } + } +} diff --git a/tools/dynwinrt-codegen/tests/python_stub_nullability_test.rs b/tools/dynwinrt-codegen/tests/python_stub_nullability_test.rs index e095a9e8..b8ec9b5b 100644 --- a/tools/dynwinrt-codegen/tests/python_stub_nullability_test.rs +++ b/tools/dynwinrt-codegen/tests/python_stub_nullability_test.rs @@ -443,19 +443,20 @@ fn reference_collection_elements_are_nullable_regardless_of_provenance() { "def lookup(self, key: str) -> NamedResource | None: ...", ); - // Mutable collection elements, item positions and element-reading - // members keep None for the same reason. + // The stock JSON runtime classes have a validated non-null element + // contract; a generic collection interface can still expose native null. assert_contains( &array, - "class JsonArray(_JsonArrayIdentity, MutableSequence[IJsonValue | None], _DynWinRTRuntimeClass):", + "class JsonArray(_JsonArrayIdentity, MutableSequence['IJsonValue'], _DynWinRTRuntimeClass):", ); + assert_contains(&array, "def get_at(self, index: int) -> 'IJsonValue': ..."); assert_contains( &array, - "def get_at(self, index: int) -> IJsonValue | None: ...", + "def __getitem__(self, index: int) -> 'IJsonValue': ...", ); assert_contains( &array, - "def __getitem__(self, index: int) -> IJsonValue | None: ...", + "class IVector_IJsonValue(MutableSequence[IJsonValue | None]):", ); assert_contains( &array, @@ -463,12 +464,13 @@ fn reference_collection_elements_are_nullable_regardless_of_provenance() { ); assert_contains( &object, - "class JsonObject(_JsonObjectIdentity, MutableMapping[str, IJsonValue | None], _DynWinRTRuntimeClass):", + "class JsonObject(_JsonObjectIdentity, MutableMapping[str, 'IJsonValue'], _DynWinRTRuntimeClass):", ); assert_contains( &object, - "def lookup(self, key: str) -> IJsonValue | None: ...", + "class IMap_String_IJsonValue(MutableMapping[str, IJsonValue | None]):", ); + assert_contains(&object, "def lookup(self, key: str) -> 'IJsonValue': ..."); assert_contains( &object, "def get_named_array(self, name: str) -> JsonArray: ...", From 1fbe2b81ce6e8a1e1b0ded114b5e76b207e5db19 Mon Sep 17 00:00:00 2001 From: Leilei Zhang Date: Tue, 29 Sep 2026 18:10:19 +0800 Subject: [PATCH 02/15] Track raw Python COM outputs in apartment lifetime scopes Register owned native results at Python return boundaries so a closing scope releases raw values before apartment teardown, while preserving borrowed sources, scalar results, callback thread affinity, and released-value errors. Reproduce and guard the shutdown crash in subprocesses. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- bindings/py/README.md | 19 +- bindings/py/src/async_runtime.rs | 4 +- bindings/py/src/delegate_method.rs | 13 +- bindings/py/src/implementation.rs | 6 +- bindings/py/src/lib.rs | 2 +- bindings/py/src/object_value.rs | 13 +- bindings/py/src/runtime.rs | 252 ++++++++++------ bindings/py/tests/test_phase1.py | 16 + tools/dynwinrt-codegen/python/README.md | 7 + .../tests/python_raw_lifetime_test.rs | 278 ++++++++++++++++++ 10 files changed, 503 insertions(+), 107 deletions(-) create mode 100644 tools/dynwinrt-codegen/tests/python_raw_lifetime_test.rs diff --git a/bindings/py/README.md b/bindings/py/README.md index cee268dc..99d030ac 100644 --- a/bindings/py/README.md +++ b/bindings/py/README.md @@ -741,8 +741,9 @@ stay on the caller's native thread but release the Python GIL while WinUI pumps messages. WinRT callbacks reacquire the GIL, and worker threads can use `DispatcherQueue.try_enqueue()` to return to the UI thread. -Use a projection lifetime scope inside the COM apartment so wrappers release -their native values before `RoUninitialize`: +Use a projection lifetime scope inside the COM apartment so projected wrappers +and raw native `DynWinRTValue` outputs release their owned COM references +before `RoUninitialize`: ```python from dynwinrt import RO_INIT_SINGLETHREADED, RoApartment, projected_lifetime_scope @@ -752,8 +753,18 @@ with RoApartment(RO_INIT_SINGLETHREADED), projected_lifetime_scope(): # Create and use WinUI objects here. ``` -Scopes nest in LIFO order. Wrappers that survive a closed scope remain Python -objects, but their native values are released: using one afterwards, as the +Native factory and method outputs are tracked automatically, even when a +generated factory returns a bare `DynWinRTValue` (for example, +`PropertyValue.create_uint32(8080)`). A raw result can escape the Python +function that created it, but after the scope closes it reports +`is_released() == True`; it cannot be used outside the apartment. Without a +scope, explicitly call `release()` on every retained native result before +leaving `RoApartment`. Pure scalar results do not own COM references and remain +usable after a scope closes. + +Scopes nest in LIFO order. Wrappers and raw native results that survive a +closed scope remain Python objects, but their COM references are released: +using one afterwards, as the object of a call, as an argument, or inside a sequence, mapping, array, or struct input, raises `RuntimeError` explaining that it was released, as it does after `release_projected(wrapper)` or `DynWinRTValue.release()`. diff --git a/bindings/py/src/async_runtime.rs b/bindings/py/src/async_runtime.rs index f8d2ce24..3e36c275 100644 --- a/bindings/py/src/async_runtime.rs +++ b/bindings/py/src/async_runtime.rs @@ -8,7 +8,7 @@ use std::sync::{Arc, Mutex, MutexGuard}; use crate::errors::{ map_dynwinrt_error, map_dynwinrt_error_with_context, map_windows_error_with_context, }; -use crate::runtime::DynWinRTValue; +use crate::runtime::{DynWinRTValue, tracked_native_value}; use pyo3::exceptions::{PyRuntimeError, PyTypeError}; use pyo3::prelude::*; use pyo3::types::PyList; @@ -632,7 +632,7 @@ impl AsyncOperation { *state = ExecutionState::Idle; } - let raw = Py::new(py, DynWinRTValue::new(result?))?; + let raw = tracked_native_value(py, result?)?; self.converter.call1(py, (raw,)) } diff --git a/bindings/py/src/delegate_method.rs b/bindings/py/src/delegate_method.rs index f67c107e..4817f0a5 100644 --- a/bindings/py/src/delegate_method.rs +++ b/bindings/py/src/delegate_method.rs @@ -6,7 +6,9 @@ use pyo3::prelude::*; use windows::core::{GUID, IInspectable, IUnknown, Interface}; use crate::errors::map_windows_error; -use crate::runtime::{DynWinRTMethodSig, DynWinRTValue, WinGUID, native_arguments}; +use crate::runtime::{ + DynWinRTMethodSig, DynWinRTValue, WinGUID, native_arguments, tracked_native_value, +}; type DelegateCall = dyn Fn(&IUnknown, &[dynwinrt::WinRTValue]) -> windows::core::Result>; @@ -50,8 +52,9 @@ impl DynWinRTDelegateMethod { &self, value: &Bound<'_, DynWinRTValue>, args: Vec, - ) -> PyResult> { + ) -> PyResult>> { // Keep native pins, not a Python value borrow, across reentrant Invoke. + let py = value.py(); let value = value.try_borrow()?.clone(); let delegate = value.query(&self.iid, "delegate Invoke()")?; let dynwinrt::WinRTValue::Object(object) = &delegate else { @@ -61,8 +64,10 @@ impl DynWinRTDelegateMethod { }; let args = native_arguments("delegate Invoke()", args)?; (self.call.0)(object, &args) - .map(|outputs| outputs.into_iter().map(DynWinRTValue::new).collect()) - .map_err(map_windows_error) + .map_err(map_windows_error)? + .into_iter() + .map(|result| tracked_native_value(py, result)) + .collect() } } diff --git a/bindings/py/src/implementation.rs b/bindings/py/src/implementation.rs index 1ca498c0..8805aa27 100644 --- a/bindings/py/src/implementation.rs +++ b/bindings/py/src/implementation.rs @@ -20,7 +20,7 @@ use windows::core::{Error, HRESULT}; use crate::errors::map_windows_error; use crate::runtime::{ DynWinRTMethodSig, DynWinRTType, DynWinRTValue, PYWINRT_E_UNRAISABLE_PYTHON_EXCEPTION, WinGUID, - native_outputs, wrap_python_callback_context, + native_outputs, tracked_native_value, wrap_python_callback_context, }; const RO_E_CLOSED: HRESULT = HRESULT(0x80000013_u32 as i32); @@ -345,12 +345,12 @@ impl DynWinRTImplementation { }) } - fn to_value(&self) -> PyResult { + fn to_value(&self, py: Python<'_>) -> PyResult> { self.with_native(|native| { native .to_value() - .map(DynWinRTValue::new) .map_err(map_windows_error) + .and_then(|value| tracked_native_value(py, value)) }) } diff --git a/bindings/py/src/lib.rs b/bindings/py/src/lib.rs index 54070e51..7cedae6d 100644 --- a/bindings/py/src/lib.rs +++ b/bindings/py/src/lib.rs @@ -552,7 +552,7 @@ class _WinRTMutableMappingMixin(_MutableMapping): async def _dynwinrt_convert_future(future, converter): try: - completed = await future + completed = _dynwinrt_track_projected(await future, 'WinRTAsync completion') return converter(completed._get_async_results()) except BaseException: if not future.done(): diff --git a/bindings/py/src/object_value.rs b/bindings/py/src/object_value.rs index fbcf11c3..346c0192 100644 --- a/bindings/py/src/object_value.rs +++ b/bindings/py/src/object_value.rs @@ -418,11 +418,6 @@ fn with_context(py: Python<'_>, error: PyErr, context: &str) -> PyErr { wrapped } -/// The one place that creates the Python values these conversions return. -fn native(value: WinRTValue) -> DynWinRTValue { - DynWinRTValue::new(value) -} - /// Borrow the native value of a `DynWinRTValue`. /// /// Every native input, including nested elements, is read through here. @@ -689,7 +684,11 @@ impl Reader<'_> { .map(|(index, element)| match element { None => Ok(py.None()), Some(object) => { - let element = Bound::new(py, native(WinRTValue::Object(object)))?; + let element = crate::runtime::tracked_native_value( + py, + WinRTValue::Object(object), + )? + .into_bound(py); self.unbox(&element, depth + 1, InputSlot::Element(index)) } }), @@ -804,7 +803,7 @@ pub fn to_winrt_object( }; match boxed { Boxed::Existing(object) => Ok(object.into_any().unbind()), - Boxed::New(value) => Ok(Bound::new(py, native(value))?.into_any().unbind()), + Boxed::New(value) => Ok(crate::runtime::tracked_native_value(py, value)?.into_any()), } } diff --git a/bindings/py/src/runtime.rs b/bindings/py/src/runtime.rs index 0b8526d9..eca11a90 100644 --- a/bindings/py/src/runtime.rs +++ b/bindings/py/src/runtime.rs @@ -870,17 +870,22 @@ impl DynWinRTOverrideInterface { #[pymethods] impl DynWinRTMethodHandle { /// Invoke this method on a COM object. - fn invoke(&self, obj: DynWinRTValue, args: Vec) -> PyResult { + fn invoke( + &self, + py: Python<'_>, + obj: DynWinRTValue, + args: Vec, + ) -> PyResult> { // Extraction retains the native object without holding a Python borrow // while an implementation callback may release the original wrapper. let raw = obj.receiver("invoke()")?.as_raw(); let wrt_args = native_arguments("invoke()", args)?; let results = self.0.invoke(raw, &wrt_args).map_err(map_dynwinrt_error)?; - if results.is_empty() { - Ok(DynWinRTValue::new(dynwinrt::WinRTValue::I32(0))) - } else { - Ok(DynWinRTValue::new(results.into_iter().next().unwrap())) - } + let value = results + .into_iter() + .next() + .unwrap_or(dynwinrt::WinRTValue::I32(0)); + tracked_native_value(py, value) } /// Invoke a blocking method on the current native thread while releasing @@ -890,7 +895,7 @@ impl DynWinRTMethodHandle { py: Python<'_>, obj: DynWinRTValue, args: Vec, - ) -> PyResult { + ) -> PyResult> { struct SameThreadCall { method: dynwinrt::MethodHandle, object: IUnknown, @@ -924,41 +929,41 @@ impl DynWinRTMethodHandle { .detach(move || call.run()) .0 .map_err(map_dynwinrt_error)?; - if results.is_empty() { - Ok(DynWinRTValue::new(dynwinrt::WinRTValue::I32(0))) - } else { - Ok(DynWinRTValue::new( - results - .into_iter() - .next() - .expect("non-empty result was checked"), - )) - } + let value = results + .into_iter() + .next() + .unwrap_or(dynwinrt::WinRTValue::I32(0)); + tracked_native_value(py, value) } /// Like `invoke`, but returns all out-parameters as a list. /// Used for methods with multiple out params (e.g. IVector.IndexOf → [index, found]). fn invoke_all( &self, + py: Python<'_>, obj: DynWinRTValue, args: Vec, - ) -> PyResult> { + ) -> PyResult>> { let raw = obj.receiver("invoke_all()")?.as_raw(); let wrt_args = native_arguments("invoke_all()", args)?; let results = self.0.invoke(raw, &wrt_args).map_err(map_dynwinrt_error)?; - Ok(results.into_iter().map(DynWinRTValue::new).collect()) + results + .into_iter() + .map(|value| tracked_native_value(py, value)) + .collect() } /// Invoke a WinRT composable factory with a runtime-provided outer host. fn invoke_composed( &self, + py: Python<'_>, factory: &DynWinRTValue, args: Vec, outer_index: usize, inner_output_index: usize, instance_output_index: usize, agile: bool, - ) -> PyResult { + ) -> PyResult> { let factory = factory.com_receiver("invoke_composed() factory")?; let args = native_arguments("invoke_composed()", args)?; dynwinrt::compose_winrt( @@ -970,8 +975,8 @@ impl DynWinRTMethodHandle { instance_output_index, agile, ) - .map(DynWinRTValue::new) .map_err(map_dynwinrt_error) + .and_then(|value| tracked_native_value(py, value)) } /// Invoke a composable factory with metadata-described local overrides. @@ -986,9 +991,10 @@ impl DynWinRTMethodHandle { instance_output_index: usize, agile: bool, override_interfaces: Vec>, - ) -> PyResult { + ) -> PyResult> { if override_interfaces.is_empty() { return self.invoke_composed( + py, factory, args, outer_index, @@ -1013,8 +1019,8 @@ impl DynWinRTMethodHandle { agile, overrides, ) - .map(DynWinRTValue::new) .map_err(map_dynwinrt_error) + .and_then(|value| tracked_native_value(py, value)) } // --- Fast paths: skip Vec alloc for common getter patterns --- @@ -1042,36 +1048,54 @@ impl DynWinRTMethodHandle { } /// Getter → DynWinRTValue object (0 args, zero Vec allocation) - fn get_obj(&self, obj: DynWinRTValue) -> PyResult { + fn get_obj(&self, py: Python<'_>, obj: DynWinRTValue) -> PyResult> { let raw = obj.com_receiver("get_obj()")?.as_raw(); self.0 .call_getter_object(raw) - .map(DynWinRTValue::new) .map_err(map_dynwinrt_error) + .and_then(|value| tracked_native_value(py, value)) } /// 1-arg invoke with hstring input → DynWinRTValue result - fn invoke_hstring(&self, obj: DynWinRTValue, arg: String) -> PyResult { + fn invoke_hstring( + &self, + py: Python<'_>, + obj: DynWinRTValue, + arg: String, + ) -> PyResult> { let raw = obj.com_receiver("invoke_hstring()")?.as_raw(); let results = self .0 .invoke(raw, &[dynwinrt::WinRTValue::HString(HSTRING::from(arg))]) .map_err(map_dynwinrt_error)?; - Ok(DynWinRTValue::new(results.into_iter().next().ok_or_else( - || PyRuntimeError::new_err("invoke_hstring: no result"), - )?)) + tracked_native_value( + py, + results + .into_iter() + .next() + .ok_or_else(|| PyRuntimeError::new_err("invoke_hstring: no result"))?, + ) } /// 1-arg invoke with i32 input → DynWinRTValue result - fn invoke_i32(&self, obj: DynWinRTValue, arg: i32) -> PyResult { + fn invoke_i32( + &self, + py: Python<'_>, + obj: DynWinRTValue, + arg: i32, + ) -> PyResult> { let raw = obj.com_receiver("invoke_i32()")?.as_raw(); let results = self .0 .invoke(raw, &[dynwinrt::WinRTValue::I32(arg)]) .map_err(map_dynwinrt_error)?; - Ok(DynWinRTValue::new(results.into_iter().next().ok_or_else( - || PyRuntimeError::new_err("invoke_i32: no result"), - )?)) + tracked_native_value( + py, + results + .into_iter() + .next() + .ok_or_else(|| PyRuntimeError::new_err("invoke_i32: no result"))?, + ) } } @@ -1083,6 +1107,37 @@ impl DynWinRTMethodHandle { #[derive(Clone)] pub struct DynWinRTValue(pub(crate) dynwinrt::WinRTValue, Lifecycle); +fn contains_com_references(typ: &dynwinrt::TypeHandle) -> bool { + let kind = typ.kind(); + kind.is_com_pointer() + || matches!(kind, dynwinrt::TypeKind::ArrayOfIUnknown) + || (matches!(kind, dynwinrt::TypeKind::Struct(_)) + && (0..typ.field_count()).any(|index| contains_com_references(&typ.field_type(index)))) +} + +/// Keep native COM ownership on the creating thread until the active lifetime +/// scope closes. Python retains the exact returned value, not an extra AddRef. +pub(crate) fn tracked_native_value( + py: Python<'_>, + value: dynwinrt::WinRTValue, +) -> PyResult> { + let owns_native = match &value { + dynwinrt::WinRTValue::Object(_) + | dynwinrt::WinRTValue::Async(_) + | dynwinrt::WinRTValue::ArrayOfIUnknown(_) => true, + dynwinrt::WinRTValue::Array(array) => contains_com_references(&array.element_type), + dynwinrt::WinRTValue::Struct(data) => contains_com_references(&data.type_handle()), + _ => false, + }; + let output = Py::new(py, DynWinRTValue::new(value))?; + if owns_native { + py.import("dynwinrt.dynwinrt")? + .getattr("_dynwinrt_track_projected")? + .call1((output.clone_ref(py), "DynWinRTValue"))?; + } + Ok(output) +} + /// Whether a value still owns its native payload. `release()` is the only /// transition and leaves `WinRTValue::Null` behind, so this state is what /// distinguishes a released value from a WinRT null reference. @@ -1242,16 +1297,16 @@ fn value_kind(value: &dynwinrt::WinRTValue) -> &'static str { #[pymethods] impl DynWinRTValue { #[staticmethod] - fn activation_factory(name: String) -> PyResult { + fn activation_factory(py: Python<'_>, name: String) -> PyResult> { WINUI_MODULES .activation_factory(&HSTRING::from(name)) - .map(DynWinRTValue::new) .map_err(map_dynwinrt_error) + .and_then(|value| tracked_native_value(py, value)) } /// Create an owned WinRT IBuffer by copying Python bytes or bytearray data. #[staticmethod] - fn from_bytes(data: &Bound<'_, PyAny>) -> PyResult { + fn from_bytes(py: Python<'_>, data: &Bound<'_, PyAny>) -> PyResult> { let bytes = if let Ok(data) = data.cast::() { data.as_bytes().to_vec() } else if let Ok(data) = data.cast::() { @@ -1262,8 +1317,8 @@ impl DynWinRTValue { )); }; dynwinrt::copy_to_ibuffer(&bytes) - .map(DynWinRTValue::new) .map_err(map_dynwinrt_error) + .and_then(|value| tracked_native_value(py, value)) } /// Compose a WinUI `Microsoft.UI.Xaml.Application` whose outer object @@ -1273,9 +1328,10 @@ impl DynWinRTValue { #[staticmethod] #[pyo3(signature = (metadata_provider, launched_callback=None))] fn create_xaml_application( + py: Python<'_>, metadata_provider: &DynWinRTValue, launched_callback: Option<&DynWinRTValue>, - ) -> PyResult { + ) -> PyResult> { let provider = metadata_provider.0.as_object().ok_or_else(|| { PyRuntimeError::new_err("create_xaml_application: metadata_provider must be an Object") })?; @@ -1290,8 +1346,8 @@ impl DynWinRTValue { .transpose()?; WINUI_MODULES .create_xaml_application(&provider, callback.as_ref()) - .map(DynWinRTValue::new) .map_err(map_dynwinrt_error) + .and_then(|value| tracked_native_value(py, value)) } // -- Scalar constructors (full parity with JS) -- @@ -1376,11 +1432,15 @@ impl DynWinRTValue { } #[staticmethod] - fn box_reference(value: &DynWinRTValue, value_type: &DynWinRTType) -> PyResult { + fn box_reference( + py: Python<'_>, + value: &DynWinRTValue, + value_type: &DynWinRTType, + ) -> PyResult> { value.check_input("DynWinRTValue.box_reference()", InputSlot::Argument(0))?; dynwinrt::box_ireference(value.0.clone(), value_type.0.clone()) - .map(DynWinRTValue::new) .map_err(map_dynwinrt_error) + .and_then(|value| tracked_native_value(py, value)) } /// Get the signed or unsigned numeric value of an enum. Returns None if not an enum. @@ -1401,24 +1461,26 @@ impl DynWinRTValue { /// Create an IVector from items. #[staticmethod] fn create_vector( + py: Python<'_>, items: Vec, element_type: &DynWinRTType, - ) -> PyResult { + ) -> PyResult> { let wrt_items = native_inputs("DynWinRTValue.create_vector()", items, InputSlot::Element)?; let iids = TABLE.vector_iids(&element_type.0); let vector = dynwinrt::vector::create_vector_from_values(&wrt_items, &element_type.0, iids) .map_err(map_dynwinrt_error)?; - Ok(DynWinRTValue::new(dynwinrt::WinRTValue::Object(vector))) + tracked_native_value(py, dynwinrt::WinRTValue::Object(vector)) } /// Create an IMap from parallel key/value lists. #[staticmethod] fn create_map( + py: Python<'_>, keys: Vec, values: Vec, key_type: &DynWinRTType, value_type: &DynWinRTType, - ) -> PyResult { + ) -> PyResult> { if keys.len() != values.len() { return Err(PyRuntimeError::new_err( "create_map: keys and values must have the same length", @@ -1432,19 +1494,19 @@ impl DynWinRTValue { keys.into_iter().zip(values).collect(); let map = dynwinrt::map::create_map_from_values(&entries, &key_type.0, &value_type.0, iids) .map_err(map_dynwinrt_error)?; - Ok(DynWinRTValue::new(dynwinrt::WinRTValue::Object(map))) + tracked_native_value(py, dynwinrt::WinRTValue::Object(map)) } /// Await an async WinRT operation (blocks the current thread). /// Releases the Python GIL while waiting so other threads can proceed. - fn wait(&self, py: Python<'_>) -> PyResult { - super::async_runtime::wait_for_async(&self.0, py).map(DynWinRTValue::new) + fn wait(&self, py: Python<'_>) -> PyResult> { + tracked_native_value(py, super::async_runtime::wait_for_async(&self.0, py)?) } - fn _get_async_results(&self) -> PyResult { + fn _get_async_results(&self, py: Python<'_>) -> PyResult> { dynwinrt::get_async_results(&self.0) - .map(DynWinRTValue::new) .map_err(map_dynwinrt_error) + .and_then(|value| tracked_native_value(py, value)) } /// Cancel the underlying WinRT async operation (calls `IAsyncInfo::Cancel`). @@ -1729,8 +1791,8 @@ impl DynWinRTValue { } /// COM QueryInterface — cast to a different interface. - fn cast(&self, iid: &WinGUID) -> PyResult { - self.query(&iid.0, "cast()").map(DynWinRTValue::new) + fn cast(&self, py: Python<'_>, iid: &WinGUID) -> PyResult> { + tracked_native_value(py, self.query(&iid.0, "cast()")?) } /// Invoke metadata-described Invoke on an IUnknown-rooted WinRT delegate. @@ -1739,27 +1801,36 @@ impl DynWinRTValue { iid: &WinGUID, signature: &DynWinRTMethodSig, args: Vec, - ) -> PyResult> { + ) -> PyResult>> { crate::delegate_method::DynWinRTDelegateMethod::create(iid, signature)?.invoke(slf, args) } /// Call IActivationFactory::ActivateInstance (vtable[6]) to create a default instance. /// Use on the result of activation_factory() for classes with parameterless constructors. - fn activate(&self) -> PyResult { + fn activate(&self, py: Python<'_>) -> PyResult> { let method = dynwinrt::MethodSignature::new(&*TABLE) .add_out(TABLE.object()) .build(6); let raw = self.com_receiver("activate()")?.as_raw(); let result = method.call_dynamic(raw, &[]).map_err(map_windows_error)?; - Ok(DynWinRTValue::new(result.into_iter().next().ok_or_else( - || PyRuntimeError::new_err("activate: no result"), - )?)) + tracked_native_value( + py, + result + .into_iter() + .next() + .ok_or_else(|| PyRuntimeError::new_err("activate: no result"))?, + ) } // -- Convenience call methods (match JS API) -- /// Call a method with no args and one out param. - fn call_0(&self, method_index: usize, return_type: &DynWinRTType) -> PyResult { + fn call_0( + &self, + py: Python<'_>, + method_index: usize, + return_type: &DynWinRTType, + ) -> PyResult> { let method = dynwinrt::MethodSignature::new(&*TABLE) .add_out(return_type.0.clone()) .build(method_index); @@ -1767,16 +1838,20 @@ impl DynWinRTValue { let result = method .call_dynamic(obj_raw, &[]) .map_err(map_windows_error)?; - Ok(DynWinRTValue::new(result.into_iter().next().unwrap())) + tracked_native_value( + py, + result.into_iter().next().expect("call_0 has one output"), + ) } /// Call a method with one arg and one out param. fn call_1( &self, + py: Python<'_>, method_index: usize, return_type: &DynWinRTType, v1: &DynWinRTValue, - ) -> PyResult { + ) -> PyResult> { let obj_raw = self.com_receiver("call_1()")?.as_raw(); v1.check_input("call_1()", InputSlot::Argument(0))?; let in_type = TABLE.handle_from_kind(v1.0.get_type_kind()); @@ -1787,17 +1862,21 @@ impl DynWinRTValue { let result = method .call_dynamic(obj_raw, &[v1.0.clone()]) .map_err(map_windows_error)?; - Ok(DynWinRTValue::new(result.into_iter().next().unwrap())) + tracked_native_value( + py, + result.into_iter().next().expect("call_1 has one output"), + ) } /// General-purpose method call with explicit types and args. fn call( &self, + py: Python<'_>, method_index: usize, return_type: &DynWinRTType, in_types: Vec, args: Vec, - ) -> PyResult { + ) -> PyResult> { let mut method = dynwinrt::MethodSignature::new(&*TABLE); for t in &in_types { method = method.add_in(t.0.clone()); @@ -1819,11 +1898,11 @@ impl DynWinRTValue { .call_dynamic(obj, &winrt_args) .map_err(map_windows_error)?; - if result.is_empty() { - Ok(DynWinRTValue::new(dynwinrt::WinRTValue::I32(0))) - } else { - Ok(DynWinRTValue::new(result.into_iter().next().unwrap())) - } + let value = result + .into_iter() + .next() + .unwrap_or(dynwinrt::WinRTValue::I32(0)); + tracked_native_value(py, value) } // -- Array / Struct extraction -- @@ -1880,18 +1959,18 @@ impl DynWinRTArray { } /// Per-element access. - fn get(&self, index: i64) -> PyResult { + fn get(&self, py: Python<'_>, index: i64) -> PyResult> { let index = checked_index(index)?; self.0 .try_get(index) - .map(DynWinRTValue::new) .map_err(map_dynwinrt_error) + .and_then(|value| tracked_native_value(py, value)) } /// Convert all elements to a list of DynWinRTValue. - fn to_values(&self) -> Vec { + fn to_values(&self, py: Python<'_>) -> PyResult>> { (0..self.0.len()) - .map(|i| DynWinRTValue::new(self.0.get(i))) + .map(|i| tracked_native_value(py, self.0.get(i))) .collect() } @@ -2136,8 +2215,8 @@ impl DynWinRTArray { } /// Wrap as DynWinRTValue::Array for passing to call(). - fn to_value(&self) -> DynWinRTValue { - DynWinRTValue::new(dynwinrt::WinRTValue::Array(self.0.clone())) + fn to_value(&self, py: Python<'_>) -> PyResult> { + tracked_native_value(py, dynwinrt::WinRTValue::Array(self.0.clone())) } fn __repr__(&self) -> String { @@ -2355,12 +2434,13 @@ impl DynWinRTStruct { .map_err(map_dynwinrt_error) } - fn get_object(&self, index: i64) -> PyResult { + fn get_object(&self, py: Python<'_>, index: i64) -> PyResult> { let index = checked_index(index)?; - match self.0.get_field_object(index).map_err(map_dynwinrt_error)? { - Some(object) => Ok(DynWinRTValue::new(dynwinrt::WinRTValue::Object(object))), - None => Ok(DynWinRTValue::new(dynwinrt::WinRTValue::Null)), - } + let value = match self.0.get_field_object(index).map_err(map_dynwinrt_error)? { + Some(object) => dynwinrt::WinRTValue::Object(object), + None => dynwinrt::WinRTValue::Null, + }; + tracked_native_value(py, value) } fn set_object(&mut self, index: i64, value: &DynWinRTValue) -> PyResult<()> { @@ -2382,8 +2462,8 @@ impl DynWinRTStruct { } /// Wrap as DynWinRTValue::Struct for passing to call(). - fn to_value(&self) -> DynWinRTValue { - DynWinRTValue::new(dynwinrt::WinRTValue::Struct(self.0.clone())) + fn to_value(&self, py: Python<'_>) -> PyResult> { + tracked_native_value(py, dynwinrt::WinRTValue::Struct(self.0.clone())) } fn __repr__(&self) -> String { @@ -2456,8 +2536,8 @@ impl DynWinRtDelegate { } /// Get the delegate as a DynWinRTValue for passing to WinRT methods. - fn to_value(&self) -> DynWinRTValue { - DynWinRTValue::new(self.0.clone()) + fn to_value(&self, py: Python<'_>) -> PyResult> { + tracked_native_value(py, self.0.clone()) } fn __repr__(&self) -> String { @@ -2591,8 +2671,8 @@ impl DynWinRtElementFactory { }) } - fn to_value(&self) -> DynWinRTValue { - DynWinRTValue::new(self.value.clone()) + fn to_value(&self, py: Python<'_>) -> PyResult> { + tracked_native_value(py, self.value.clone()) } fn release_callbacks(&self) -> PyResult<()> { @@ -2873,11 +2953,11 @@ mod tests { .unwrap(); let receiver = DynWinRTValue::new(owner.to_value().unwrap().cast(&iid).unwrap()); let method = DynWinRTMethodHandle(interface.method(6).unwrap()); - let direct = method.invoke(receiver.clone(), vec![]).unwrap(); + let direct = method.invoke(py, receiver.clone(), vec![]).unwrap(); let detached = method.invoke_detached(py, receiver, vec![]).unwrap(); - for result in [direct, detached] { + for result in [&direct, &detached] { assert!(matches!( - result.0, + &result.borrow(py).0, dynwinrt::WinRTValue::HString(value) if value == "native observer" )); } diff --git a/bindings/py/tests/test_phase1.py b/bindings/py/tests/test_phase1.py index 7e1e3c59..42439a31 100644 --- a/bindings/py/tests/test_phase1.py +++ b/bindings/py/tests/test_phase1.py @@ -1241,6 +1241,22 @@ def test_projected_lifetime_scope_releases_native_values_before_apartment_exit() release_projected(SimpleNamespace(_obj=second)) +def test_projected_lifetime_scope_tracks_raw_native_outputs_automatically(): + with RoApartment(1), projected_lifetime_scope(): + factory = DynWinRTValue.activation_factory("Windows.Foundation.Uri") + cast = factory.cast(WinGUID.parse(IID_IURI_FACTORY)) + scalar = DynWinRTValue.from_u32(8080) + assert not factory.is_released() + assert not cast.is_released() + + assert factory.is_released() + assert cast.is_released() + assert scalar.to_u32() == 8080 + assert not scalar.is_released() + with pytest.raises(RuntimeError, match="released"): + cast.identity_raw() + + def test_projected_lifetime_scope_enforces_lifo_order(): outer = projected_lifetime_scope() inner = projected_lifetime_scope() diff --git a/tools/dynwinrt-codegen/python/README.md b/tools/dynwinrt-codegen/python/README.md index 140706f1..320a9809 100644 --- a/tools/dynwinrt-codegen/python/README.md +++ b/tools/dynwinrt-codegen/python/README.md @@ -52,6 +52,13 @@ with RoApartment(), projected_lifetime_scope(): print(uri.host) ``` +The matching Python binding also tracks raw native `DynWinRTValue` results +from generated factories and methods in an active lifetime scope. For example, +`PropertyValue.create_uint32(8080)` returns a raw value; if it escapes the +scope, its owned COM reference has already been released before the apartment +exits. A raw result retained without a scope must instead be released +explicitly inside its apartment. + ## CLI options | Option | Description | diff --git a/tools/dynwinrt-codegen/tests/python_raw_lifetime_test.rs b/tools/dynwinrt-codegen/tests/python_raw_lifetime_test.rs new file mode 100644 index 00000000..c1ce0c75 --- /dev/null +++ b/tools/dynwinrt-codegen/tests/python_raw_lifetime_test.rs @@ -0,0 +1,278 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +const WINDOWS_WINMD: &str = + r"C:\Program Files (x86)\Windows Kits\10\UnionMetadata\10.0.26100.0\Windows.winmd"; + +struct Generated { + root: PathBuf, + package: String, +} + +impl Generated { + fn new() -> Option { + if !Path::new(WINDOWS_WINMD).is_file() { + eprintln!("Skipping raw lifetime regression: Windows.winmd not found"); + return None; + } + let repo = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap(); + let package = format!("raw_lifetime_{}", std::process::id()); + let root = repo.join("target").join(&package); + let output = Command::new(env!("CARGO_BIN_EXE_dynwinrt-codegen")) + .args([ + "generate", + "--winmd", + WINDOWS_WINMD, + "--class-name", + "Windows.Foundation.PropertyValue", + "--lang", + "py", + "--output", + ]) + .arg(&root) + .output() + .expect("generate PropertyValue bindings"); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + Some(Self { root, package }) + } + + fn python(&self) -> PathBuf { + std::env::var_os("DYNWINRT_TEST_PYTHON") + .map(PathBuf::from) + .unwrap_or_else(|| { + let repo = self.root.parent().unwrap().parent().unwrap(); + let venv = repo.join(r"bindings\py\.venv\Scripts\python.exe"); + if venv.is_file() { + venv + } else { + PathBuf::from("python") + } + }) + } + + fn run(&self, scenario: &str, script: &str) { + let output = Command::new(self.python()) + .args(["-B", "-c", &script.replace("PY_PACKAGE", &self.package)]) + .env("PYTHONPATH", self.root.parent().unwrap()) + .output() + .expect("run isolated Python lifetime regression"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert_eq!( + output.status.code(), + Some(0), + "{scenario}: Python exited {:?}\n{stdout}\n{stderr}", + output.status.code() + ); + assert!(stdout.contains(scenario), "{scenario}:\n{stdout}\n{stderr}"); + } +} + +impl Drop for Generated { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.root); + } +} + +#[test] +fn generated_raw_outputs_release_before_apartment_exit_even_when_they_escape() { + let Some(generated) = Generated::new() else { + return; + }; + let available = Command::new(generated.python()) + .args([ + "-c", + "from dynwinrt import DynWinRTImplementationHandle, RoApartment", + ]) + .output() + .is_ok_and(|output| output.status.success()); + assert!( + available || std::env::var("DYNWINRT_REQUIRE_IMPLEMENTATION_RUNTIME").as_deref() != Ok("1"), + "raw lifetime regression requires the matching Python binding" + ); + if !available { + eprintln!("Skipping raw lifetime regression: matching Python binding not installed"); + return; + } + + generated.run( + "unscoped-explicit-release", + r#" +from dynwinrt import RoApartment +from PY_PACKAGE.windows.foundation import PropertyValue +with RoApartment(): + raw = PropertyValue.create_uint32(8080) + assert not raw.is_released() + raw.release() +assert raw.is_released() +print('unscoped-explicit-release', flush=True) +"#, + ); + generated.run( + "explicit-release-control", + r#" +from dynwinrt import RoApartment, projected_lifetime_scope +from PY_PACKAGE.windows.foundation import PropertyValue +with RoApartment(), projected_lifetime_scope() as scope: + raw = PropertyValue.create_uint32(8080) + assert not raw.is_released() + raw.release() + assert raw.is_released() +assert scope.disposed and raw.is_released() +print('explicit-release-control', flush=True) +"#, + ); + generated.run( + "escaped-raw-shutdown", + r#" +from dynwinrt import RoApartment, WinGUID, projected_lifetime_scope +from PY_PACKAGE.windows.foundation import PropertyValue +def escaped(): + with RoApartment(), projected_lifetime_scope() as scope: + raw = PropertyValue.create_uint32(8080) + assert not raw.is_released() + return raw, scope +raw, scope = escaped() +assert scope.disposed and raw.is_released() +try: + raw.cast(WinGUID.parse('00000000-0000-0000-c000-000000000046')) +except RuntimeError as error: + assert 'released' in str(error) +else: + raise AssertionError('escaped native value remained callable') +print('escaped-raw-shutdown', flush=True) +# Keep raw alive through interpreter shutdown, past RoApartment.__exit__. +"#, + ); + generated.run( + "direct-native-outputs", + r#" +from dynwinrt import DynWinRTValue, RoApartment, projected_lifetime_scope +from PY_PACKAGE.windows__foundation__property_value import ( + IID_IPropertyValueStatics, _IPropertyValueStatics, +) +with RoApartment(), projected_lifetime_scope(): + scalar = DynWinRTValue.from_u32(8080) + factory = DynWinRTValue.activation_factory( + 'Windows.Foundation.PropertyValue' + ).cast(IID_IPropertyValueStatics) + raw = _IPropertyValueStatics.method(11).invoke( + factory, [DynWinRTValue.from_u32(8080)] + ) + outputs = _IPropertyValueStatics.method(11).invoke_all( + factory, [DynWinRTValue.from_u32(8080)] + ) + assert len(outputs) == 1 + assert not raw.is_released() and not outputs[0].is_released() +assert factory.is_released() and raw.is_released() and outputs[0].is_released() +assert not scalar.is_released() and scalar.to_u32() == 8080 +print('direct-native-outputs', flush=True) +"#, + ); + generated.run( + "borrowed-source-retains-ownership", + r#" +from dynwinrt import DynWinRTValue, RoApartment, projected_lifetime_scope +from PY_PACKAGE.windows__foundation__property_value import IID_IPropertyValueStatics +with RoApartment(): + source = DynWinRTValue.activation_factory('Windows.Foundation.PropertyValue') + with projected_lifetime_scope(): + view = source.cast(IID_IPropertyValueStatics) + assert not source.is_released() and not view.is_released() + assert view.is_released() and not source.is_released() + independent = source.cast(IID_IPropertyValueStatics) + independent.release() + source.release() +assert source.is_released() +print('borrowed-source-retains-ownership', flush=True) +"#, + ); + generated.run( + "original-error-preserved", + r#" +from dynwinrt import RoApartment, projected_lifetime_scope +from PY_PACKAGE.windows.foundation import PropertyValue +try: + with RoApartment(), projected_lifetime_scope(): + raw = PropertyValue.create_uint32(8080) + raise ValueError('original failure') +except ValueError as error: + assert str(error) == 'original failure' +else: + raise AssertionError('scope suppressed the original failure') +assert raw.is_released() +print('original-error-preserved', flush=True) +"#, + ); + generated.run( + "nested-and-callback-scopes", + r#" +from concurrent.futures import ThreadPoolExecutor +from dynwinrt import RoApartment, projected_lifetime_scope +from dynwinrt.dynwinrt import _dynwinrt_wrap_delegate_callback +from PY_PACKAGE.windows.foundation import PropertyValue + +with RoApartment(), projected_lifetime_scope(): + callback = _dynwinrt_wrap_delegate_callback( + lambda: PropertyValue.create_uint32(1) + ) + same_thread = callback() + with projected_lifetime_scope(): + nested = PropertyValue.create_uint32(2) + assert nested.is_released() and not same_thread.is_released() + + def foreign_thread(): + with RoApartment(): + raw = callback() + assert not raw.is_released() + raw.release() + return raw.is_released() + + with ThreadPoolExecutor(max_workers=1) as executor: + assert executor.submit(foreign_thread).result() + assert not same_thread.is_released() +assert same_thread.is_released() +print('nested-and-callback-scopes', flush=True) +"#, + ); + generated.run( + "foreign-scope-rejected", + r#" +from concurrent.futures import ThreadPoolExecutor +from contextvars import copy_context +from dynwinrt import RoApartment, projected_lifetime_scope +from PY_PACKAGE.windows.foundation import PropertyValue + +with RoApartment(), projected_lifetime_scope() as scope: + inherited = copy_context() + + def foreign_thread(): + with RoApartment(): + return inherited.run(lambda: PropertyValue.create_uint32(8080)) + + with ThreadPoolExecutor(max_workers=1) as executor: + try: + executor.submit(foreign_thread).result() + except RuntimeError as error: + assert 'different thread' in str(error) + else: + raise AssertionError('foreign output entered the owner thread scope') + assert not scope.disposed +assert scope.disposed +print('foreign-scope-rejected', flush=True) +"#, + ); +} From 2898a21842a7f6417e31432e9b364bd81f286c8a Mon Sep 17 00:00:00 2001 From: Leilei Zhang Date: Tue, 29 Sep 2026 18:24:16 +0800 Subject: [PATCH 03/15] Validate generated Python interface IID before native storage QueryInterface-check standalone constructors before retaining or caching a pointer, preserve borrowed constructor sources on cache hits, and fail closed for unknown IIDs. Keep generic and observable projections safe with real WinRT and strict typing regressions. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- bindings/py/README.md | 7 + bindings/py/src/lib.rs | 8 +- tools/dynwinrt-codegen/python/README.md | 6 + .../codegen/winrt/python/generator/class.rs | 2 +- .../codegen/winrt/python/generator/types.rs | 44 ++- .../tests/observable_map_test.rs | 18 +- .../tests/observable_vector_test.rs | 14 +- .../tests/python_identity_cache_test.rs | 9 + .../python_interface_constructor_iid_test.rs | 341 ++++++++++++++++++ .../tests/python_json_native_null_test.rs | 16 +- 10 files changed, 444 insertions(+), 21 deletions(-) create mode 100644 tools/dynwinrt-codegen/tests/python_interface_constructor_iid_test.rs diff --git a/bindings/py/README.md b/bindings/py/README.md index 99d030ac..803ffc0c 100644 --- a/bindings/py/README.md +++ b/bindings/py/README.md @@ -440,6 +440,13 @@ wrapper to an interface view. Use `InterfaceClass.from_value(raw)` for a raw `DynWinRTValue`. `as_interface()` accepts generated interface classes only; passing a runtime class raises `TypeError` that points to `project_as()`. Do not call the internal `_from_native()` method from application code. +Legacy direct construction with `InterfaceClass(raw)` also checks the +interface IID before retaining or caching the pointer. A non-implementing +object raises `E_NOINTERFACE` before any interface method can dispatch; a +successful view owns its own QueryInterface reference and does not consume the +raw source, even when the view is returned from the identity cache. Stubs +require a `DynWinRTValue` for explicit raw projection, but Python's type system +cannot prove its runtime IID. ### Views of `Object`-valued maps diff --git a/bindings/py/src/lib.rs b/bindings/py/src/lib.rs index 7cedae6d..2a5f779c 100644 --- a/bindings/py/src/lib.rs +++ b/bindings/py/src/lib.rs @@ -152,13 +152,17 @@ def _dynwinrt_cache_projected(value): pass return value -def _dynwinrt_projected_from_native(wrapper_type, native, initializer_name): +def _dynwinrt_projected_from_native( + wrapper_type, native, initializer_name, *, release_redundant=True +): key = _dynwinrt_projected_cache_key(wrapper_type, native) if key is not None: cached = _projected_wrapper_cache.get(key) if cached is not None: if _dynwinrt_projected_wrapper_is_live(cached): - _dynwinrt_release_redundant_native(native, cached) + # A direct interface constructor borrows its source. + if release_redundant: + _dynwinrt_release_redundant_native(native, cached) return cached _projected_wrapper_cache.pop(key, None) wrapper = object.__new__(wrapper_type) diff --git a/tools/dynwinrt-codegen/python/README.md b/tools/dynwinrt-codegen/python/README.md index 320a9809..4be41721 100644 --- a/tools/dynwinrt-codegen/python/README.md +++ b/tools/dynwinrt-codegen/python/README.md @@ -144,6 +144,12 @@ retain the receiving interface subclass: `TaggedBuffer.from_value(raw)` and `value.as_interface(TaggedBuffer)` return `TaggedBuffer`, not `IBuffer`. Independent static factories such as `IBuffer.from_bytes` keep their declared base-interface result. +Direct runtime construction such as `IBuffer(raw)` also QueryInterface-checks +the IID before retaining or caching a native pointer. It raises `E_NOINTERFACE` +for a mismatched object and does not release the caller's raw value on a cache +hit; the resulting view owns a separate reference. A `DynWinRTValue` annotation +alone cannot establish the runtime IID, so prefer `from_value()` or +`as_interface()` for explicit intent. WinRT `Object` inputs accept a `DynWinRTValue` or a projected native wrapper whose `_obj` is a `DynWinRTValue`, including interface views and runtime-class diff --git a/tools/dynwinrt-codegen/src/codegen/winrt/python/generator/class.rs b/tools/dynwinrt-codegen/src/codegen/winrt/python/generator/class.rs index bd2a725b..a68c0c1a 100644 --- a/tools/dynwinrt-codegen/src/codegen/winrt/python/generator/class.rs +++ b/tools/dynwinrt-codegen/src/codegen/winrt/python/generator/class.rs @@ -858,7 +858,7 @@ pub fn generate_class<'a>( out.push_str(" def __new__(cls, *args, **kwargs):\n"); out.push_str( " if len(args) == 1 and not kwargs and isinstance(args[0], DynWinRTValue):\n\ - \x20 return _dynwinrt_projected_from_native(cls, args[0], '_set_native')\n\ + \x20 return _dynwinrt_projected_from_native(cls, args[0], '_set_native', release_redundant=False)\n\ \x20 return super().__new__(cls)\n\n", ); out.push_str(" def _set_native(self, obj: DynWinRTValue):\n"); diff --git a/tools/dynwinrt-codegen/src/codegen/winrt/python/generator/types.rs b/tools/dynwinrt-codegen/src/codegen/winrt/python/generator/types.rs index 0938cfb1..11719432 100644 --- a/tools/dynwinrt-codegen/src/codegen/winrt/python/generator/types.rs +++ b/tools/dynwinrt-codegen/src/codegen/winrt/python/generator/types.rs @@ -321,27 +321,33 @@ pub fn generate_interface(context: &PythonProjectionContext, iface: &InterfaceMe out.push_str(" def __new__(cls, *args, **kwargs):\n"); out.push_str( " if len(args) == 1 and not kwargs and isinstance(args[0], DynWinRTValue):\n\ - \x20 return _dynwinrt_projected_from_native(cls, args[0], '_set_native')\n\ + \x20 return _dynwinrt_projected_from_native(cls, args[0], '_set_native', release_redundant=False)\n\ \x20 return super().__new__(cls)\n\n", ); out.push_str(" def _set_native(self, obj: DynWinRTValue, *, cache=True):\n"); if let Some(identity) = &observable_collection { let companion_name = context.projected_name(identity); + out.push_str(&format!( + " _observable_obj = obj.cast(IID_{})\n", + iface.name + )); out.push_str(&format!( " {}._set_native(self, obj)\n", py_runtime_symbol(context, identity, &companion_name) )); + out.push_str(" self._observable_obj = _observable_obj\n"); + } else if iface.generic_piid.is_some() { out.push_str(&format!( - " self._observable_obj = obj.cast(IID_{})\n", + " self._obj = obj.cast(IID_{})\n", iface.name )); - } else if iface.generic_piid.is_some() { + } else if !iface.iid.is_empty() { out.push_str(&format!( " self._obj = obj.cast(IID_{})\n", iface.name )); } else { - out.push_str(" self._obj = obj\n"); + out.push_str(" raise TypeError('Cannot construct an interface without an IID')\n"); } out.push_str(" self._dynwinrt_native_ready = True\n"); out.push_str(&format!( @@ -778,6 +784,36 @@ mod tests { assert!(code.contains("_dynwinrt_interface_iid = IID_IWidget")); assert!(code.contains("@classmethod\n def from_value(cls, obj: DynWinRTValue)")); assert!(code.contains("return cls._from_native(obj.cast(IID_IWidget))")); + assert!(code.contains( + "return _dynwinrt_projected_from_native(cls, args[0], '_set_native', release_redundant=False)" + )); + let initializer = code + .split("def _set_native(self, obj: DynWinRTValue, *, cache=True):\n") + .nth(1) + .unwrap() + .split(" def __init__") + .next() + .unwrap(); + let cast = initializer + .find("self._obj = obj.cast(IID_IWidget)") + .unwrap(); + let cache = initializer.find("_dynwinrt_cache_projected(self)").unwrap(); + assert!(cast < cache, "{initializer}"); + assert!(!initializer.contains("self._obj = obj\n"), "{initializer}"); + } + + #[test] + fn interface_without_an_iid_rejects_direct_native_construction() { + let iface = InterfaceMeta { + name: "IUnresolved".into(), + namespace: "Contoso".into(), + ..Default::default() + }; + let context = PythonProjectionContext::standalone([iface.type_identity()]).unwrap(); + let code = generate_interface(&context, &iface); + assert!(code.contains("raise TypeError('Cannot construct an interface without an IID')")); + assert!(!code.contains("self._obj = obj\n")); + assert!(!code.contains("def from_value(cls, obj: DynWinRTValue)")); } #[test] diff --git a/tools/dynwinrt-codegen/tests/observable_map_test.rs b/tools/dynwinrt-codegen/tests/observable_map_test.rs index b0df5b48..af04b3a4 100644 --- a/tools/dynwinrt-codegen/tests/observable_map_test.rs +++ b/tools/dynwinrt-codegen/tests/observable_map_test.rs @@ -224,14 +224,16 @@ fn observable_map_projects_python_mutable_mapping_and_typed_events() { py.contains(&format!("class IObservableMap_String_Object({map_base}):")), "{py}" ); - assert!( - py.contains(&format!(" {map_base}._set_native(self, obj)\n")), - "{py}" - ); - assert!( - py.contains("self._observable_obj = obj.cast(IID_IObservableMap_String_Object)"), - "{py}" - ); + let observable_cast = py + .find("_observable_obj = obj.cast(IID_IObservableMap_String_Object)") + .expect("observable IID validation"); + let companion = py + .find(&format!("{map_base}._set_native(self, obj)")) + .expect("mutable map companion"); + let saved = py + .find("self._observable_obj = _observable_obj") + .expect("validated observable pointer"); + assert!(observable_cast < companion && companion < saved, "{py}"); assert!( py.contains( "_dynwinrt_map_dispatch = (_dynwinrt_symbol('i_map_string_object', 'IID_IMap_String_Object'), '_obj')" diff --git a/tools/dynwinrt-codegen/tests/observable_vector_test.rs b/tools/dynwinrt-codegen/tests/observable_vector_test.rs index 821c47bd..157eac49 100644 --- a/tools/dynwinrt-codegen/tests/observable_vector_test.rs +++ b/tools/dynwinrt-codegen/tests/observable_vector_test.rs @@ -117,10 +117,16 @@ fn observable_vector_projects_python_mutable_sequence_and_typed_events() { assert!(py.contains( "class IObservableVector_Object(_dynwinrt_symbol('i_vector_object', 'IVector_Object')):" )); - assert!( - py.contains("_dynwinrt_symbol('i_vector_object', 'IVector_Object')._set_native(self, obj)") - ); - assert!(py.contains("self._observable_obj = obj.cast(IID_IObservableVector_Object)")); + let observable_cast = py + .find("_observable_obj = obj.cast(IID_IObservableVector_Object)") + .expect("observable IID validation"); + let companion = py + .find("_dynwinrt_symbol('i_vector_object', 'IVector_Object')._set_native(self, obj)") + .expect("mutable vector companion"); + let saved = py + .find("self._observable_obj = _observable_obj") + .expect("validated observable pointer"); + assert!(observable_cast < companion && companion < saved, "{py}"); let create_signature = "def create(items: Iterable[DynWinRTValue | _DynWinRTObject | None]) -> 'IObservableVector_Object':"; assert!(py.contains(create_signature), "{py}"); assert!( diff --git a/tools/dynwinrt-codegen/tests/python_identity_cache_test.rs b/tools/dynwinrt-codegen/tests/python_identity_cache_test.rs index dffc7572..d6800be6 100644 --- a/tools/dynwinrt-codegen/tests/python_identity_cache_test.rs +++ b/tools/dynwinrt-codegen/tests/python_identity_cache_test.rs @@ -174,6 +174,10 @@ fn interface_generation_uses_projected_identity_cache() { py.contains("return cls._from_native(obj.cast(IID_IWidget))"), "from_value should reuse the cached wrapper path:\n{py}" ); + assert!( + py.contains("self._obj = obj.cast(IID_IWidget)") && py.contains("release_redundant=False"), + "direct interface projection must validate IID without consuming its source:\n{py}" + ); let pyi = common::generate_interface_stub( &iface, &HashSet::from(["IWidget".to_string()]), @@ -237,6 +241,11 @@ fn embedded_interface_projection_preserves_subclasses_and_qi_helpers() { inline.contains("return cls._from_native(obj.cast(IID_IExtra))"), "{inline}" ); + assert!( + inline.contains("self._obj = obj.cast(IID_IExtra)") + && inline.contains("release_redundant=False"), + "{inline}" + ); assert!( inline.contains("def as_interface(self, interface_class):"), "{inline}" diff --git a/tools/dynwinrt-codegen/tests/python_interface_constructor_iid_test.rs b/tools/dynwinrt-codegen/tests/python_interface_constructor_iid_test.rs new file mode 100644 index 00000000..95e66f7a --- /dev/null +++ b/tools/dynwinrt-codegen/tests/python_interface_constructor_iid_test.rs @@ -0,0 +1,341 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; +use std::sync::atomic::{AtomicU64, Ordering}; + +const WINDOWS_WINMD: &str = + r"C:\Program Files (x86)\Windows Kits\10\UnionMetadata\10.0.26100.0\Windows.winmd"; +static NEXT: AtomicU64 = AtomicU64::new(0); + +struct Generated { + root: PathBuf, + package: String, +} + +impl Generated { + fn new() -> Option { + if !Path::new(WINDOWS_WINMD).is_file() { + eprintln!("Skipping interface constructor regression: Windows.winmd not found"); + return None; + } + let repo = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap(); + let package = format!( + "checked_interface_{}_{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + ); + let root = repo.join("target").join(&package); + let output = Command::new(env!("CARGO_BIN_EXE_dynwinrt-codegen")) + .args([ + "generate", + "--winmd", + WINDOWS_WINMD, + "--class-name", + "Windows.Foundation.Uri,Windows.Storage.Streams.Buffer", + "--lang", + "py", + "--output", + ]) + .arg(&root) + .output() + .expect("generate Uri and Buffer bindings"); + assert_success(output); + Some(Self { root, package }) + } + + fn python(&self) -> PathBuf { + std::env::var_os("DYNWINRT_TEST_PYTHON") + .map(PathBuf::from) + .unwrap_or_else(|| { + let repo = self.root.parent().unwrap().parent().unwrap(); + let venv = repo.join(r"bindings\py\.venv\Scripts\python.exe"); + if venv.is_file() { + venv + } else { + PathBuf::from("python") + } + }) + } + + fn module(&self, name: &str) -> String { + fs::read_to_string(self.root.join(name)).expect(name) + } + + fn run(&self, script: &str) -> Output { + Command::new(self.python()) + .args(["-B", "-c", &script.replace("PY_PACKAGE", &self.package)]) + .env("PYTHONPATH", self.root.parent().unwrap()) + .output() + .expect("run isolated constructor consumer") + } +} + +impl Drop for Generated { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.root); + } +} + +fn assert_success(output: Output) { + assert!( + output.status.success(), + "exit {:?}\n{}\n{}", + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn generated_interface_constructor_validates_before_retaining_or_caching() { + let Some(generated) = Generated::new() else { + return; + }; + let buffer = generated.module("windows__storage__streams__i_buffer.py"); + let initializer = buffer + .split("def _set_native(self, obj: DynWinRTValue, *, cache=True):\n") + .nth(1) + .expect("IBuffer native initializer") + .split(" def __init__") + .next() + .unwrap(); + assert!( + initializer.contains("self._obj = obj.cast(IID_IBuffer)") + && !initializer.contains("self._obj = obj\n"), + "{initializer}" + ); + let validated = initializer + .find("self._obj = obj.cast(IID_IBuffer)") + .unwrap(); + let cached = initializer.find("_dynwinrt_cache_projected(self)").unwrap(); + assert!(validated < cached, "{initializer}"); + assert!( + buffer.contains( + "return _dynwinrt_projected_from_native(cls, args[0], '_set_native', release_redundant=False)" + ) && buffer.contains("return cls._from_native(obj.cast(IID_IBuffer))"), + "{buffer}" + ); + let support = generated.module("_runtime.py"); + assert!( + support.contains("_dynwinrt_projected_from_native"), + "{support}" + ); +} + +#[test] +fn real_winrt_rejects_incompatible_direct_and_indirect_interface_projection() { + let Some(generated) = Generated::new() else { + return; + }; + let available = Command::new(generated.python()) + .args([ + "-c", + "from dynwinrt import DynWinRTImplementationHandle, RoApartment", + ]) + .output() + .is_ok_and(|output| output.status.success()); + assert!( + available || std::env::var("DYNWINRT_REQUIRE_IMPLEMENTATION_RUNTIME").as_deref() != Ok("1"), + "interface constructor regression requires the matching Python binding" + ); + if !available { + eprintln!("Skipping constructor native regression: matching binding not installed"); + return; + } + let script = r#" +from dynwinrt import DynWinRTValue, RoApartment, projected_lifetime_scope +from PY_PACKAGE.windows.foundation import Uri +from PY_PACKAGE.windows.storage.streams import Buffer, IBuffer + +with RoApartment(), projected_lifetime_scope(): + uri = Uri('https://example.com/unsafe') + identity = uri._obj.identity_raw() + for construct in ( + lambda: IBuffer(uri._obj), + lambda: IBuffer.__new__(IBuffer, uri._obj), + lambda: IBuffer._from_native(uri._obj), + lambda: IBuffer.from_value(uri._obj), + lambda: uri.as_interface(IBuffer), + ): + try: + construct() + except OSError as error: + assert error.winerror == -2147467262, error # E_NOINTERFACE + else: + raise AssertionError('IBuffer accepted a native Uri pointer') + assert uri._obj.identity_raw() == identity + assert not uri._obj.is_released() + + uninitialized = object.__new__(IBuffer) + try: + IBuffer._set_native(uninitialized, uri._obj) + except OSError as error: + assert error.winerror == -2147467262 + else: + raise AssertionError('_set_native stored a native Uri pointer') + assert not hasattr(uninitialized, '_obj') + + source = DynWinRTValue.from_bytes(b'owned buffer') + first = IBuffer(source) + assert first._obj is not source + assert first._obj.identity_raw() == source.identity_raw() + assert first.to_bytes() == b'owned buffer' + assert IBuffer(source) is first + assert IBuffer.from_value(source) is first + assert first.as_interface(IBuffer) is first + assert not source.is_released() and not first._obj.is_released() + + class TaggedBuffer(IBuffer): + def tag(self): + return 'tagged' + + tagged = TaggedBuffer(source) + assert tagged.tag() == 'tagged' + assert TaggedBuffer.from_value(source) is tagged + assert tagged.as_interface(TaggedBuffer) is tagged + assert tagged._obj.identity_raw() == first._obj.identity_raw() + + source.release() + assert first.to_bytes() == tagged.to_bytes() == b'owned buffer' + projected = Buffer.from_bytes(b'projected buffer') + view = projected.as_interface(IBuffer) + assert projected.to_bytes() == view.to_bytes() == b'projected buffer' + assert projected._obj.identity_raw() == view._obj.identity_raw() +print('checked-constructor-native-ok', flush=True) +"#; + let output = generated.run(script); + let stdout = String::from_utf8_lossy(&output.stdout).into_owned(); + assert_success(output); + assert!(stdout.contains("checked-constructor-native-ok")); +} + +const VALID_CONSUMER: &str = r#" +from typing import assert_type +from dynwinrt import DynWinRTValue +from PY_PACKAGE.windows.foundation import Uri +from PY_PACKAGE.windows.storage.streams import Buffer, IBuffer + +class TaggedBuffer(IBuffer): + def tag(self) -> str: + return 'tagged' + +def project(raw: DynWinRTValue, buffer: Buffer, uri: Uri) -> None: + assert_type(IBuffer.from_value(raw), IBuffer) + assert_type(IBuffer.from_bytes(b'data'), IBuffer) + assert_type(buffer.as_interface(IBuffer), IBuffer) + assert_type(TaggedBuffer.from_value(raw), TaggedBuffer) + # Only the native QueryInterface can decide whether this raw value is an IBuffer. + assert_type(IBuffer.from_value(uri._obj), IBuffer) +"#; + +const INVALID_CONSUMER: &str = r#" +from PY_PACKAGE.windows.foundation import Uri +from PY_PACKAGE.windows.storage.streams import IBuffer + +def misuse(uri: Uri) -> None: + IBuffer.from_value(uri) + uri.as_interface(Uri) + IBuffer(None) +"#; + +#[test] +fn interface_projection_typing_keeps_raw_values_explicit() { + let Some(generated) = Generated::new() else { + return; + }; + let available = Command::new(generated.python()) + .args(["-m", "mypy", "--version"]) + .output() + .is_ok_and(|output| output.status.success()); + assert!( + available || std::env::var("DYNWINRT_REQUIRE_MYPY").as_deref() != Ok("1"), + "strict interface constructor typing requires mypy" + ); + if !available { + eprintln!("Skipping constructor typing: mypy not installed"); + return; + } + let repo = generated.root.parent().unwrap().parent().unwrap(); + for (source, errors) in [(VALID_CONSUMER, 0), (INVALID_CONSUMER, 4)] { + let source = source.replace("PY_PACKAGE", &generated.package); + let mut command = Command::new(generated.python()); + command.args([ + "-m", + "mypy", + "--strict", + "--no-incremental", + "--no-pretty", + "--cache-dir", + ]); + command.arg(generated.root.join("mypy-cache")); + command + .args(["-c", &source]) + .current_dir(generated.root.parent().unwrap()); + command.env( + "MYPYPATH", + std::env::join_paths([repo.join(r"bindings\py"), repo.join("target")]) + .expect("MYPYPATH"), + ); + let output = command.output().expect("run mypy"); + let diagnostics = format!( + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + diagnostics.matches(": error:").count(), + errors, + "{diagnostics}" + ); + assert_eq!(output.status.success(), errors == 0, "{diagnostics}"); + } + if let Some(pyright) = std::env::var_os("DYNWINRT_PYRIGHT") { + for (file, source, errors) in [ + ("valid.py", VALID_CONSUMER, 0), + ("invalid.py", INVALID_CONSUMER, 3), + ] { + let path = generated.root.join(file); + fs::write( + &path, + format!( + "# pyright: strict\n{}", + source.replace("PY_PACKAGE", &generated.package) + ), + ) + .unwrap(); + let output = Command::new(&pyright) + .args(["--pythonpath"]) + .arg(generated.python()) + .arg(&path) + .env("PYTHONPATH", generated.root.parent().unwrap()) + .output() + .expect("run pyright"); + let diagnostics = format!( + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + if errors == 0 { + assert!(!diagnostics.contains(" - error: "), "{diagnostics}"); + } else { + for line in [7, 8, 9] { + assert!( + diagnostics + .lines() + .any(|entry| entry.contains(&format!("invalid.py:{line}:")) + && entry.contains(" - error: ")), + "{diagnostics}" + ); + } + } + assert_eq!(output.status.success(), errors == 0, "{diagnostics}"); + } + } +} diff --git a/tools/dynwinrt-codegen/tests/python_json_native_null_test.rs b/tools/dynwinrt-codegen/tests/python_json_native_null_test.rs index a1d493eb..1aea198d 100644 --- a/tools/dynwinrt-codegen/tests/python_json_native_null_test.rs +++ b/tools/dynwinrt-codegen/tests/python_json_native_null_test.rs @@ -188,8 +188,12 @@ fn stock_json_mutators_fail_before_native_mutation_but_custom_generics_keep_null let script = r#" import operator from dynwinrt import DynWinRTArray, DynWinRTType, DynWinRTValue, RoApartment, release_projected -from JSON_PACKAGE.windows__data__json__json_array import JsonArray, IVector_IJsonValue, IID_IJsonValue -from JSON_PACKAGE.windows__data__json__json_object import JsonObject, IMap_String_IJsonValue +from JSON_PACKAGE.windows__data__json__json_array import ( + JsonArray, IVector_IJsonValue, IID_IJsonValue, IID_IVector_IJsonValue, +) +from JSON_PACKAGE.windows__data__json__json_object import ( + JsonObject, IMap_String_IJsonValue, IID_IMap_String_IJsonValue, +) from JSON_PACKAGE.windows__data__json__json_value import JsonValue def rejected_without_mutation(receiver, mutation): @@ -263,6 +267,10 @@ with RoApartment(): generic_vector = IVector_IJsonValue.from_value( DynWinRTValue.create_vector([native_null], element) ) + borrowed_vector = generic_vector._obj.cast(IID_IVector_IJsonValue) + assert IVector_IJsonValue(borrowed_vector) is generic_vector + assert not borrowed_vector.is_released() + borrowed_vector.release() assert generic_vector[0] is None generic_vector.append(None) generic_vector.replace_all(array_of_null) @@ -283,6 +291,10 @@ with RoApartment(): element, ) ) + borrowed_map = generic_map._obj.cast(IID_IMap_String_IJsonValue) + assert IMap_String_IJsonValue(borrowed_map) is generic_map + assert not borrowed_map.is_released() + borrowed_map.release() generic_map.update({'next': None}) assert generic_map['original'] is None and generic_map['next'] is None try: From f1bb5169adb8343c786b2f36d312051a82073b76 Mon Sep 17 00:00:00 2001 From: Leilei Zhang Date: Tue, 29 Sep 2026 18:30:11 +0800 Subject: [PATCH 04/15] Reject Async receivers for Python low-level convenience calls Require an Object before call_0/call_1 can prepare or dispatch a caller-specified native signature. Exercise pre-dispatch rejection and valid IID-cast low-level calls in an isolated Python process. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- bindings/py/README.md | 7 ++ bindings/py/src/runtime.rs | 4 +- .../py/tests/test_async_receiver_guard.py | 95 +++++++++++++++++++ 3 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 bindings/py/tests/test_async_receiver_guard.py diff --git a/bindings/py/README.md b/bindings/py/README.md index 803ffc0c..9fa24bf4 100644 --- a/bindings/py/README.md +++ b/bindings/py/README.md @@ -237,6 +237,13 @@ def work(action: DynWinRTValue) -> None: operation = ThreadPool.run_async(work) ``` +The low-level `DynWinRTValue.call_0()` and `call_1()` helpers require an +Object holding the intended interface, not a raw Async value. They reject an +Async receiver with `RuntimeError` before native dispatch: its `IAsyncInfo` +identity is not proof of the caller-supplied vtable slot and signature. Cast +to the specific interface IID first when making a metadata-checked low-level +call, or use the generated async wrapper and its `wait()`/await API. + `ThreadPool.run_async(handler)` intentionally retains its original single argument and exact callback annotation, so mypy can infer the type of an unannotated callback lambda. Its priority and options overloads remain diff --git a/bindings/py/src/runtime.rs b/bindings/py/src/runtime.rs index eca11a90..f8b5a7a9 100644 --- a/bindings/py/src/runtime.rs +++ b/bindings/py/src/runtime.rs @@ -1831,10 +1831,10 @@ impl DynWinRTValue { method_index: usize, return_type: &DynWinRTType, ) -> PyResult> { + let obj_raw = self.receiver("call_0()")?.as_raw(); let method = dynwinrt::MethodSignature::new(&*TABLE) .add_out(return_type.0.clone()) .build(method_index); - let obj_raw = self.com_receiver("call_0()")?.as_raw(); let result = method .call_dynamic(obj_raw, &[]) .map_err(map_windows_error)?; @@ -1852,7 +1852,7 @@ impl DynWinRTValue { return_type: &DynWinRTType, v1: &DynWinRTValue, ) -> PyResult> { - let obj_raw = self.com_receiver("call_1()")?.as_raw(); + let obj_raw = self.receiver("call_1()")?.as_raw(); v1.check_input("call_1()", InputSlot::Argument(0))?; let in_type = TABLE.handle_from_kind(v1.0.get_type_kind()); let method = dynwinrt::MethodSignature::new(&*TABLE) diff --git a/bindings/py/tests/test_async_receiver_guard.py b/bindings/py/tests/test_async_receiver_guard.py new file mode 100644 index 00000000..0598f012 --- /dev/null +++ b/bindings/py/tests/test_async_receiver_guard.py @@ -0,0 +1,95 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Run unsafe-if-regressed receiver checks outside the pytest process.""" + +import subprocess +import sys + + +def test_raw_async_cannot_receive_unchecked_convenience_calls(tmp_path): + script = r""" +import sys +from dynwinrt import ( + DynWinRTMethodSig, DynWinRTType, DynWinRTValue, RoApartment, WinGUID, +) + +storage_iid = WinGUID.parse('fa3f6186-4214-428c-a64c-14c9ac7315ea') +statics_iid = WinGUID.parse('5984c710-daf2-43c8-8bb4-a4d3eacfd03f') +async_info_iid = WinGUID.parse('00000036-0000-0000-c000-000000000046') +uri_factory_iid = WinGUID.parse('44a9796f-723e-4fdf-a218-033e75b0c084') + +with RoApartment(): + storage_type = DynWinRTType.runtime_class( + 'Windows.Storage.StorageFile', + DynWinRTType.interface(storage_iid), + ) + statics = DynWinRTType.register_interface( + 'IStorageFileStaticsAsyncReceiverGuard', statics_iid, + ).add_method( + 'GetFileFromPathAsync', + DynWinRTMethodSig() + .add_in(DynWinRTType.hstring()) + .add_out(DynWinRTType.i_async_operation(storage_type)), + ) + factory = DynWinRTValue.activation_factory( + 'Windows.Storage.StorageFile' + ).cast(statics_iid) + operation = statics.method(6).invoke( + factory, [DynWinRTValue.from_hstring(sys.argv[1])] + ) + + released_arg = DynWinRTValue.from_hstring('not sent to native code') + released_arg.release() + for operation_name, invoke in ( + ('call_1()', lambda: operation.call_1( + 6, DynWinRTType.object(), released_arg, + )), + ('call_0()', lambda: operation.call_0(6, DynWinRTType.u32_type())), + ): + try: + invoke() + except RuntimeError as error: + assert str(error) == ( + f'{operation_name} requires an Object value, got Async' + ), error + else: + raise AssertionError(f'{operation_name} accepted an Async receiver') + + live_arg = DynWinRTValue.from_hstring('still not sent to native code') + try: + operation.call_1(6, DynWinRTType.object(), live_arg) + except RuntimeError as error: + assert str(error) == 'call_1() requires an Object value, got Async' + else: + raise AssertionError('call_1() accepted an Async receiver with a live argument') + + # An explicit IID cast remains a normal low-level Object receiver. + info = operation.cast(async_info_iid) + assert info.call_0(6, DynWinRTType.u32_type()).to_u32() >= 0 + + uri_factory = DynWinRTValue.activation_factory( + 'Windows.Foundation.Uri' + ).cast(uri_factory_iid) + uri = uri_factory.call_1( + 6, DynWinRTType.object(), + DynWinRTValue.from_hstring('https://example.com/checked'), + ) + assert not uri.is_null() + uri.release() + uri_factory.release() + info.release() + operation.cancel() + operation.release() + factory.release() +print('async-receiver-rejected-before-dispatch', flush=True) +""" + result = subprocess.run( + [sys.executable, "-B", "-c", script, str(tmp_path / "missing-file")], + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, (result.returncode, result.stdout, result.stderr) + assert "async-receiver-rejected-before-dispatch" in result.stdout From 91046c180fe184a98ac9d80f1fa9a9ce3453d578 Mon Sep 17 00:00:00 2001 From: Leilei Zhang Date: Tue, 29 Sep 2026 19:43:13 +0800 Subject: [PATCH 05/15] Avoid rooting temporary raw COM values in lifetime scopes Observe raw outputs weakly so temporary casts and callback inputs drop without accumulating native references; keep projected wrappers strongly tracked and release surviving raw owners before apartment shutdown. Refresh seven generated Python snapshots for the interface-IID constructor change. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- bindings/py/README.md | 5 +++- bindings/py/src/lib.rs | 24 +++++++++++++++++ bindings/py/src/runtime.rs | 20 +++++++++++--- bindings/py/tests/test_phase1.py | 26 ++++++++++++++++++- .../snapshots/data_writer_py/data_writer.py | 2 +- ...i_iterator_i_www_form_url_decoder_entry.py | 2 +- .../tests/snapshots/uri_py/i_stringable.py | 4 +-- ...ntime_class_with_absolute_canonical_uri.py | 4 +-- .../uri_py/i_www_form_url_decoder_entry.py | 4 +-- .../tests/snapshots/uri_py/uri.py | 4 +-- .../snapshots/uri_py/www_form_url_decoder.py | 4 +-- 11 files changed, 81 insertions(+), 18 deletions(-) diff --git a/bindings/py/README.md b/bindings/py/README.md index 9fa24bf4..1390e2c0 100644 --- a/bindings/py/README.md +++ b/bindings/py/README.md @@ -774,7 +774,10 @@ function that created it, but after the scope closes it reports `is_released() == True`; it cannot be used outside the apartment. Without a scope, explicitly call `release()` on every retained native result before leaving `RoApartment`. Pure scalar results do not own COM references and remain -usable after a scope closes. +usable after a scope closes. The scope retains projected wrappers strongly but +observes raw native outputs weakly: temporary casts and callback inputs can +drop normally instead of accumulating references until the scope closes. +Raw results still held by Python at scope exit are released there. Scopes nest in LIFO order. Wrappers and raw native results that survive a closed scope remain Python objects, but their COM references are released: diff --git a/bindings/py/src/lib.rs b/bindings/py/src/lib.rs index 2a5f779c..14cb9a6c 100644 --- a/bindings/py/src/lib.rs +++ b/bindings/py/src/lib.rs @@ -177,6 +177,7 @@ def _dynwinrt_projected_from_native( class ProjectedLifetimeScope: def __init__(self): self._registry = {} + self._native_refs = _WeakValueDictionary() self._token = None self._owner_thread = None self._active = False @@ -225,9 +226,18 @@ class ProjectedLifetimeScope: if not self._active or self._disposed: raise RuntimeError('Cannot track values in an inactive projection lifetime scope.') for native in _dynwinrt_projected_native_values(value): + self._native_refs.pop(id(native), None) self._registry.setdefault(id(native), (native, type_name)) return value + def track_native(self, value): + self._require_owner_thread('track values in') + if not self._active or self._disposed: + raise RuntimeError('Cannot track values in an inactive projection lifetime scope.') + if id(value) not in self._registry: + self._native_refs[id(value)] = value + return value + def close(self): self._require_owner_thread('close') if self._disposed: @@ -250,6 +260,13 @@ class ProjectedLifetimeScope: except BaseException as error: if first_error is None: first_error = error + for key, native in reversed(list(self._native_refs.items())): + try: + native.release() + del self._native_refs[key] + except BaseException as error: + if first_error is None: + first_error = error if first_error is not None: self._retry_pending = True raise first_error @@ -298,6 +315,12 @@ def _dynwinrt_track_projected(value, type_name=None): scope.track(value, type_name) return value +def _dynwinrt_track_native(value): + scope = _active_projected_lifetime_scope.get() + if scope is not None and scope._active and not scope._disposed: + scope.track_native(value) + return value + def project_as(value, wrapper_type): '''Borrow a projected value and expose it as a generated runtime class. @@ -682,6 +705,7 @@ for _name in ( None, )?; + super::runtime::init_native_tracking(m)?; Ok(()) } } diff --git a/bindings/py/src/runtime.rs b/bindings/py/src/runtime.rs index f8b5a7a9..32fad1b1 100644 --- a/bindings/py/src/runtime.rs +++ b/bindings/py/src/runtime.rs @@ -6,6 +6,7 @@ use std::sync::{Arc, Mutex}; use dynwinrt; use pyo3::exceptions::{PyIndexError, PyOverflowError, PyRuntimeError, PyTypeError}; use pyo3::prelude::*; +use pyo3::sync::PyOnceLock; use pyo3::types::PyDict; use windows::Win32::System::WinRT::{ RO_INIT_MULTITHREADED, RO_INIT_SINGLETHREADED, RO_INIT_TYPE, RoInitialize, @@ -1103,10 +1104,19 @@ impl DynWinRTMethodHandle { // DynWinRTValue — main value container // ====================================================================== -#[pyclass(from_py_object)] +#[pyclass(from_py_object, weakref)] #[derive(Clone)] pub struct DynWinRTValue(pub(crate) dynwinrt::WinRTValue, Lifecycle); +static TRACK_NATIVE: PyOnceLock> = PyOnceLock::new(); + +pub(crate) fn init_native_tracking(module: &Bound<'_, PyModule>) -> PyResult<()> { + TRACK_NATIVE.get_or_try_init(module.py(), || { + Ok::, PyErr>(module.getattr("_dynwinrt_track_native")?.unbind()) + })?; + Ok(()) +} + fn contains_com_references(typ: &dynwinrt::TypeHandle) -> bool { let kind = typ.kind(); kind.is_com_pointer() @@ -1131,9 +1141,11 @@ pub(crate) fn tracked_native_value( }; let output = Py::new(py, DynWinRTValue::new(value))?; if owns_native { - py.import("dynwinrt.dynwinrt")? - .getattr("_dynwinrt_track_projected")? - .call1((output.clone_ref(py), "DynWinRTValue"))?; + // Embedding tests may create values before the extension module (and + // therefore any projected lifetime scope) has been initialized. + if let Some(track) = TRACK_NATIVE.get(py) { + track.call1(py, (output.clone_ref(py),))?; + } } Ok(output) } diff --git a/bindings/py/tests/test_phase1.py b/bindings/py/tests/test_phase1.py index 42439a31..3cea9182 100644 --- a/bindings/py/tests/test_phase1.py +++ b/bindings/py/tests/test_phase1.py @@ -1232,6 +1232,9 @@ def test_projected_lifetime_scope_releases_native_values_before_apartment_exit() _dynwinrt_track_projected(SimpleNamespace(_obj=second), "UriFactory") assert not first.is_null() assert not second.is_null() + assert id(first) in scope._registry + assert id(second) in scope._registry + assert not scope._native_refs assert scope.disposed assert first.is_null() @@ -1242,21 +1245,42 @@ def test_projected_lifetime_scope_releases_native_values_before_apartment_exit() def test_projected_lifetime_scope_tracks_raw_native_outputs_automatically(): - with RoApartment(1), projected_lifetime_scope(): + with RoApartment(1), projected_lifetime_scope() as scope: factory = DynWinRTValue.activation_factory("Windows.Foundation.Uri") cast = factory.cast(WinGUID.parse(IID_IURI_FACTORY)) scalar = DynWinRTValue.from_u32(8080) assert not factory.is_released() assert not cast.is_released() + assert not scope._registry + assert id(factory) in scope._native_refs + assert id(cast) in scope._native_refs assert factory.is_released() assert cast.is_released() + assert not scope._native_refs assert scalar.to_u32() == 8080 assert not scalar.is_released() with pytest.raises(RuntimeError, match="released"): cast.identity_raw() +def test_projected_lifetime_scope_does_not_root_temporary_native_results(): + with RoApartment(1), projected_lifetime_scope() as scope: + temporary = DynWinRTValue.activation_factory("Windows.Foundation.Uri") + reference = weakref.ref(temporary) + assert id(temporary) in scope._native_refs + del temporary + gc.collect() + assert reference() is None + assert not scope._native_refs + + retained = DynWinRTValue.activation_factory("Windows.Foundation.Uri") + assert id(retained) in scope._native_refs + assert not retained.is_released() + assert retained.is_released() + assert not scope._native_refs + + def test_projected_lifetime_scope_enforces_lifo_order(): outer = projected_lifetime_scope() inner = projected_lifetime_scope() diff --git a/tools/dynwinrt-codegen/tests/snapshots/data_writer_py/data_writer.py b/tools/dynwinrt-codegen/tests/snapshots/data_writer_py/data_writer.py index 42b53708..6e9331c0 100644 --- a/tools/dynwinrt-codegen/tests/snapshots/data_writer_py/data_writer.py +++ b/tools/dynwinrt-codegen/tests/snapshots/data_writer_py/data_writer.py @@ -287,7 +287,7 @@ class IClosable: _dynwinrt_interface_iid = IID_IClosable def __new__(cls, *args, **kwargs): if len(args) == 1 and not kwargs and isinstance(args[0], DynWinRTValue): - return _dynwinrt_projected_from_native(cls, args[0], '_set_native') + return _dynwinrt_projected_from_native(cls, args[0], '_set_native', release_redundant=False) return super().__new__(cls) def _set_native(self, obj: DynWinRTValue): diff --git a/tools/dynwinrt-codegen/tests/snapshots/uri_py/i_iterator_i_www_form_url_decoder_entry.py b/tools/dynwinrt-codegen/tests/snapshots/uri_py/i_iterator_i_www_form_url_decoder_entry.py index f6494623..6bf31f02 100644 --- a/tools/dynwinrt-codegen/tests/snapshots/uri_py/i_iterator_i_www_form_url_decoder_entry.py +++ b/tools/dynwinrt-codegen/tests/snapshots/uri_py/i_iterator_i_www_form_url_decoder_entry.py @@ -43,7 +43,7 @@ def implement(handlers, *additional): _dynwinrt_interface_iid = IID_IIterator_IWwwFormUrlDecoderEntry def __new__(cls, *args, **kwargs): if len(args) == 1 and not kwargs and isinstance(args[0], DynWinRTValue): - return _dynwinrt_projected_from_native(cls, args[0], '_set_native') + return _dynwinrt_projected_from_native(cls, args[0], '_set_native', release_redundant=False) return super().__new__(cls) def _set_native(self, obj: DynWinRTValue, *, cache=True): diff --git a/tools/dynwinrt-codegen/tests/snapshots/uri_py/i_stringable.py b/tools/dynwinrt-codegen/tests/snapshots/uri_py/i_stringable.py index 1de16db4..07197a7a 100644 --- a/tools/dynwinrt-codegen/tests/snapshots/uri_py/i_stringable.py +++ b/tools/dynwinrt-codegen/tests/snapshots/uri_py/i_stringable.py @@ -89,11 +89,11 @@ def from_implementation(cls, owner: DynWinRTImplementation | DynWinRTImplementat _dynwinrt_interface_iid = IID_IStringable def __new__(cls, *args, **kwargs): if len(args) == 1 and not kwargs and isinstance(args[0], DynWinRTValue): - return _dynwinrt_projected_from_native(cls, args[0], '_set_native') + return _dynwinrt_projected_from_native(cls, args[0], '_set_native', release_redundant=False) return super().__new__(cls) def _set_native(self, obj: DynWinRTValue, *, cache=True): - self._obj = obj + self._obj = obj.cast(IID_IStringable) self._dynwinrt_native_ready = True _dynwinrt_track_projected(self, 'Windows.Foundation.IStringable') if cache: diff --git a/tools/dynwinrt-codegen/tests/snapshots/uri_py/i_uri_runtime_class_with_absolute_canonical_uri.py b/tools/dynwinrt-codegen/tests/snapshots/uri_py/i_uri_runtime_class_with_absolute_canonical_uri.py index c343c075..e8ab2803 100644 --- a/tools/dynwinrt-codegen/tests/snapshots/uri_py/i_uri_runtime_class_with_absolute_canonical_uri.py +++ b/tools/dynwinrt-codegen/tests/snapshots/uri_py/i_uri_runtime_class_with_absolute_canonical_uri.py @@ -98,11 +98,11 @@ def from_implementation(cls, owner: DynWinRTImplementation | DynWinRTImplementat _dynwinrt_interface_iid = IID_IUriRuntimeClassWithAbsoluteCanonicalUri def __new__(cls, *args, **kwargs): if len(args) == 1 and not kwargs and isinstance(args[0], DynWinRTValue): - return _dynwinrt_projected_from_native(cls, args[0], '_set_native') + return _dynwinrt_projected_from_native(cls, args[0], '_set_native', release_redundant=False) return super().__new__(cls) def _set_native(self, obj: DynWinRTValue, *, cache=True): - self._obj = obj + self._obj = obj.cast(IID_IUriRuntimeClassWithAbsoluteCanonicalUri) self._dynwinrt_native_ready = True _dynwinrt_track_projected(self, 'Windows.Foundation.IUriRuntimeClassWithAbsoluteCanonicalUri') if cache: diff --git a/tools/dynwinrt-codegen/tests/snapshots/uri_py/i_www_form_url_decoder_entry.py b/tools/dynwinrt-codegen/tests/snapshots/uri_py/i_www_form_url_decoder_entry.py index ed4851fa..5eb3005e 100644 --- a/tools/dynwinrt-codegen/tests/snapshots/uri_py/i_www_form_url_decoder_entry.py +++ b/tools/dynwinrt-codegen/tests/snapshots/uri_py/i_www_form_url_decoder_entry.py @@ -98,11 +98,11 @@ def from_implementation(cls, owner: DynWinRTImplementation | DynWinRTImplementat _dynwinrt_interface_iid = IID_IWwwFormUrlDecoderEntry def __new__(cls, *args, **kwargs): if len(args) == 1 and not kwargs and isinstance(args[0], DynWinRTValue): - return _dynwinrt_projected_from_native(cls, args[0], '_set_native') + return _dynwinrt_projected_from_native(cls, args[0], '_set_native', release_redundant=False) return super().__new__(cls) def _set_native(self, obj: DynWinRTValue, *, cache=True): - self._obj = obj + self._obj = obj.cast(IID_IWwwFormUrlDecoderEntry) self._dynwinrt_native_ready = True _dynwinrt_track_projected(self, 'Windows.Foundation.IWwwFormUrlDecoderEntry') if cache: diff --git a/tools/dynwinrt-codegen/tests/snapshots/uri_py/uri.py b/tools/dynwinrt-codegen/tests/snapshots/uri_py/uri.py index 83ae20d3..ac50edf4 100644 --- a/tools/dynwinrt-codegen/tests/snapshots/uri_py/uri.py +++ b/tools/dynwinrt-codegen/tests/snapshots/uri_py/uri.py @@ -222,7 +222,7 @@ class IUriRuntimeClassWithAbsoluteCanonicalUri: _dynwinrt_interface_iid = IID_IUriRuntimeClassWithAbsoluteCanonicalUri def __new__(cls, *args, **kwargs): if len(args) == 1 and not kwargs and isinstance(args[0], DynWinRTValue): - return _dynwinrt_projected_from_native(cls, args[0], '_set_native') + return _dynwinrt_projected_from_native(cls, args[0], '_set_native', release_redundant=False) return super().__new__(cls) def _set_native(self, obj: DynWinRTValue): @@ -261,7 +261,7 @@ class IStringable: _dynwinrt_interface_iid = IID_IStringable def __new__(cls, *args, **kwargs): if len(args) == 1 and not kwargs and isinstance(args[0], DynWinRTValue): - return _dynwinrt_projected_from_native(cls, args[0], '_set_native') + return _dynwinrt_projected_from_native(cls, args[0], '_set_native', release_redundant=False) return super().__new__(cls) def _set_native(self, obj: DynWinRTValue): diff --git a/tools/dynwinrt-codegen/tests/snapshots/uri_py/www_form_url_decoder.py b/tools/dynwinrt-codegen/tests/snapshots/uri_py/www_form_url_decoder.py index 15657987..789d3d4f 100644 --- a/tools/dynwinrt-codegen/tests/snapshots/uri_py/www_form_url_decoder.py +++ b/tools/dynwinrt-codegen/tests/snapshots/uri_py/www_form_url_decoder.py @@ -120,7 +120,7 @@ class IVectorView_IWwwFormUrlDecoderEntry(_WinRTSequenceMixin): _dynwinrt_interface_iid = IID_IVectorView_IWwwFormUrlDecoderEntry def __new__(cls, *args, **kwargs): if len(args) == 1 and not kwargs and isinstance(args[0], DynWinRTValue): - return _dynwinrt_projected_from_native(cls, args[0], '_set_native') + return _dynwinrt_projected_from_native(cls, args[0], '_set_native', release_redundant=False) return super().__new__(cls) def _set_native(self, obj: DynWinRTValue): @@ -166,7 +166,7 @@ class IIterable_IWwwFormUrlDecoderEntry(_WinRTIterableMixin): _dynwinrt_interface_iid = IID_IIterable_IWwwFormUrlDecoderEntry def __new__(cls, *args, **kwargs): if len(args) == 1 and not kwargs and isinstance(args[0], DynWinRTValue): - return _dynwinrt_projected_from_native(cls, args[0], '_set_native') + return _dynwinrt_projected_from_native(cls, args[0], '_set_native', release_redundant=False) return super().__new__(cls) def _set_native(self, obj: DynWinRTValue): From 9170cae08a3bf808f1ae5f659e3026b00f644c70 Mon Sep 17 00:00:00 2001 From: Leilei Zhang Date: Tue, 29 Sep 2026 20:04:14 +0800 Subject: [PATCH 06/15] Declare native lifetime tracking in Python stubs Match ProjectedLifetimeScope.track_native in the packaged Python stub and test the installed wheel surface, fixing hosted mypy.stubtest parity without changing native lifetime behavior. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- bindings/py/dynwinrt.pyi | 1 + bindings/py/tests/test_typing_package.py | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/bindings/py/dynwinrt.pyi b/bindings/py/dynwinrt.pyi index 70500be2..1f621b84 100644 --- a/bindings/py/dynwinrt.pyi +++ b/bindings/py/dynwinrt.pyi @@ -122,6 +122,7 @@ class ProjectedLifetimeScope: def track( self, value: _Tracked, type_name: Optional[str] = ... ) -> _Tracked: ... + def track_native(self, value: "DynWinRTValue") -> "DynWinRTValue": ... def close(self) -> None: ... diff --git a/bindings/py/tests/test_typing_package.py b/bindings/py/tests/test_typing_package.py index b838bf21..333defa5 100644 --- a/bindings/py/tests/test_typing_package.py +++ b/bindings/py/tests/test_typing_package.py @@ -32,3 +32,15 @@ def test_wheel_exports_typed_implementation_surface(): signature = inspect.signature(dynwinrt.DynWinRTInterfacePlan.create) assert signature.parameters["required_iids"].default == () assert "def from_hresult(" in stub + + +def test_wheel_stubs_raw_native_scope_tracking(): + stub = (Path(dynwinrt.__file__).parent / "__init__.pyi").read_text(encoding="utf-8") + scope = stub.split("class ProjectedLifetimeScope:", 1)[1].split( + "\ndef projected_lifetime_scope()", 1 + )[0] + assert 'def track_native(self, value: "DynWinRTValue") -> "DynWinRTValue": ...' in scope + assert tuple(inspect.signature(dynwinrt.ProjectedLifetimeScope.track_native).parameters) == ( + "self", + "value", + ) From 5b6b2ea665b712d7dc0ffbce9b5cf28f619e3657 Mon Sep 17 00:00:00 2001 From: Leilei Zhang Date: Tue, 29 Sep 2026 20:23:04 +0800 Subject: [PATCH 07/15] Model temporary COM cast drops in Python projection fixture Match DynWinRTValue's owned-reference drop in the projection-only test double and accept the checked-constructor cache argument. Assert that an unconsumed temporary returns the native owner's reference count to baseline. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../dynwinrt-codegen/tests/winrt_implementation_test.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tools/dynwinrt-codegen/tests/winrt_implementation_test.rs b/tools/dynwinrt-codegen/tests/winrt_implementation_test.rs index 9aa88922..775b7d90 100644 --- a/tools/dynwinrt-codegen/tests/winrt_implementation_test.rs +++ b/tools/dynwinrt-codegen/tests/winrt_implementation_test.rs @@ -815,6 +815,9 @@ class Value: self.releases += 1 if self.kind == 'object' and hasattr(self.value, 'owner'): self.value.owner.references -= 1 self.kind, self.value = 'null', None + def __del__(self): + # Native DynWinRTValue drops release an unconsumed COM reference. + if getattr(self, 'kind', None) == 'object': self.release() def invoke_delegate(self, iid, signature, args): if iid == '41c64fe4-5f4d-4cf8-8a39-c8e2a9f396a1': assert [kind for kind, typ in signature.parameters] == ['Out'] @@ -888,7 +891,7 @@ class Array: def to_i32_list(self): return [value.value for value in self.values] runtime = types.ModuleType('generated._runtime') runtime.__getattr__ = lambda name: None -def from_native(cls, obj, setter): +def from_native(cls, obj, setter, *, release_redundant=True): instance = object.__new__(cls) getattr(instance, setter)(obj) return instance @@ -1006,6 +1009,10 @@ class LargeInteger(Handlers): raises('invalid implementation result', lambda: Contract.implementation(LargeInteger()).dispatch(8,[values,Value.from_u32(1)])) handle = Contract.implement(handlers) owner = Implementation.last +probe = Value('object', types.SimpleNamespace(owner=owner)) +assert owner.references == 2 +del probe +assert owner.references == 1 assert owner.callback(0,6,[])[0].value == 'changed' raises('Unknown implementation interface index', lambda: owner.callback(-1,6,[])) view = Contract.from_implementation(owner) From 0d97f69817d1ef81ea09c399e83ad1172d0f4244 Mon Sep 17 00:00:00 2001 From: Leilei Zhang Date: Tue, 29 Sep 2026 20:47:31 +0800 Subject: [PATCH 08/15] Match stock JSON Python annotations to native null rules Make stock JsonArray/JsonObject runtime method inputs non-null in both normal and --no-pyi output, keeping generic IVector/IMap nullable. Verify generated source and inspect.signature contracts without changing JS/TS projections. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- bindings/py/README.md | 7 +- tools/dynwinrt-codegen/python/README.md | 3 +- .../codegen/winrt/python/generator/class.rs | 3 + .../codegen/winrt/python/generator/types.rs | 1 + .../src/codegen/winrt/python/method.rs | 14 +- .../src/codegen/winrt/python/type_helpers.rs | 87 +++++++---- .../tests/python_json_native_null_test.rs | 145 +++++++++++++++--- 7 files changed, 199 insertions(+), 61 deletions(-) diff --git a/bindings/py/README.md b/bindings/py/README.md index 1390e2c0..b64255e2 100644 --- a/bindings/py/README.md +++ b/bindings/py/README.md @@ -72,9 +72,10 @@ and `IMap` views, raw `DynWinRTArray` inputs, `replace_all`, slice assignment, `extend` and `update`. A rejected null leaves the JSON collection unchanged. Use `JsonValue.create_null_value()` to store JSON **semantic** null; it is a non-null `IJsonValue` object. Their class stubs type -elements as non-null, while generic interface stubs retain `| None` because -custom implementations can store a native null and a view's origin is only -known at runtime. +elements as non-null, and the generated `.py` input annotations agree even +with `--no-pyi`. Generic interface annotations retain `| None` because custom +implementations can store a native null and a view's origin is only known at +runtime. Other arguments keep accepting `None` where they did before. The stubs are optimistic, like the generated TypeScript declarations: the runtime still diff --git a/tools/dynwinrt-codegen/python/README.md b/tools/dynwinrt-codegen/python/README.md index 4be41721..95efaf8a 100644 --- a/tools/dynwinrt-codegen/python/README.md +++ b/tools/dynwinrt-codegen/python/README.md @@ -166,7 +166,8 @@ returns `DynWinRTValue | None`. Sequence item assignment, slice assignment, and integer indices take one item and slices take an iterable of items. Generic nullable `collections.abc` contracts remain unchanged. The stock `JsonArray` and `JsonObject` classes instead have non-null `IJsonValue` -element/value contracts in their stubs. Their native implementations reject +element/value contracts in both stubs and runtime method input annotations +(including `--no-pyi` output). Their native implementations reject `None` (including through generic interface views) before mutation; use `JsonValue.create_null_value()` for JSON semantic null. A custom `IVector` or `IMap` may still store a native null. diff --git a/tools/dynwinrt-codegen/src/codegen/winrt/python/generator/class.rs b/tools/dynwinrt-codegen/src/codegen/winrt/python/generator/class.rs index a68c0c1a..05bb189e 100644 --- a/tools/dynwinrt-codegen/src/codegen/winrt/python/generator/class.rs +++ b/tools/dynwinrt-codegen/src/codegen/winrt/python/generator/class.rs @@ -42,6 +42,7 @@ pub fn generate_class<'a>( let context = context.as_ref(); let collection_iface = class_interface(class); let collection_kind = collection_iface.and_then(interface_kind); + let stock_json_receiver = stock_json_class_contract(class).is_some(); let non_null_json = collection_iface .zip(collection_kind) .and_then(|(iface, kind)| non_null_json_collection(kind, &iface.generic_args)); @@ -645,6 +646,7 @@ pub fn generate_class<'a>( .name .strip_prefix("put_") .is_some_and(|suffix| property_getters.contains(suffix)), + stock_json_receiver, } }; // Python evaluates decorators while building the class. Emit every getter @@ -907,6 +909,7 @@ pub fn generate_class<'a>( .iter() .any(|candidate| candidate.name == format!("get_{suffix}")) }), + stock_json_receiver: false, }; let members = reorder_getters_before_setters(&req_iface.methods) .into_iter() diff --git a/tools/dynwinrt-codegen/src/codegen/winrt/python/generator/types.rs b/tools/dynwinrt-codegen/src/codegen/winrt/python/generator/types.rs index 11719432..04fe0424 100644 --- a/tools/dynwinrt-codegen/src/codegen/winrt/python/generator/types.rs +++ b/tools/dynwinrt-codegen/src/codegen/winrt/python/generator/types.rs @@ -645,6 +645,7 @@ pub fn generate_interface(context: &PythonProjectionContext, iface: &InterfaceMe .iter() .any(|candidate| candidate.name == format!("get_{suffix}")) }), + stock_json_receiver: false, }; let members = reorder_getters_before_setters(&iface.methods) .into_iter() diff --git a/tools/dynwinrt-codegen/src/codegen/winrt/python/method.rs b/tools/dynwinrt-codegen/src/codegen/winrt/python/method.rs index e05a9d7c..44b8fa98 100644 --- a/tools/dynwinrt-codegen/src/codegen/winrt/python/method.rs +++ b/tools/dynwinrt-codegen/src/codegen/winrt/python/method.rs @@ -25,6 +25,7 @@ use super::signature::{ use super::type_helpers::{ method_pydoc, py_factory_return_type, py_method_abi_output_count, py_method_outputs, py_method_return_type, py_property_type, py_runtime_method_param_list, + py_runtime_method_param_list_for_receiver, }; fn is_delegate_type(typ: &TypeMeta, context: &PythonProjectionContext) -> bool { @@ -579,6 +580,7 @@ pub(crate) struct InstanceOverload<'a> { pub(crate) method: &'a MethodMeta, pub(crate) sibling_methods: Option<&'a [MethodMeta]>, pub(crate) property_has_getter: bool, + pub(crate) stock_json_receiver: bool, } /// Render an instance accessor (property or event method). @@ -594,6 +596,7 @@ pub(crate) fn generate_instance_accessor( None, overload.sibling_methods, overload.property_has_getter, + overload.stock_json_receiver, ) } @@ -627,6 +630,7 @@ pub(crate) fn generate_instance_method_group<'a>( Some(attribute), overload.sibling_methods, overload.property_has_getter, + overload.stock_json_receiver, )); } if overloads.len() == 1 { @@ -796,6 +800,7 @@ pub(crate) fn generate_method_body( name_override: Option<&str>, sibling_methods: Option<&[MethodMeta]>, property_has_getter: bool, + stock_json_receiver: bool, ) -> String { let in_params = get_in_params(method); let return_type = method.return_type.as_ref(); @@ -950,7 +955,8 @@ pub(crate) fn generate_method_body( iface_var, method.vtable_index, obj_expr, arg )); } else { - let py_params = py_runtime_method_param_list(method, context); + let py_params = + py_runtime_method_param_list_for_receiver(method, context, stock_json_receiver); let py_return = py_method_return_type(method, AnnotationSurface::Runtime, context); let method_name = name_override .map(|s| s.to_string()) @@ -1037,6 +1043,7 @@ mod tests { method: candidate.method, sibling_methods: None, property_has_getter: true, + stock_json_receiver: false, }, context, ) @@ -1116,6 +1123,7 @@ mod tests { method: candidate.method, sibling_methods: None, property_has_getter: true, + stock_json_receiver: false, }, &PythonProjectionContext::default(), ); @@ -1241,6 +1249,7 @@ mod tests { None, None, true, + false, ); assert!(code.contains("def load_async(self) -> WinRTCoroutine[int]:")); @@ -1276,6 +1285,7 @@ mod tests { None, None, true, + false, ); assert!(code.contains( @@ -1441,6 +1451,7 @@ print(Runner().run(DynWinRtDelegate())) None, Some(&siblings), true, + false, ); assert!(code.contains("def on_changed(self, callback:")); @@ -1492,6 +1503,7 @@ print(Runner().run(DynWinRtDelegate())) None, Some(std::slice::from_ref(&add)), true, + false, ); assert!(code.contains("'routed_event_handler', 'IID_RoutedEventHandler'")); diff --git a/tools/dynwinrt-codegen/src/codegen/winrt/python/type_helpers.rs b/tools/dynwinrt-codegen/src/codegen/winrt/python/type_helpers.rs index f607aa11..92c19725 100644 --- a/tools/dynwinrt-codegen/src/codegen/winrt/python/type_helpers.rs +++ b/tools/dynwinrt-codegen/src/codegen/winrt/python/type_helpers.rs @@ -758,6 +758,34 @@ pub(super) fn py_method_param_list( py_method_param_list_for_receiver(method, context, false) } +fn collection_input_param_type( + typ: &TypeMeta, + role: Option, + context: &PythonProjectionContext, + stock_json_receiver: bool, +) -> Option { + match role { + Some(role @ (CollectionInputRole::Element | CollectionInputRole::Value)) + if stock_json_receiver + && super::collections::non_null_json_input(role, typ).is_some() => + { + if let TypeMeta::Array(element) = typ { + Some(format!( + "DynWinRTArray | Sequence[{}]", + py_param_type_safe(element, context) + )) + } else { + Some(py_param_type_safe(typ, context)) + } + } + Some(CollectionInputRole::Element | CollectionInputRole::Value) => { + Some(py_collection_contract_input_type(typ, context)) + } + Some(CollectionInputRole::Key) => Some(py_collection_input_type(typ, context)), + None => None, + } +} + pub(super) fn py_method_param_list_for_receiver( method: &MethodMeta, context: &PythonProjectionContext, @@ -778,29 +806,15 @@ pub(super) fn py_method_param_list_for_receiver( .collection_inputs .iter() .find_map(|(parameter, role)| (*parameter == index).then_some(*role)); - let param_type = match role { - Some(role @ (CollectionInputRole::Element | CollectionInputRole::Value)) - if stock_json_receiver - && super::collections::non_null_json_input(role, ¶m.typ).is_some() => - { - if let TypeMeta::Array(element) = ¶m.typ { - format!( - "DynWinRTArray | Sequence[{}]", - py_param_type_safe(element, context) - ) - } else { - py_param_type_safe(¶m.typ, context) - } - } - Some(CollectionInputRole::Element | CollectionInputRole::Value) => { - py_collection_contract_input_type(¶m.typ, context) - } - Some(CollectionInputRole::Key) => py_collection_input_type(¶m.typ, context), - None if context.is_delegate_type(¶m.typ) => { - super::delegates::py_delegate_param_type(¶m.typ, context) - } - None => py_param_type_safe(¶m.typ, context), - }; + let param_type = + collection_input_param_type(¶m.typ, role, context, stock_json_receiver) + .unwrap_or_else(|| { + if context.is_delegate_type(¶m.typ) { + super::delegates::py_delegate_param_type(¶m.typ, context) + } else { + py_param_type_safe(¶m.typ, context) + } + }); format!("{}: {}", to_snake_case(¶m.name), param_type) }) .collect::>() @@ -812,6 +826,14 @@ pub(super) fn py_method_param_list_for_receiver( pub(super) fn py_runtime_method_param_list( method: &MethodMeta, context: &PythonProjectionContext, +) -> String { + py_runtime_method_param_list_for_receiver(method, context, false) +} + +pub(super) fn py_runtime_method_param_list_for_receiver( + method: &MethodMeta, + context: &PythonProjectionContext, + stock_json_receiver: bool, ) -> String { method .params @@ -828,16 +850,15 @@ pub(super) fn py_runtime_method_param_list( .collection_inputs .iter() .find_map(|(parameter, role)| (*parameter == index).then_some(*role)); - let param_type = match role { - Some(CollectionInputRole::Element | CollectionInputRole::Value) => { - py_collection_contract_input_type(¶m.typ, context) - } - Some(CollectionInputRole::Key) => py_collection_input_type(¶m.typ, context), - None if context.is_delegate_type(¶m.typ) => { - super::delegates::py_runtime_delegate_param_type().to_string() - } - None => py_param_type_safe(¶m.typ, context), - }; + let param_type = + collection_input_param_type(¶m.typ, role, context, stock_json_receiver) + .unwrap_or_else(|| { + if context.is_delegate_type(¶m.typ) { + super::delegates::py_runtime_delegate_param_type().to_string() + } else { + py_param_type_safe(¶m.typ, context) + } + }); format!("{}: {}", to_snake_case(¶m.name), param_type) }) .collect::>() diff --git a/tools/dynwinrt-codegen/tests/python_json_native_null_test.rs b/tools/dynwinrt-codegen/tests/python_json_native_null_test.rs index 1aea198d..dd83ea64 100644 --- a/tools/dynwinrt-codegen/tests/python_json_native_null_test.rs +++ b/tools/dynwinrt-codegen/tests/python_json_native_null_test.rs @@ -41,6 +41,14 @@ struct Generated { impl Generated { fn new() -> Option { + Self::generate(false) + } + + fn without_stubs() -> Option { + Self::generate(true) + } + + fn generate(no_pyi: bool) -> Option { if !Path::new(WINDOWS_WINMD).is_file() { eprintln!("Skipping JSON SDK regression: Windows.winmd not found"); return None; @@ -51,17 +59,21 @@ impl Generated { NEXT.fetch_add(1, Ordering::Relaxed) ); let root = repo_root().join("target").join(&package); - let output = Command::new(env!("CARGO_BIN_EXE_dynwinrt-codegen")) - .args([ - "generate", - "--winmd", - WINDOWS_WINMD, - "--class-name", - JSON_CLASSES, - "--lang", - "py", - "--output", - ]) + let mut command = Command::new(env!("CARGO_BIN_EXE_dynwinrt-codegen")); + command.args([ + "generate", + "--winmd", + WINDOWS_WINMD, + "--class-name", + JSON_CLASSES, + "--lang", + "py", + ]); + if no_pyi { + command.arg("--no-pyi"); + } + let output = command + .arg("--output") .arg(&root) .output() .expect("generate stock JSON bindings"); @@ -112,6 +124,21 @@ fn assert_success(output: Output) { ); } +fn matching_runtime_available() -> bool { + let available = Command::new(python()) + .args([ + "-c", + "from dynwinrt import DynWinRTInterfacePlan, DynWinRTValue; assert hasattr(DynWinRTValue, '_validate_non_null_collection_input')", + ]) + .output() + .is_ok_and(|output| output.status.success()); + assert!( + available || std::env::var("DYNWINRT_REQUIRE_IMPLEMENTATION_RUNTIME").as_deref() != Ok("1"), + "the JSON runtime regressions require the matching Python binding" + ); + available +} + #[test] fn stock_json_generation_preserves_receiver_dependent_contract() { let Some(generated) = Generated::new() else { @@ -122,6 +149,41 @@ fn stock_json_generation_preserves_receiver_dependent_contract() { let array_stub = generated.module("windows__data__json__json_array.pyi"); let object_stub = generated.module("windows__data__json__json_object.pyi"); + let (stock_array, generic_vector) = array + .split_once("\nclass IVector_IJsonValue(") + .expect("standalone vector view"); + for declaration in [ + "def index_of(self, value: 'IJsonValue')", + "def set_at(self, index: int, value: 'IJsonValue')", + "def insert_at(self, index: int, value: 'IJsonValue')", + "def append(self, value: 'IJsonValue')", + "def replace_all(self, items: DynWinRTArray | Sequence['IJsonValue'])", + ] { + assert!( + stock_array.contains(declaration), + "{declaration}:\n{stock_array}" + ); + } + assert!( + generic_vector.contains("def append(self, value: IJsonValue | None)") + && generic_vector.contains( + "def replace_all(self, items: DynWinRTArray | Sequence[IJsonValue | None])" + ), + "{generic_vector}" + ); + let (stock_object, generic_map) = object + .split_once("\nclass IMap_String_IJsonValue(") + .expect("standalone map view"); + assert!( + stock_object.contains("def insert(self, key: str, value: 'IJsonValue')") + && stock_object.contains("def set_named_value(self, name: str, value: 'IJsonValue')"), + "{stock_object}" + ); + assert!( + generic_map.contains("def insert(self, key: str, value: IJsonValue | None)"), + "{generic_map}" + ); + for code in [&array, &object] { assert!( code.contains("_dynwinrt_non_null_collection_contract ="), @@ -166,22 +228,59 @@ fn stock_json_generation_preserves_receiver_dependent_contract() { } #[test] -fn stock_json_mutators_fail_before_native_mutation_but_custom_generics_keep_null() { - let Some(generated) = Generated::new() else { +fn stock_json_no_pyi_runtime_annotations_match_the_native_input_contract() { + let Some(generated) = Generated::without_stubs() else { return; }; - let available = Command::new(python()) - .args([ - "-c", - "from dynwinrt import DynWinRTInterfacePlan, DynWinRTValue; assert hasattr(DynWinRTValue, '_validate_non_null_collection_input')", - ]) - .output() - .is_ok_and(|output| output.status.success()); + if !matching_runtime_available() { + eprintln!("Skipping JSON runtime annotations: matching Python binding not installed"); + return; + } assert!( - available || std::env::var("DYNWINRT_REQUIRE_IMPLEMENTATION_RUNTIME").as_deref() != Ok("1"), - "the JSON native regression requires the matching Python binding" + !generated + .root + .join("windows__data__json__json_array.pyi") + .exists() ); - if !available { + assert!( + !generated + .root + .join("windows__data__json__json_object.pyi") + .exists() + ); + assert_success(generated.python( + r#" +from inspect import signature +from JSON_PACKAGE.windows__data__json__json_array import JsonArray, IVector_IJsonValue +from JSON_PACKAGE.windows__data__json__json_object import JsonObject, IMap_String_IJsonValue + +for owner, name, parameter in ( + (JsonArray, 'append', 'value'), + (JsonArray, 'set_at', 'value'), + (JsonArray, 'insert_at', 'value'), + (JsonArray, 'replace_all', 'items'), + (JsonObject, 'insert', 'value'), + (JsonObject, 'set_named_value', 'value'), +): + annotation = str(signature(getattr(owner, name)).parameters[parameter].annotation) + assert 'IJsonValue' in annotation and 'None' not in annotation, (owner, name, annotation) +for owner, name, parameter in ( + (IVector_IJsonValue, 'append', 'value'), + (IVector_IJsonValue, 'replace_all', 'items'), + (IMap_String_IJsonValue, 'insert', 'value'), +): + annotation = str(signature(getattr(owner, name)).parameters[parameter].annotation) + assert 'IJsonValue' in annotation and 'None' in annotation, (owner, name, annotation) +"#, + )); +} + +#[test] +fn stock_json_mutators_fail_before_native_mutation_but_custom_generics_keep_null() { + let Some(generated) = Generated::new() else { + return; + }; + if !matching_runtime_available() { eprintln!("Skipping JSON native regression: matching Python binding not installed"); return; } From fbbde4f04f1eade97e211136a1c468b563f75147 Mon Sep 17 00:00:00 2001 From: Leilei Zhang Date: Tue, 29 Sep 2026 21:15:39 +0800 Subject: [PATCH 09/15] Release COM-bearing Python containers before apartment teardown Weakly track independent DynWinRTArray and DynWinRTStruct owners at creation, extraction, and nested-field boundaries. Deterministically release surviving COM references at scope exit, reject all post-release access, and leave scalar containers live. Cover five previously crashing subprocess paths, nested arrays, and native reference balance. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- bindings/py/README.md | 10 + bindings/py/dynwinrt.pyi | 9 + bindings/py/src/errors.rs | 6 + bindings/py/src/runtime.rs | 386 +++++++++++------- .../tests/test_native_container_lifetime.py | 313 ++++++++++++++ bindings/py/tests/test_typing_package.py | 5 + 6 files changed, 585 insertions(+), 144 deletions(-) create mode 100644 bindings/py/tests/test_native_container_lifetime.py diff --git a/bindings/py/README.md b/bindings/py/README.md index b64255e2..36827b7a 100644 --- a/bindings/py/README.md +++ b/bindings/py/README.md @@ -780,6 +780,16 @@ observes raw native outputs weakly: temporary casts and callback inputs can drop normally instead of accumulating references until the scope closes. Raw results still held by Python at scope exit are released there. +`DynWinRTArray` and `DynWinRTStruct` can independently own COM references, +including after `DynWinRTValue.as_array()` or `.as_struct()` clones an already +tracked value. COM-bearing containers created or extracted inside a scope are +also observed weakly and released before the apartment exits. Retained +containers then report `is_released()` and reject reads, writes and +`to_value()` with the released-object `RuntimeError`, rather than exposing +silently emptied storage. Scalar-only containers remain usable after the +scope. Without a scope, call `release()` on COM-bearing arrays and structs +inside their apartment; repeated release is safe. + Scopes nest in LIFO order. Wrappers and raw native results that survive a closed scope remain Python objects, but their COM references are released: using one afterwards, as the diff --git a/bindings/py/dynwinrt.pyi b/bindings/py/dynwinrt.pyi index 1f621b84..0b2501a9 100644 --- a/bindings/py/dynwinrt.pyi +++ b/bindings/py/dynwinrt.pyi @@ -122,7 +122,12 @@ class ProjectedLifetimeScope: def track( self, value: _Tracked, type_name: Optional[str] = ... ) -> _Tracked: ... + @overload def track_native(self, value: "DynWinRTValue") -> "DynWinRTValue": ... + @overload + def track_native(self, value: "DynWinRTArray") -> "DynWinRTArray": ... + @overload + def track_native(self, value: "DynWinRTStruct") -> "DynWinRTStruct": ... def close(self) -> None: ... @@ -576,6 +581,8 @@ class WinRTCoroutineWithProgress( @final class DynWinRTArray: + def is_released(self) -> bool: ... + def release(self) -> None: ... def __len__(self) -> int: ... def get(self, index: int) -> DynWinRTValue: ... def to_values(self) -> List[DynWinRTValue]: ... @@ -629,6 +636,8 @@ class DynWinRTArray: @final class DynWinRTStruct: + def is_released(self) -> bool: ... + def release(self) -> None: ... @staticmethod def create(typ: DynWinRTType) -> DynWinRTStruct: ... def get_i8(self, index: int) -> int: ... diff --git a/bindings/py/src/errors.rs b/bindings/py/src/errors.rs index 340ece66..0b341304 100644 --- a/bindings/py/src/errors.rs +++ b/bindings/py/src/errors.rs @@ -32,6 +32,12 @@ pub(crate) fn released_receiver_error() -> PyErr { PyRuntimeError::new_err(format!("This WinRT object {RELEASED_REASON}")) } +pub(crate) fn released_native_container_error(name: &str) -> PyErr { + PyRuntimeError::new_err(format!( + "This {name} has been released (its projected_lifetime_scope() exited, or {name}.release() was called) and can no longer be used." + )) +} + /// Where a value was handed to native code, with a 0-based index. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum InputSlot { diff --git a/bindings/py/src/runtime.rs b/bindings/py/src/runtime.rs index 32fad1b1..f2b45da3 100644 --- a/bindings/py/src/runtime.rs +++ b/bindings/py/src/runtime.rs @@ -15,7 +15,8 @@ use windows::core::{GUID, HSTRING, IInspectable, IUnknown, Interface}; use crate::errors::{ InputSlot, map_dynwinrt_error, map_dynwinrt_error_with_context, map_windows_error, - non_object_receiver_error, released_input_error, released_receiver_error, + non_object_receiver_error, released_input_error, released_native_container_error, + released_receiver_error, }; /// Shared MetadataTable — created once, used everywhere. @@ -1125,6 +1126,13 @@ fn contains_com_references(typ: &dynwinrt::TypeHandle) -> bool { && (0..typ.field_count()).any(|index| contains_com_references(&typ.field_type(index)))) } +fn track_native_owner(py: Python<'_>, owner: Py) -> PyResult<()> { + if let Some(track) = TRACK_NATIVE.get(py) { + track.call1(py, (owner,))?; + } + Ok(()) +} + /// Keep native COM ownership on the creating thread until the active lifetime /// scope closes. Python retains the exact returned value, not an extra AddRef. pub(crate) fn tracked_native_value( @@ -1141,11 +1149,28 @@ pub(crate) fn tracked_native_value( }; let output = Py::new(py, DynWinRTValue::new(value))?; if owns_native { - // Embedding tests may create values before the extension module (and - // therefore any projected lifetime scope) has been initialized. - if let Some(track) = TRACK_NATIVE.get(py) { - track.call1(py, (output.clone_ref(py),))?; - } + track_native_owner(py, output.clone_ref(py).into_any())?; + } + Ok(output) +} + +fn tracked_native_array(py: Python<'_>, array: dynwinrt::ArrayData) -> PyResult> { + let owns_com = contains_com_references(&array.element_type); + let output = Py::new(py, DynWinRTArray(Some(array)))?; + if owns_com { + track_native_owner(py, output.clone_ref(py).into_any())?; + } + Ok(output) +} + +fn tracked_native_struct( + py: Python<'_>, + data: dynwinrt::ValueTypeData, +) -> PyResult> { + let owns_com = contains_com_references(&data.type_handle()); + let output = Py::new(py, DynWinRTStruct(Some(data)))?; + if owns_com { + track_native_owner(py, output.clone_ref(py).into_any())?; } Ok(output) } @@ -1923,9 +1948,10 @@ impl DynWinRTValue { self.0.as_array().is_some() } - fn as_array(&self) -> PyResult { + fn as_array(&self, py: Python<'_>) -> PyResult> { + self.ensure_live()?; match &self.0 { - dynwinrt::WinRTValue::Array(data) => Ok(DynWinRTArray(data.clone())), + dynwinrt::WinRTValue::Array(data) => tracked_native_array(py, data.clone()), _ => Err(PyRuntimeError::new_err("Value is not an Array")), } } @@ -1934,9 +1960,10 @@ impl DynWinRTValue { self.0.as_struct().is_some() } - fn as_struct(&self) -> PyResult { + fn as_struct(&self, py: Python<'_>) -> PyResult> { + self.ensure_live()?; match &self.0 { - dynwinrt::WinRTValue::Struct(data) => Ok(DynWinRTStruct(data.clone())), + dynwinrt::WinRTValue::Struct(data) => tracked_native_struct(py, data.clone()), _ => Err(PyRuntimeError::new_err("Value is not a Struct")), } } @@ -1946,122 +1973,144 @@ impl DynWinRTValue { // DynWinRTArray — array container with blittable fast paths // ====================================================================== -#[pyclass(unsendable, from_py_object)] +#[pyclass(unsendable, from_py_object, weakref)] #[derive(Clone)] -pub struct DynWinRTArray(dynwinrt::ArrayData); +pub struct DynWinRTArray(Option); impl DynWinRTArray { + fn data(&self) -> PyResult<&dynwinrt::ArrayData> { + self.0 + .as_ref() + .ok_or_else(|| released_native_container_error("DynWinRTArray")) + } + + fn scalar_array(typ: dynwinrt::TypeHandle, values: &[dynwinrt::WinRTValue]) -> Self { + Self(Some(dynwinrt::ArrayData::from_values(typ, values))) + } + fn from_elements( operation: &str, values: Vec, element_type: &DynWinRTType, - ) -> PyResult { + ) -> PyResult { let values = native_inputs(operation, values, InputSlot::Element)?; - Ok(Self(dynwinrt::ArrayData::from_values( + Ok(dynwinrt::ArrayData::from_values( element_type.0.clone(), &values, - ))) + )) } } #[pymethods] impl DynWinRTArray { - fn __len__(&self) -> usize { - self.0.len() + fn __len__(&self) -> PyResult { + Ok(self.data()?.len()) } /// Per-element access. fn get(&self, py: Python<'_>, index: i64) -> PyResult> { + let data = self.data()?; let index = checked_index(index)?; - self.0 - .try_get(index) + data.try_get(index) .map_err(map_dynwinrt_error) .and_then(|value| tracked_native_value(py, value)) } /// Convert all elements to a list of DynWinRTValue. fn to_values(&self, py: Python<'_>) -> PyResult>> { - (0..self.0.len()) - .map(|i| tracked_native_value(py, self.0.get(i))) + let data = self.data()?; + (0..data.len()) + .map(|i| tracked_native_value(py, data.get(i))) .collect() } // -- Typed list extraction (works for both Values and CoTaskMem arrays) -- - fn to_i8_list(&self) -> Vec { - (0..self.0.len()) - .map(|i| self.0.get(i).as_i32().unwrap_or(0)) - .collect() + fn to_i8_list(&self) -> PyResult> { + let data = self.data()?; + Ok((0..data.len()) + .map(|i| data.get(i).as_i32().unwrap_or(0)) + .collect()) } - fn to_u8_list(&self) -> Vec { - (0..self.0.len()) - .map(|i| match self.0.get(i) { + fn to_u8_list(&self) -> PyResult> { + let data = self.data()?; + Ok((0..data.len()) + .map(|i| match data.get(i) { dynwinrt::WinRTValue::U8(v) => v, other => other.as_i32().unwrap_or(0) as u8, }) - .collect() + .collect()) } - fn to_i16_list(&self) -> Vec { - (0..self.0.len()) - .map(|i| self.0.get(i).as_i32().unwrap_or(0)) - .collect() + fn to_i16_list(&self) -> PyResult> { + let data = self.data()?; + Ok((0..data.len()) + .map(|i| data.get(i).as_i32().unwrap_or(0)) + .collect()) } - fn to_u16_list(&self) -> Vec { - (0..self.0.len()) - .map(|i| self.0.get(i).as_i32().unwrap_or(0) as u32) - .collect() + fn to_u16_list(&self) -> PyResult> { + let data = self.data()?; + Ok((0..data.len()) + .map(|i| data.get(i).as_i32().unwrap_or(0) as u32) + .collect()) } fn to_i32_list(&self) -> PyResult> { - (0..self.0.len()) - .map(|i| self.0.get_i32(i).map_err(map_dynwinrt_error)) + let data = self.data()?; + (0..data.len()) + .map(|i| data.get_i32(i).map_err(map_dynwinrt_error)) .collect() } fn to_u32_list(&self) -> PyResult> { - (0..self.0.len()) - .map(|i| self.0.get_u32(i).map_err(map_dynwinrt_error)) + let data = self.data()?; + (0..data.len()) + .map(|i| data.get_u32(i).map_err(map_dynwinrt_error)) .collect() } - fn to_f32_list(&self) -> Vec { - (0..self.0.len()) - .map(|i| match self.0.get(i) { + fn to_f32_list(&self) -> PyResult> { + let data = self.data()?; + Ok((0..data.len()) + .map(|i| match data.get(i) { dynwinrt::WinRTValue::F32(v) => v, dynwinrt::WinRTValue::F64(v) => v as f32, other => other.as_i32().unwrap_or(0) as f32, }) - .collect() + .collect()) } - fn to_f64_list(&self) -> Vec { - (0..self.0.len()) - .map(|i| match self.0.get(i) { + fn to_f64_list(&self) -> PyResult> { + let data = self.data()?; + Ok((0..data.len()) + .map(|i| match data.get(i) { dynwinrt::WinRTValue::F64(v) => v, dynwinrt::WinRTValue::F32(v) => v as f64, other => other.as_i32().unwrap_or(0) as f64, }) - .collect() + .collect()) } - fn to_i64_list(&self) -> Vec { - (0..self.0.len()) - .map(|i| match self.0.get(i) { + fn to_i64_list(&self) -> PyResult> { + let data = self.data()?; + Ok((0..data.len()) + .map(|i| match data.get(i) { dynwinrt::WinRTValue::I64(v) => v, other => other.as_i32().unwrap_or(0) as i64, }) - .collect() + .collect()) } - fn to_u64_list(&self) -> Vec { - (0..self.0.len()) - .map(|i| match self.0.get(i) { + fn to_u64_list(&self) -> PyResult> { + let data = self.data()?; + Ok((0..data.len()) + .map(|i| match data.get(i) { dynwinrt::WinRTValue::U64(v) => v, other => other.as_i32().unwrap_or(0) as u64, }) - .collect() + .collect()) } - fn to_string_list(&self) -> Vec { - (0..self.0.len()) - .map(|i| match self.0.get(i) { + fn to_string_list(&self) -> PyResult> { + let data = self.data()?; + Ok((0..data.len()) + .map(|i| match data.get(i) { dynwinrt::WinRTValue::HString(s) => s.to_string(), other => format!("{:?}", other), }) - .collect() + .collect()) } // -- Construction from Python lists -- @@ -2077,16 +2126,13 @@ impl DynWinRTArray { )?)) }) .collect::>()?; - Ok(DynWinRTArray(dynwinrt::ArrayData::from_values( - TABLE.i8_type(), - &wvals, - ))) + Ok(Self::scalar_array(TABLE.i8_type(), &wvals)) } #[staticmethod] fn from_u8_values(values: Vec) -> DynWinRTArray { let wvals: Vec = values.into_iter().map(dynwinrt::WinRTValue::U8).collect(); - DynWinRTArray(dynwinrt::ArrayData::from_values(TABLE.u8_type(), &wvals)) + Self::scalar_array(TABLE.u8_type(), &wvals) } #[staticmethod] fn from_i16_values(values: Vec) -> PyResult { @@ -2099,10 +2145,7 @@ impl DynWinRTArray { )?)) }) .collect::>()?; - Ok(DynWinRTArray(dynwinrt::ArrayData::from_values( - TABLE.i16_type(), - &wvals, - ))) + Ok(Self::scalar_array(TABLE.i16_type(), &wvals)) } #[staticmethod] fn from_u16_values(values: Vec) -> PyResult { @@ -2115,46 +2158,43 @@ impl DynWinRTArray { )?)) }) .collect::>()?; - Ok(DynWinRTArray(dynwinrt::ArrayData::from_values( - TABLE.u16_type(), - &wvals, - ))) + Ok(Self::scalar_array(TABLE.u16_type(), &wvals)) } #[staticmethod] fn from_i32_values(values: Vec) -> DynWinRTArray { let wvals: Vec = values.into_iter().map(dynwinrt::WinRTValue::I32).collect(); - DynWinRTArray(dynwinrt::ArrayData::from_values(TABLE.i32_type(), &wvals)) + Self::scalar_array(TABLE.i32_type(), &wvals) } #[staticmethod] fn from_u32_values(values: Vec) -> DynWinRTArray { let wvals: Vec = values.into_iter().map(dynwinrt::WinRTValue::U32).collect(); - DynWinRTArray(dynwinrt::ArrayData::from_values(TABLE.u32_type(), &wvals)) + Self::scalar_array(TABLE.u32_type(), &wvals) } #[staticmethod] fn from_f32_values(values: Vec) -> DynWinRTArray { let wvals: Vec = values.into_iter().map(dynwinrt::WinRTValue::F32).collect(); - DynWinRTArray(dynwinrt::ArrayData::from_values(TABLE.f32_type(), &wvals)) + Self::scalar_array(TABLE.f32_type(), &wvals) } #[staticmethod] fn from_f64_values(values: Vec) -> DynWinRTArray { let wvals: Vec = values.into_iter().map(dynwinrt::WinRTValue::F64).collect(); - DynWinRTArray(dynwinrt::ArrayData::from_values(TABLE.f64_type(), &wvals)) + Self::scalar_array(TABLE.f64_type(), &wvals) } #[staticmethod] fn from_i64_values(values: Vec) -> DynWinRTArray { let wvals: Vec = values.into_iter().map(dynwinrt::WinRTValue::I64).collect(); - DynWinRTArray(dynwinrt::ArrayData::from_values(TABLE.i64_type(), &wvals)) + Self::scalar_array(TABLE.i64_type(), &wvals) } #[staticmethod] fn from_u64_values(values: Vec) -> DynWinRTArray { let wvals: Vec = values.into_iter().map(dynwinrt::WinRTValue::U64).collect(); - DynWinRTArray(dynwinrt::ArrayData::from_values(TABLE.u64_type(), &wvals)) + Self::scalar_array(TABLE.u64_type(), &wvals) } #[staticmethod] fn from_string_values(values: Vec) -> DynWinRTArray { @@ -2162,18 +2202,19 @@ impl DynWinRTArray { .into_iter() .map(|s| dynwinrt::WinRTValue::HString(HSTRING::from(&s))) .collect(); - DynWinRTArray(dynwinrt::ArrayData::from_values( - TABLE.make(dynwinrt::TypeKind::HString), - &wvals, - )) + Self::scalar_array(TABLE.make(dynwinrt::TypeKind::HString), &wvals) } #[staticmethod] fn from_values( + py: Python<'_>, values: Vec, element_type: &DynWinRTType, - ) -> PyResult { - Self::from_elements("DynWinRTArray.from_values()", values, element_type) + ) -> PyResult> { + tracked_native_array( + py, + Self::from_elements("DynWinRTArray.from_values()", values, element_type)?, + ) } /// Build a DynWinRTArray of WinRT object/interface elements. @@ -2184,24 +2225,29 @@ impl DynWinRTArray { /// and the element type drives ABI size and IID computation. #[staticmethod] fn from_object_values( + py: Python<'_>, values: Vec, element_type: &DynWinRTType, - ) -> PyResult { - Self::from_elements("DynWinRTArray.from_object_values()", values, element_type) + ) -> PyResult> { + tracked_native_array( + py, + Self::from_elements("DynWinRTArray.from_object_values()", values, element_type)?, + ) } /// Return the u8 array data as a Python `bytes` object. Safe for both /// `Values`-backed and `CoTaskMem`-backed arrays. - fn to_bytes<'py>(&self, py: Python<'py>) -> Bound<'py, pyo3::types::PyBytes> { - let len = self.0.len(); + fn to_bytes<'py>(&self, py: Python<'py>) -> PyResult> { + let data = self.data()?; + let len = data.len(); let mut buf: Vec = Vec::with_capacity(len); for i in 0..len { - buf.push(match self.0.get(i) { + buf.push(match data.get(i) { dynwinrt::WinRTValue::U8(v) => v, other => other.as_i32().unwrap_or(0) as u8, }); } - pyo3::types::PyBytes::new(py, &buf) + Ok(pyo3::types::PyBytes::new(py, &buf)) } /// Build a u8 DynWinRTArray from a Python `bytes` or `bytearray` (much more @@ -2220,19 +2266,27 @@ impl DynWinRTArray { }; let wvals: Vec = slice.into_iter().map(dynwinrt::WinRTValue::U8).collect(); - Ok(DynWinRTArray(dynwinrt::ArrayData::from_values( - TABLE.u8_type(), - &wvals, - ))) + Ok(Self::scalar_array(TABLE.u8_type(), &wvals)) } /// Wrap as DynWinRTValue::Array for passing to call(). fn to_value(&self, py: Python<'_>) -> PyResult> { - tracked_native_value(py, dynwinrt::WinRTValue::Array(self.0.clone())) + tracked_native_value(py, dynwinrt::WinRTValue::Array(self.data()?.clone())) + } + + fn is_released(&self) -> bool { + self.0.is_none() + } + + fn release(&mut self) { + drop(self.0.take()); } fn __repr__(&self) -> String { - format!("DynWinRTArray(len={})", self.0.len()) + match &self.0 { + Some(data) => format!("DynWinRTArray(len={})", data.len()), + None => "DynWinRTArray(released)".to_string(), + } } } @@ -2240,28 +2294,46 @@ impl DynWinRTArray { // DynWinRTStruct — typed field access by index // ====================================================================== -#[pyclass(unsendable, from_py_object)] +#[pyclass(unsendable, from_py_object, weakref)] #[derive(Clone)] -pub struct DynWinRTStruct(dynwinrt::ValueTypeData); +pub struct DynWinRTStruct(Option); + +impl DynWinRTStruct { + fn data(&self) -> PyResult<&dynwinrt::ValueTypeData> { + self.0 + .as_ref() + .ok_or_else(|| released_native_container_error("DynWinRTStruct")) + } + + fn data_mut(&mut self) -> PyResult<&mut dynwinrt::ValueTypeData> { + self.0 + .as_mut() + .ok_or_else(|| released_native_container_error("DynWinRTStruct")) + } +} #[pymethods] impl DynWinRTStruct { /// Create a zero-initialized struct of the given type. #[staticmethod] - fn create(typ: &DynWinRTType) -> DynWinRTStruct { - DynWinRTStruct(typ.0.default_value()) + fn create(py: Python<'_>, typ: &DynWinRTType) -> PyResult> { + tracked_native_struct(py, typ.0.default_value()) } // -- Blittable field access (get/set pairs) -- fn get_i8(&self, index: i64) -> PyResult { - get_typed_field(&self.0, index, dynwinrt::TypeKind::I8, &[], |value: i8| { - value as i32 - }) + get_typed_field( + self.data()?, + index, + dynwinrt::TypeKind::I8, + &[], + |value: i8| value as i32, + ) } fn set_i8(&mut self, index: i64, value: i32) -> PyResult<()> { set_typed_field( - &mut self.0, + self.data_mut()?, index, checked_i8(value, "set_i8")?, dynwinrt::TypeKind::I8, @@ -2270,13 +2342,17 @@ impl DynWinRTStruct { } fn get_u8(&self, index: i64) -> PyResult { - get_typed_field(&self.0, index, dynwinrt::TypeKind::U8, &[], |value: u8| { - value as u32 - }) + get_typed_field( + self.data()?, + index, + dynwinrt::TypeKind::U8, + &[], + |value: u8| value as u32, + ) } fn set_u8(&mut self, index: i64, value: u32) -> PyResult<()> { set_typed_field( - &mut self.0, + self.data_mut()?, index, checked_u8(value, "set_u8")?, dynwinrt::TypeKind::U8, @@ -2286,7 +2362,7 @@ impl DynWinRTStruct { fn get_i16(&self, index: i64) -> PyResult { get_typed_field( - &self.0, + self.data()?, index, dynwinrt::TypeKind::I16, &[], @@ -2295,7 +2371,7 @@ impl DynWinRTStruct { } fn set_i16(&mut self, index: i64, value: i32) -> PyResult<()> { set_typed_field( - &mut self.0, + self.data_mut()?, index, checked_i16(value, "set_i16")?, dynwinrt::TypeKind::I16, @@ -2305,7 +2381,7 @@ impl DynWinRTStruct { fn get_u16(&self, index: i64) -> PyResult { get_typed_field( - &self.0, + self.data()?, index, dynwinrt::TypeKind::U16, &[dynwinrt::TypeKind::Char16], @@ -2314,7 +2390,7 @@ impl DynWinRTStruct { } fn set_u16(&mut self, index: i64, value: u32) -> PyResult<()> { set_typed_field( - &mut self.0, + self.data_mut()?, index, checked_u16(value, "set_u16")?, dynwinrt::TypeKind::U16, @@ -2324,7 +2400,7 @@ impl DynWinRTStruct { fn get_i32(&self, index: i64) -> PyResult { get_typed_field( - &self.0, + self.data()?, index, dynwinrt::TypeKind::I32, &[], @@ -2332,12 +2408,12 @@ impl DynWinRTStruct { ) } fn set_i32(&mut self, index: i64, value: i32) -> PyResult<()> { - set_typed_field(&mut self.0, index, value, dynwinrt::TypeKind::I32, &[]) + set_typed_field(self.data_mut()?, index, value, dynwinrt::TypeKind::I32, &[]) } fn get_u32(&self, index: i64) -> PyResult { get_typed_field( - &self.0, + self.data()?, index, dynwinrt::TypeKind::U32, &[], @@ -2345,12 +2421,12 @@ impl DynWinRTStruct { ) } fn set_u32(&mut self, index: i64, value: u32) -> PyResult<()> { - set_typed_field(&mut self.0, index, value, dynwinrt::TypeKind::U32, &[]) + set_typed_field(self.data_mut()?, index, value, dynwinrt::TypeKind::U32, &[]) } fn get_f32(&self, index: i64) -> PyResult { get_typed_field( - &self.0, + self.data()?, index, dynwinrt::TypeKind::F32, &[], @@ -2359,7 +2435,7 @@ impl DynWinRTStruct { } fn set_f32(&mut self, index: i64, value: f64) -> PyResult<()> { set_typed_field( - &mut self.0, + self.data_mut()?, index, value as f32, dynwinrt::TypeKind::F32, @@ -2369,7 +2445,7 @@ impl DynWinRTStruct { fn get_f64(&self, index: i64) -> PyResult { get_typed_field( - &self.0, + self.data()?, index, dynwinrt::TypeKind::F64, &[], @@ -2377,12 +2453,12 @@ impl DynWinRTStruct { ) } fn set_f64(&mut self, index: i64, value: f64) -> PyResult<()> { - set_typed_field(&mut self.0, index, value, dynwinrt::TypeKind::F64, &[]) + set_typed_field(self.data_mut()?, index, value, dynwinrt::TypeKind::F64, &[]) } fn get_i64(&self, index: i64) -> PyResult { get_typed_field( - &self.0, + self.data()?, index, dynwinrt::TypeKind::I64, &[], @@ -2390,12 +2466,12 @@ impl DynWinRTStruct { ) } fn set_i64(&mut self, index: i64, value: i64) -> PyResult<()> { - set_typed_field(&mut self.0, index, value, dynwinrt::TypeKind::I64, &[]) + set_typed_field(self.data_mut()?, index, value, dynwinrt::TypeKind::I64, &[]) } fn get_u64(&self, index: i64) -> PyResult { get_typed_field( - &self.0, + self.data()?, index, dynwinrt::TypeKind::U64, &[], @@ -2403,14 +2479,14 @@ impl DynWinRTStruct { ) } fn set_u64(&mut self, index: i64, value: u64) -> PyResult<()> { - set_typed_field(&mut self.0, index, value, dynwinrt::TypeKind::U64, &[]) + set_typed_field(self.data_mut()?, index, value, dynwinrt::TypeKind::U64, &[]) } // -- Non-blittable field access -- fn get_hstring(&self, index: i64) -> PyResult { let index = checked_index(index)?; - self.0 + self.data()? .get_field_hstring(index) .map(|value| value.to_string()) .map_err(map_dynwinrt_error) @@ -2418,37 +2494,48 @@ impl DynWinRTStruct { fn set_hstring(&mut self, index: i64, value: String) -> PyResult<()> { let index = checked_index(index)?; - self.0 + self.data_mut()? .set_field_hstring(index, HSTRING::from(&value)) .map_err(map_dynwinrt_error) } fn get_guid(&self, index: i64) -> PyResult { - get_typed_field(&self.0, index, dynwinrt::TypeKind::Guid, &[], WinGUID) + get_typed_field(self.data()?, index, dynwinrt::TypeKind::Guid, &[], WinGUID) } fn set_guid(&mut self, index: i64, value: &WinGUID) -> PyResult<()> { - set_typed_field(&mut self.0, index, value.0, dynwinrt::TypeKind::Guid, &[]) + set_typed_field( + self.data_mut()?, + index, + value.0, + dynwinrt::TypeKind::Guid, + &[], + ) } - fn get_struct(&self, index: i64) -> PyResult { + fn get_struct(&self, py: Python<'_>, index: i64) -> PyResult> { let index = checked_index(index)?; - self.0 + let data = self + .data()? .get_field_struct_checked(index) - .map(DynWinRTStruct) - .map_err(map_dynwinrt_error) + .map_err(map_dynwinrt_error)?; + tracked_native_struct(py, data) } fn set_struct(&mut self, index: i64, value: &DynWinRTStruct) -> PyResult<()> { let index = checked_index(index)?; - self.0 - .set_field_struct_checked(index, &value.0) + self.data_mut()? + .set_field_struct_checked(index, value.data()?) .map_err(map_dynwinrt_error) } fn get_object(&self, py: Python<'_>, index: i64) -> PyResult> { let index = checked_index(index)?; - let value = match self.0.get_field_object(index).map_err(map_dynwinrt_error)? { + let value = match self + .data()? + .get_field_object(index) + .map_err(map_dynwinrt_error)? + { Some(object) => dynwinrt::WinRTValue::Object(object), None => dynwinrt::WinRTValue::Null, }; @@ -2457,14 +2544,13 @@ impl DynWinRTStruct { fn set_object(&mut self, index: i64, value: &DynWinRTValue) -> PyResult<()> { let index = checked_index(index)?; + let data = self.data_mut()?; value.check_input("DynWinRTStruct.set_object()", InputSlot::Field(index))?; match &value.0 { - dynwinrt::WinRTValue::Object(obj) => self - .0 + dynwinrt::WinRTValue::Object(obj) => data .set_field_object(index, Some(obj)) .map_err(map_dynwinrt_error), - dynwinrt::WinRTValue::Null => self - .0 + dynwinrt::WinRTValue::Null => data .set_field_object(index, None) .map_err(map_dynwinrt_error), _ => Err(PyTypeError::new_err( @@ -2475,11 +2561,23 @@ impl DynWinRTStruct { /// Wrap as DynWinRTValue::Struct for passing to call(). fn to_value(&self, py: Python<'_>) -> PyResult> { - tracked_native_value(py, dynwinrt::WinRTValue::Struct(self.0.clone())) + tracked_native_value(py, dynwinrt::WinRTValue::Struct(self.data()?.clone())) + } + + fn is_released(&self) -> bool { + self.0.is_none() + } + + fn release(&mut self) { + drop(self.0.take()); } fn __repr__(&self) -> String { - "DynWinRTStruct(...)".to_string() + if self.is_released() { + "DynWinRTStruct(released)".to_string() + } else { + "DynWinRTStruct(...)".to_string() + } } } @@ -2848,7 +2946,7 @@ mod tests { dynwinrt::WinRTValue::HResult(windows::core::HRESULT(0)), dynwinrt::WinRTValue::HResult(windows::core::HRESULT(0x80004005u32 as i32)), ]; - let array = DynWinRTArray(dynwinrt::ArrayData::from_values(TABLE.hresult(), &values)); + let array = DynWinRTArray::scalar_array(TABLE.hresult(), &values); assert_eq!(array.to_i32_list().unwrap(), vec![0, 0x80004005u32 as i32]); } diff --git a/bindings/py/tests/test_native_container_lifetime.py b/bindings/py/tests/test_native_container_lifetime.py new file mode 100644 index 00000000..419247b2 --- /dev/null +++ b/bindings/py/tests/test_native_container_lifetime.py @@ -0,0 +1,313 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Native container owners must be disposed before their COM apartment exits.""" + +import gc +import subprocess +import sys +import weakref + +import pytest + +from dynwinrt import ( + DynWinRTArray, + DynWinRTStruct, + DynWinRTType, + DynWinRTValue, + RoApartment, + WinGUID, + projected_lifetime_scope, +) + + +_ESCAPING_CONTAINER = r""" +import gc +import sys +from dynwinrt import ( + DynWinRTArray, DynWinRTStruct, DynWinRTType, RoApartment, + projected_lifetime_scope, to_winrt_object, +) + +mode = sys.argv[1] +with RoApartment(), projected_lifetime_scope(): + boxed = to_winrt_object(8080) + element = DynWinRTType.object() + if mode in ('from_values', 'from_object_values', 'as_array'): + constructor = ( + DynWinRTArray.from_values if mode == 'from_values' + else DynWinRTArray.from_object_values + ) + original = constructor([boxed], element) + if mode == 'as_array': + raw = original.to_value() + escaped = raw.as_array() + del original + else: + escaped = original + elif mode == 'array_of_struct': + shape = DynWinRTType.struct_type('Tests.ScopedStructElement', [element]) + field = DynWinRTStruct.create(shape) + field.set_object(0, boxed) + original = DynWinRTArray.from_values([field.to_value()], shape) + escaped = original.to_value().as_array() + del original, field + else: + shape = DynWinRTType.struct_type('Tests.ScopedObject', [element]) + original = DynWinRTStruct.create(shape) + original.set_object(0, boxed) + if mode == 'as_struct': + raw = original.to_value() + escaped = raw.as_struct() + del original + else: + escaped = original +assert boxed.is_released() +assert escaped.is_released() +if mode in ('from_values', 'from_object_values', 'as_array', 'array_of_struct'): + operations = (lambda: len(escaped), lambda: escaped.get(0), + escaped.to_values, escaped.to_value, escaped.to_i32_list) + name = 'DynWinRTArray' +else: + operations = (lambda: escaped.get_object(0), lambda: escaped.set_object(0, boxed), + escaped.to_value) + name = 'DynWinRTStruct' +for operation in operations: + try: + operation() + except RuntimeError as error: + assert f'{name}.release()' in str(error), error + else: + raise AssertionError(f'{name} accepted use after its native reference was released') +print('scope-exited', mode, flush=True) +del escaped +gc.collect() +print('clean-exit', mode, flush=True) +""" + +_BALANCE_CONTAINER = r""" +import gc +import sys +from dynwinrt import ( + DynWinRTArray, DynWinRTStruct, DynWinRTType, DynWinRTValue, + DynWinRTInterfacePlan, DynWinRTImplementationMethod, DynWinRTImplementation, + DynWinRTMethodSig, RoApartment, WinGUID, projected_lifetime_scope, +) + +mode = sys.argv[1] +signature = DynWinRTMethodSig().add_out(DynWinRTType.hstring()) +iid = WinGUID.parse('96369f54-8eb6-48f0-abce-c1b211e627c3') +interface = DynWinRTType.register_interface( + 'Tests.IStringableContainerLifetime', iid, +).add_method('ToString', signature) +plan = DynWinRTInterfacePlan.create( + 'Tests.IStringableContainerLifetime', + interface, + [DynWinRTImplementationMethod('ToString', 6, signature)], +) +with RoApartment(), projected_lifetime_scope(): + owner = DynWinRTImplementation.create( + [plan], + lambda _interface, _slot, _args: [DynWinRTValue.from_hstring('alive')], + 'DynWinRT.Tests.ContainerOwner', + ) + source = owner.to_value() + if mode == 'array': + container = DynWinRTArray.from_object_values([source], DynWinRTType.object()) + clone = container.to_value().as_array() + elif mode == 'array_of_struct': + shape = DynWinRTType.struct_type('Tests.ContainerOwnerRefElement', [DynWinRTType.object()]) + field = DynWinRTStruct.create(shape) + field.set_object(0, source) + container = DynWinRTArray.from_values([field.to_value()], shape) + clone = container.to_value().as_array() + del field + else: + shape = DynWinRTType.struct_type('Tests.ContainerOwnerRef', [DynWinRTType.object()]) + container = DynWinRTStruct.create(shape) + container.set_object(0, source) + clone = container.to_value().as_struct() + owner.release() + assert not owner.is_closed +assert owner.is_closed, 'a container retained a native owner past its apartment' +assert container.is_released() and clone.is_released() +del clone, container +gc.collect() +print('balanced-references', mode, flush=True) +""" + + +@pytest.mark.parametrize( + "mode", + [ + "from_values", + "from_object_values", + "as_array", + "array_of_struct", + "create_struct", + "as_struct", + ], +) +def test_escaping_com_container_drops_after_apartment_exit(mode): + result = subprocess.run( + [sys.executable, "-B", "-c", _ESCAPING_CONTAINER, mode], + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, ( + mode, + hex(result.returncode & 0xFFFFFFFF), + result.stdout, + result.stderr, + ) + assert f"clean-exit {mode}" in result.stdout + + +@pytest.mark.parametrize("mode", ["array", "array_of_struct", "struct"]) +def test_scope_balances_native_implementation_container_references(mode): + result = subprocess.run( + [sys.executable, "-B", "-c", _BALANCE_CONTAINER, mode], + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, ( + mode, + hex(result.returncode & 0xFFFFFFFF), + result.stdout, + result.stderr, + ) + assert f"balanced-references {mode}" in result.stdout + + +def test_scalar_containers_remain_usable_after_scope_exit(): + with RoApartment(), projected_lifetime_scope(): + numbers = DynWinRTArray.from_i32_values([7, 11]) + copy = numbers.to_value().as_array() + shape = DynWinRTType.struct_type("Tests.ScopedScalar", [DynWinRTType.i32_type()]) + record = DynWinRTStruct.create(shape) + record.set_i32(0, 8080) + record_copy = record.to_value().as_struct() + + for container in (numbers, copy, record, record_copy): + assert not container.is_released() + assert numbers.to_i32_list() == copy.to_i32_list() == [7, 11] + assert record.get_i32(0) == record_copy.get_i32(0) == 8080 + + +def test_explicit_release_of_com_containers_is_idempotent_and_keeps_source_live(): + with RoApartment(): + boxed = DynWinRTValue.activation_factory("Windows.Foundation.Uri") + array = DynWinRTArray.from_object_values([boxed], DynWinRTType.object()) + shape = DynWinRTType.struct_type("Tests.ExplicitObject", [DynWinRTType.object()]) + record = DynWinRTStruct.create(shape) + record.set_object(0, boxed) + assert not array.is_released() and not record.is_released() + + for container in (array, record): + container.release() + container.release() + assert container.is_released() + assert not boxed.is_released() + with pytest.raises(RuntimeError, match="DynWinRTArray.release"): + array.to_value() + with pytest.raises(RuntimeError, match="DynWinRTStruct.release"): + record.get_object(0) + boxed.release() + + +def test_scope_does_not_root_temporary_com_containers(): + with RoApartment(), projected_lifetime_scope() as scope: + boxed = DynWinRTValue.activation_factory("Windows.Foundation.Uri") + array = DynWinRTArray.from_object_values([boxed], DynWinRTType.object()) + array_id = id(array) + array_ref = weakref.ref(array) + assert scope.track_native(array) is array + assert array_id in scope._native_refs + del array + gc.collect() + assert array_ref() is None and array_id not in scope._native_refs + + shape = DynWinRTType.struct_type("Tests.TemporaryObject", [DynWinRTType.object()]) + record = DynWinRTStruct.create(shape) + record.set_object(0, boxed) + record_id = id(record) + record_ref = weakref.ref(record) + assert record_id in scope._native_refs + del record + gc.collect() + assert record_ref() is None and record_id not in scope._native_refs + + assert boxed.is_released() + + +def test_released_array_rejects_every_read_and_conversion(): + array = DynWinRTArray.from_i32_values([7]) + array.release() + array.release() + assert array.is_released() + for operation in ( + lambda: len(array), + lambda: array.get(0), + array.to_values, + array.to_i8_list, + array.to_u8_list, + array.to_i16_list, + array.to_u16_list, + array.to_i32_list, + array.to_u32_list, + array.to_f32_list, + array.to_f64_list, + array.to_i64_list, + array.to_u64_list, + array.to_string_list, + array.to_bytes, + array.to_value, + ): + with pytest.raises(RuntimeError, match=r"DynWinRTArray\.release\(\)"): + operation() + + +def test_released_struct_rejects_every_field_operation(): + shape = DynWinRTType.struct_type("Tests.ReleasedFields", [DynWinRTType.i32_type()]) + record = DynWinRTStruct.create(shape) + replacement = DynWinRTStruct.create(shape) + record.release() + record.release() + assert record.is_released() + for operation in ( + lambda: record.get_i8(0), + lambda: record.set_i8(0, 1), + lambda: record.get_u8(0), + lambda: record.set_u8(0, 1), + lambda: record.get_i16(0), + lambda: record.set_i16(0, 1), + lambda: record.get_u16(0), + lambda: record.set_u16(0, 1), + lambda: record.get_i32(0), + lambda: record.set_i32(0, 1), + lambda: record.get_u32(0), + lambda: record.set_u32(0, 1), + lambda: record.get_i64(0), + lambda: record.set_i64(0, 1), + lambda: record.get_u64(0), + lambda: record.set_u64(0, 1), + lambda: record.get_f32(0), + lambda: record.set_f32(0, 1.0), + lambda: record.get_f64(0), + lambda: record.set_f64(0, 1.0), + lambda: record.get_hstring(0), + lambda: record.set_hstring(0, "released"), + lambda: record.get_guid(0), + lambda: record.set_guid(0, WinGUID.parse("00000000-0000-0000-0000-000000000000")), + lambda: record.get_object(0), + lambda: record.set_object(0, DynWinRTValue.null_value()), + lambda: record.get_struct(0), + lambda: record.set_struct(0, replacement), + record.to_value, + ): + with pytest.raises(RuntimeError, match=r"DynWinRTStruct\.release\(\)"): + operation() diff --git a/bindings/py/tests/test_typing_package.py b/bindings/py/tests/test_typing_package.py index 333defa5..93674704 100644 --- a/bindings/py/tests/test_typing_package.py +++ b/bindings/py/tests/test_typing_package.py @@ -40,6 +40,11 @@ def test_wheel_stubs_raw_native_scope_tracking(): "\ndef projected_lifetime_scope()", 1 )[0] assert 'def track_native(self, value: "DynWinRTValue") -> "DynWinRTValue": ...' in scope + for name in ("DynWinRTArray", "DynWinRTStruct"): + assert f'def track_native(self, value: "{name}") -> "{name}": ...' in scope + class_body = stub.split(f"class {name}:", 1)[1].split("\n@final\nclass ", 1)[0] + assert "def is_released(self) -> bool: ..." in class_body + assert "def release(self) -> None: ..." in class_body assert tuple(inspect.signature(dynwinrt.ProjectedLifetimeScope.track_native).parameters) == ( "self", "value", From 04a9816c8c0a0d1668eb69b766b7617415a13b00 Mon Sep 17 00:00:00 2001 From: Leilei Zhang Date: Tue, 29 Sep 2026 21:40:19 +0800 Subject: [PATCH 10/15] Cover nested COM structs and borrowed callback lifetimes Exercise get_struct and nested Object-field clones through apartment teardown and owner reference balance, prove releasing a container preserves its external source identity, and verify borrowed native callback arrays survive an inner scope without adopting the parameter. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../tests/test_native_container_lifetime.py | 90 ++++++++++++++++++- 1 file changed, 88 insertions(+), 2 deletions(-) diff --git a/bindings/py/tests/test_native_container_lifetime.py b/bindings/py/tests/test_native_container_lifetime.py index 419247b2..913035a7 100644 --- a/bindings/py/tests/test_native_container_lifetime.py +++ b/bindings/py/tests/test_native_container_lifetime.py @@ -52,6 +52,15 @@ original = DynWinRTArray.from_values([field.to_value()], shape) escaped = original.to_value().as_array() del original, field + elif mode == 'get_struct': + inner_shape = DynWinRTType.struct_type('Tests.ScopedInner', [element]) + outer_shape = DynWinRTType.struct_type('Tests.ScopedOuter', [inner_shape]) + inner = DynWinRTStruct.create(inner_shape) + inner.set_object(0, boxed) + outer = DynWinRTStruct.create(outer_shape) + outer.set_struct(0, inner) + escaped = outer.get_struct(0) + del inner, outer else: shape = DynWinRTType.struct_type('Tests.ScopedObject', [element]) original = DynWinRTStruct.create(shape) @@ -122,6 +131,15 @@ container = DynWinRTArray.from_values([field.to_value()], shape) clone = container.to_value().as_array() del field + elif mode == 'get_struct': + inner_shape = DynWinRTType.struct_type('Tests.ContainerOwnerRefInner', [DynWinRTType.object()]) + outer_shape = DynWinRTType.struct_type('Tests.ContainerOwnerRefOuter', [inner_shape]) + inner = DynWinRTStruct.create(inner_shape) + inner.set_object(0, source) + container = DynWinRTStruct.create(outer_shape) + container.set_struct(0, inner) + clone = container.get_struct(0) + del inner else: shape = DynWinRTType.struct_type('Tests.ContainerOwnerRef', [DynWinRTType.object()]) container = DynWinRTStruct.create(shape) @@ -136,6 +154,56 @@ print('balanced-references', mode, flush=True) """ +_BORROWED_CALLBACK = r""" +from dynwinrt import ( + DynWinRTArray, DynWinRTType, DynWinRTValue, DynWinRTImplementation, + DynWinRTImplementationMethod, DynWinRTInterfacePlan, DynWinRTMethodSig, + RoApartment, WinGUID, projected_lifetime_scope, +) + +signature = DynWinRTMethodSig().add_in( + DynWinRTType.array_type(DynWinRTType.object()) +) +iid = WinGUID.parse('13fd99ec-a997-4497-aabc-247345013f26') +interface = DynWinRTType.register_interface( + 'Tests.IBorrowedContainerCallback', iid, +).add_method('AcceptArray', signature) +plan = DynWinRTInterfacePlan.create( + 'Tests.IBorrowedContainerCallback', interface, + [DynWinRTImplementationMethod('AcceptArray', 6, signature)], +) +borrowed = [] +copies = [] + +def dispatch(index, slot, args): + assert (index, slot) == (0, 6) and len(args) == 1 + borrowed.append(args[0]) + copies.append(args[0].as_array()) + assert len(copies[0]) == 1 + return [] + +with RoApartment(): + with projected_lifetime_scope(): + owner = DynWinRTImplementation.create( + [plan], dispatch, 'DynWinRT.Tests.BorrowedArrayOwner' + ) + source = DynWinRTValue.activation_factory('Windows.Foundation.Uri') + array = DynWinRTArray.from_object_values([source], DynWinRTType.object()) + raw = array.to_value() + receiver = owner.to_value().cast(iid) + assert interface.method(6).invoke_all(receiver, [raw]) == [] + + assert source.is_released() and array.is_released() and copies[0].is_released() + assert not borrowed[0].is_released(), 'the scope consumed a borrowed callback parameter' + live_copy = borrowed[0].as_array() + assert len(live_copy) == 1 + live_copy.release() + borrowed[0].release() + owner.release() + assert owner.is_closed +print('borrowed-callback-retained', flush=True) +""" + @pytest.mark.parametrize( "mode", @@ -146,6 +214,7 @@ "array_of_struct", "create_struct", "as_struct", + "get_struct", ], ) def test_escaping_com_container_drops_after_apartment_exit(mode): @@ -165,7 +234,7 @@ def test_escaping_com_container_drops_after_apartment_exit(mode): assert f"clean-exit {mode}" in result.stdout -@pytest.mark.parametrize("mode", ["array", "array_of_struct", "struct"]) +@pytest.mark.parametrize("mode", ["array", "array_of_struct", "struct", "get_struct"]) def test_scope_balances_native_implementation_container_references(mode): result = subprocess.run( [sys.executable, "-B", "-c", _BALANCE_CONTAINER, mode], @@ -183,6 +252,22 @@ def test_scope_balances_native_implementation_container_references(mode): assert f"balanced-references {mode}" in result.stdout +def test_borrowed_callback_array_survives_scope_within_its_apartment(): + result = subprocess.run( + [sys.executable, "-B", "-c", _BORROWED_CALLBACK], + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, ( + hex(result.returncode & 0xFFFFFFFF), + result.stdout, + result.stderr, + ) + assert "borrowed-callback-retained" in result.stdout + + def test_scalar_containers_remain_usable_after_scope_exit(): with RoApartment(), projected_lifetime_scope(): numbers = DynWinRTArray.from_i32_values([7, 11]) @@ -201,6 +286,7 @@ def test_scalar_containers_remain_usable_after_scope_exit(): def test_explicit_release_of_com_containers_is_idempotent_and_keeps_source_live(): with RoApartment(): boxed = DynWinRTValue.activation_factory("Windows.Foundation.Uri") + identity = boxed.identity_raw() array = DynWinRTArray.from_object_values([boxed], DynWinRTType.object()) shape = DynWinRTType.struct_type("Tests.ExplicitObject", [DynWinRTType.object()]) record = DynWinRTStruct.create(shape) @@ -211,7 +297,7 @@ def test_explicit_release_of_com_containers_is_idempotent_and_keeps_source_live( container.release() container.release() assert container.is_released() - assert not boxed.is_released() + assert not boxed.is_released() and boxed.identity_raw() == identity with pytest.raises(RuntimeError, match="DynWinRTArray.release"): array.to_value() with pytest.raises(RuntimeError, match="DynWinRTStruct.release"): From 61c319eed27bcf421fc37614fc99bbc7a67fb62c Mon Sep 17 00:00:00 2001 From: Leilei Zhang Date: Wed, 30 Sep 2026 09:47:36 +0800 Subject: [PATCH 11/15] Validate Python array payloads before owning native references Reject mismatched scalar/object and unsupported nested array elements before an ArrayData takes independent ownership. Reuse native WinRT argument validation and typed QueryInterface while preserving null, enum, Char16 and HRESULT aliases. Detect COM owners recursively in actual array payloads as well as declared metadata at every Python tracking boundary; reproduce and prevent both apartment-exit crashes with source-reference balance regressions. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- bindings/py/README.md | 8 + bindings/py/src/runtime.rs | 27 +-- .../tests/test_native_container_lifetime.py | 146 +++++++++++++++ crates/dynwinrt/src/array.rs | 166 ++++++++++++++++++ .../src/metadata_table/type_handle.rs | 12 ++ crates/dynwinrt/src/native_call.rs | 5 +- crates/dynwinrt/src/value.rs | 11 ++ 7 files changed, 352 insertions(+), 23 deletions(-) diff --git a/bindings/py/README.md b/bindings/py/README.md index 36827b7a..bf83ca09 100644 --- a/bindings/py/README.md +++ b/bindings/py/README.md @@ -790,6 +790,14 @@ silently emptied storage. Scalar-only containers remain usable after the scope. Without a scope, call `release()` on COM-bearing arrays and structs inside their apartment; repeated release is safe. +`DynWinRTArray.from_values()` and `from_object_values()` validate every +element against its declared native type before retaining an independent +reference. Mismatched scalar/object or struct identities and unsupported +nested array elements raise `OSError` instead of storing a value that cannot +be marshaled safely; nullable interface elements still accept native null. +For arrays produced by lower-level native paths, scope tracking also checks +the **actual owned elements**, not only the declared array element type. + Scopes nest in LIFO order. Wrappers and raw native results that survive a closed scope remain Python objects, but their COM references are released: using one afterwards, as the diff --git a/bindings/py/src/runtime.rs b/bindings/py/src/runtime.rs index f2b45da3..39302c75 100644 --- a/bindings/py/src/runtime.rs +++ b/bindings/py/src/runtime.rs @@ -1118,14 +1118,6 @@ pub(crate) fn init_native_tracking(module: &Bound<'_, PyModule>) -> PyResult<()> Ok(()) } -fn contains_com_references(typ: &dynwinrt::TypeHandle) -> bool { - let kind = typ.kind(); - kind.is_com_pointer() - || matches!(kind, dynwinrt::TypeKind::ArrayOfIUnknown) - || (matches!(kind, dynwinrt::TypeKind::Struct(_)) - && (0..typ.field_count()).any(|index| contains_com_references(&typ.field_type(index)))) -} - fn track_native_owner(py: Python<'_>, owner: Py) -> PyResult<()> { if let Some(track) = TRACK_NATIVE.get(py) { track.call1(py, (owner,))?; @@ -1139,14 +1131,7 @@ pub(crate) fn tracked_native_value( py: Python<'_>, value: dynwinrt::WinRTValue, ) -> PyResult> { - let owns_native = match &value { - dynwinrt::WinRTValue::Object(_) - | dynwinrt::WinRTValue::Async(_) - | dynwinrt::WinRTValue::ArrayOfIUnknown(_) => true, - dynwinrt::WinRTValue::Array(array) => contains_com_references(&array.element_type), - dynwinrt::WinRTValue::Struct(data) => contains_com_references(&data.type_handle()), - _ => false, - }; + let owns_native = value.contains_com_references(); let output = Py::new(py, DynWinRTValue::new(value))?; if owns_native { track_native_owner(py, output.clone_ref(py).into_any())?; @@ -1155,7 +1140,7 @@ pub(crate) fn tracked_native_value( } fn tracked_native_array(py: Python<'_>, array: dynwinrt::ArrayData) -> PyResult> { - let owns_com = contains_com_references(&array.element_type); + let owns_com = array.contains_com_references(); let output = Py::new(py, DynWinRTArray(Some(array)))?; if owns_com { track_native_owner(py, output.clone_ref(py).into_any())?; @@ -1167,7 +1152,7 @@ fn tracked_native_struct( py: Python<'_>, data: dynwinrt::ValueTypeData, ) -> PyResult> { - let owns_com = contains_com_references(&data.type_handle()); + let owns_com = data.type_handle().contains_com_references(); let output = Py::new(py, DynWinRTStruct(Some(data)))?; if owns_com { track_native_owner(py, output.clone_ref(py).into_any())?; @@ -1994,10 +1979,8 @@ impl DynWinRTArray { element_type: &DynWinRTType, ) -> PyResult { let values = native_inputs(operation, values, InputSlot::Element)?; - Ok(dynwinrt::ArrayData::from_values( - element_type.0.clone(), - &values, - )) + dynwinrt::ArrayData::try_from_values(element_type.0.clone(), &values) + .map_err(map_windows_error) } } diff --git a/bindings/py/tests/test_native_container_lifetime.py b/bindings/py/tests/test_native_container_lifetime.py index 913035a7..d9bbf68c 100644 --- a/bindings/py/tests/test_native_container_lifetime.py +++ b/bindings/py/tests/test_native_container_lifetime.py @@ -204,6 +204,85 @@ def dispatch(index, slot, args): print('borrowed-callback-retained', flush=True) """ +_REJECT_INVALID_ARRAY = r""" +import sys +from dynwinrt import ( + DynWinRTArray, DynWinRTType, DynWinRTValue, DynWinRTImplementation, + DynWinRTImplementationMethod, DynWinRTInterfacePlan, DynWinRTMethodSig, + RoApartment, WinGUID, projected_lifetime_scope, +) + +mode = sys.argv[1] +signature = DynWinRTMethodSig().add_out(DynWinRTType.hstring()) +iid = WinGUID.parse('96369f54-8eb6-48f0-abce-c1b211e627c3') +interface = DynWinRTType.register_interface( + 'Tests.IArrayContractOwner', iid, +).add_method('ToString', signature) +plan = DynWinRTInterfacePlan.create( + 'Tests.IArrayContractOwner', interface, + [DynWinRTImplementationMethod('ToString', 6, signature)], +) +with RoApartment(), projected_lifetime_scope(): + owner = DynWinRTImplementation.create( + [plan], lambda *_: [DynWinRTValue.from_hstring('alive')], + 'DynWinRT.Tests.ArrayContractOwner', + ) + source = owner.to_value() + identity = source.identity_raw() + if mode in ('i32_object', 'i32_object_helper', 'i32_object_late'): + constructor = ( + DynWinRTArray.from_values if mode != 'i32_object_helper' + else DynWinRTArray.from_object_values + ) + elements = ( + [DynWinRTValue.from_i32(17), source] + if mode == 'i32_object_late' else [source] + ) + declared = DynWinRTType.i32_type() + elif mode == 'object_scalar_late': + constructor = DynWinRTArray.from_values + elements = [source, DynWinRTValue.from_i32(17)] + declared = DynWinRTType.object() + elif mode == 'wrong_iid': + constructor = DynWinRTArray.from_object_values + elements = [source] + declared = DynWinRTType.interface( + WinGUID.parse('905a0fe0-bc53-11df-8c49-001e4fc686da') + ) # IBuffer is not implemented by the IStringable fixture. + else: + constructor = DynWinRTArray.from_values + declared = ( + DynWinRTType.i32_type() + if mode == 'nested_scalar' + else DynWinRTType.array_type(DynWinRTType.object()) + ) + elements = [] + if mode in ('nested_object', 'nested_scalar'): + inner = DynWinRTArray.from_object_values( + [source], DynWinRTType.object() + ) + elements = [inner.to_value()] + try: + constructor(elements, declared) + except OSError as error: + expected_hresult = -2147467262 if mode == 'wrong_iid' else -2147024809 + assert error.winerror == expected_hresult, error # E_NOINTERFACE / E_INVALIDARG + assert ('Array element 0' in str(error) + or 'Array element 1' in str(error) + or 'nested WinRT arrays' in str(error)), error + else: + raise AssertionError(f'{mode} accepted an unsupported array element contract') + + assert not source.is_released() and source.identity_raw() == identity + if mode in ('nested_object', 'nested_scalar'): + elements[0].release() + inner.release() + source.release() + owner.release() + assert owner.is_closed, f'{mode} retained a native reference on rejection' +print('array-contract-rejected-before-owning', mode, flush=True) +""" + @pytest.mark.parametrize( "mode", @@ -252,6 +331,73 @@ def test_scope_balances_native_implementation_container_references(mode): assert f"balanced-references {mode}" in result.stdout +@pytest.mark.parametrize( + "mode", + [ + "i32_object", + "i32_object_helper", + "i32_object_late", + "object_scalar_late", + "wrong_iid", + "nested_object", + "nested_scalar", + "nested_empty", + ], +) +def test_invalid_array_contract_fails_before_retaining_native_references(mode): + result = subprocess.run( + [sys.executable, "-B", "-c", _REJECT_INVALID_ARRAY, mode], + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, ( + mode, + hex(result.returncode & 0xFFFFFFFF), + result.stdout, + result.stderr, + ) + assert f"array-contract-rejected-before-owning {mode}" in result.stdout + + +def test_checked_array_contracts_keep_valid_null_scalars_and_struct_owners(): + with RoApartment(), projected_lifetime_scope(): + source = DynWinRTValue.activation_factory("Windows.Foundation.Uri") + identity = source.identity_raw() + object_type = DynWinRTType.object() + + objects = DynWinRTArray.from_object_values( + [source, DynWinRTValue.null_value()], object_type + ) + assert objects.get(0).identity_raw() == identity + assert objects.get(1).is_null() + assert DynWinRTArray.from_values([DynWinRTValue.null_value()], object_type).get(0).is_null() + + signed = DynWinRTType.enum_type("Tests.CheckedArrayEnum", ["One"], [1]) + assert DynWinRTArray.from_values([DynWinRTValue.from_i32(1)], signed).get(0).to_int() == 1 + assert DynWinRTArray.from_values( + [DynWinRTValue.from_u16(ord("x"))], DynWinRTType.char16() + ).get(0).to_int() == ord("x") + assert DynWinRTArray.from_values( + [DynWinRTValue.from_i32(8080)], DynWinRTType.i32_type() + ).to_i32_list() == [8080] + assert DynWinRTArray.from_values( + [DynWinRTValue.from_i32(-1)], DynWinRTType.hresult() + ).to_i32_list() == [-1] + + shape = DynWinRTType.struct_type("Tests.CheckedArrayStruct", [object_type]) + record = DynWinRTStruct.create(shape) + record.set_object(0, source) + structured = DynWinRTArray.from_values([record.to_value()], shape) + assert structured.get(0).as_struct().get_object(0).identity_raw() == identity + + structured.release() + objects.release() + record.release() + assert not source.is_released() and source.identity_raw() == identity + + def test_borrowed_callback_array_survives_scope_within_its_apartment(): result = subprocess.run( [sys.executable, "-B", "-c", _BORROWED_CALLBACK], diff --git a/crates/dynwinrt/src/array.rs b/crates/dynwinrt/src/array.rs index ae3ac68b..b6ab54fe 100644 --- a/crates/dynwinrt/src/array.rs +++ b/crates/dynwinrt/src/array.rs @@ -164,6 +164,67 @@ impl ArrayData { } } + /// Validate a Python-facing WinRT array before taking independent + /// ownership of its elements. Nested arrays have no supported element ABI. + pub fn try_from_values( + element_type: TypeHandle, + values: &[WinRTValue], + ) -> windows_core::Result { + if matches!( + element_type.kind(), + TypeKind::Array(_) + | TypeKind::ArrayOfIUnknown + | TypeKind::Generic { .. } + | TypeKind::OutValue(_) + ) { + return Err(windows_core::Error::new( + windows_core::HRESULT(0x80070057u32 as i32), + "nested WinRT arrays or ABI-only element types have no supported array element contract", + )); + } + for (index, value) in values.iter().enumerate() { + if matches!( + value, + WinRTValue::Array(_) + | WinRTValue::ArrayOfIUnknown(_) + | WinRTValue::RawPtr(_) + | WinRTValue::OutValue(..) + ) { + return Err(windows_core::Error::new( + windows_core::HRESULT(0x80070057u32 as i32), + &format!( + "Array element {index}: nested WinRT arrays or raw ABI values are unsupported" + ), + )); + } + crate::native_call::validate_array_element(&element_type, value, index)?; + } + let mut prepared = Vec::with_capacity(values.len()); + for (index, value) in values.iter().enumerate() { + let coerced = + crate::native_call::coerce_input_object(&element_type, value).map_err(|error| { + windows_core::Error::new( + error.code(), + &format!("Array element {index}: {}", error.message()), + ) + })?; + prepared.push(coerced.unwrap_or_else(|| value.clone())); + } + Ok(Self::from_owned_values(element_type, prepared)) + } + + /// Inspect the owned values as well as the declared element type: internal + /// arrays may carry a mismatched payload even if metadata claims a scalar. + pub fn contains_com_references(&self) -> bool { + self.element_type.contains_com_references() + || match &self.buffer { + ArrayBuffer::Values(values) => { + values.iter().any(WinRTValue::contains_com_references) + } + ArrayBuffer::CoTaskMem { .. } => false, + } + } + pub(crate) fn from_owned_values(element_type: TypeHandle, values: Vec) -> Self { Self { element_type, @@ -616,6 +677,111 @@ fn serialize_to_buffer(element_type: &TypeHandle, values: &[WinRTValue]) -> Vec< mod tests { use super::*; use crate::metadata_table::MetadataTable; + use windows::Foundation::{IStringable, IUriRuntimeClass, Uri}; + use windows::Win32::System::WinRT::{RO_INIT_MULTITHREADED, RoInitialize}; + use windows_core::h; + + #[test] + fn checked_elements_reject_mismatched_and_nested_payloads_before_ownership() + -> windows_core::Result<()> { + let _ = unsafe { RoInitialize(RO_INIT_MULTITHREADED) }; + let uri = Uri::CreateUri(h!("https://example.com"))?; + let source: IUriRuntimeClass = uri.cast()?; + let source = WinRTValue::Object(source.cast()?); + let table = MetadataTable::new(); + let wrong = ArrayData::try_from_values(table.i32_type(), &[source.clone()]) + .expect_err("COM input cannot be stored as I32"); + assert_eq!(wrong.code().0, 0x80070057u32 as i32); + assert!(wrong.message().contains("Array element 0")); + + let inner = WinRTValue::Array(ArrayData::from_values( + table.object(), + std::slice::from_ref(&source), + )); + for (element, values) in [ + (table.array(&table.object()), vec![]), + (table.array(&table.object()), vec![inner.clone()]), + (table.i32_type(), vec![inner]), + ] { + let error = ArrayData::try_from_values(element, &values) + .expect_err("nested arrays have no WinRT element ABI"); + assert_eq!(error.code().0, 0x80070057u32 as i32); + assert!(error.message().contains("nested WinRT arrays")); + } + assert!( + ArrayData::try_from_values( + table.object(), + &[WinRTValue::RawPtr(1usize as *mut c_void)] + ) + .is_err() + ); + assert!(source.as_object().is_some()); + Ok(()) + } + + #[test] + fn checked_elements_preserve_typed_qi_null_enum_char16_and_structs() -> windows_core::Result<()> + { + let _ = unsafe { RoInitialize(RO_INIT_MULTITHREADED) }; + let uri = Uri::CreateUri(h!("https://example.com"))?; + let default: IUriRuntimeClass = uri.cast()?; + let expected: IStringable = uri.cast()?; + let source = WinRTValue::Object(default.cast()?); + let table = MetadataTable::new(); + + let typed = ArrayData::try_from_values( + table.interface(IStringable::IID), + &[source.clone(), WinRTValue::Null], + )?; + assert_eq!(typed.len(), 2); + assert_eq!( + typed.get(0).as_object().unwrap().as_raw(), + expected.as_raw() + ); + assert!(typed.get(1).is_null_object()); + assert!(typed.contains_com_references()); + + let signed = table.enum_type("Tests.ArrayEnum", vec![("One".to_string(), 1)]); + let enumeration = ArrayData::try_from_values(signed, &[WinRTValue::I32(1)])?; + assert_eq!(enumeration.get(0).as_i32(), Some(1)); + let char16 = + ArrayData::try_from_values(table.char16_type(), &[WinRTValue::U16('x' as u16)])?; + assert_eq!(char16.get(0).as_i32(), Some('x' as i32)); + let hresult = ArrayData::try_from_values(table.hresult(), &[WinRTValue::I32(-1)])?; + assert_eq!(hresult.get_i32(0).unwrap(), -1); + + let struct_type = table.struct_type("Tests.ArrayObjectField", &[table.object()]); + let mut struct_value = struct_type.default_value(); + struct_value + .set_field_object(0, source.as_object().as_ref()) + .expect("matching Object field"); + let structs = ArrayData::try_from_values(struct_type, &[WinRTValue::Struct(struct_value)])?; + assert!(structs.contains_com_references()); + Ok(()) + } + + #[test] + fn actual_payloads_keep_mislabeled_arrays_visible_to_scope_tracking() -> windows_core::Result<()> + { + let _ = unsafe { RoInitialize(RO_INIT_MULTITHREADED) }; + let uri = Uri::CreateUri(h!("https://example.com"))?; + let source = WinRTValue::Object(uri.cast()?); + let table = MetadataTable::new(); + let mislabeled = ArrayData::from_values(table.i32_type(), &[source.clone()]); + assert!(mislabeled.contains_com_references()); + assert!(WinRTValue::Array(mislabeled).contains_com_references()); + + let inner = WinRTValue::Array(ArrayData::from_values(table.object(), &[source])); + let nested = ArrayData::from_values(table.i32_type(), &[inner]); + assert!(nested.contains_com_references()); + assert!(WinRTValue::Array(nested).contains_com_references()); + assert!(!ArrayData::empty(table.i32_type()).contains_com_references()); + assert!( + ArrayData::empty(table.array(&table.object())).contains_com_references(), + "CoTaskMem-backed arrays still rely on their declared ABI layout" + ); + Ok(()) + } #[test] fn test_null_com_element_returns_null_variant() { diff --git a/crates/dynwinrt/src/metadata_table/type_handle.rs b/crates/dynwinrt/src/metadata_table/type_handle.rs index 24a45012..59411cb2 100644 --- a/crates/dynwinrt/src/metadata_table/type_handle.rs +++ b/crates/dynwinrt/src/metadata_table/type_handle.rs @@ -136,6 +136,18 @@ impl TypeHandle { } } + /// Whether a value with this declared layout can own COM references. + pub fn contains_com_references(&self) -> bool { + match self.kind { + kind if kind.is_com_pointer() => true, + TypeKind::ArrayOfIUnknown => true, + TypeKind::Struct(_) => (0..self.field_count()) + .any(|index| self.field_type(index).contains_com_references()), + TypeKind::Array(_) => self.array_element_type().contains_com_references(), + _ => false, + } + } + /// Create a zero-initialized ValueTypeData. Only valid for Struct types. pub fn default_value(&self) -> ValueTypeData { ValueTypeData::new(self) diff --git a/crates/dynwinrt/src/native_call.rs b/crates/dynwinrt/src/native_call.rs index f3d4989e..08536dd3 100644 --- a/crates/dynwinrt/src/native_call.rs +++ b/crates/dynwinrt/src/native_call.rs @@ -1328,7 +1328,7 @@ fn invalid_argument(message: &str) -> windows_core::Error { windows_core::Error::new(windows_core::HRESULT(0x80070057u32 as i32), message) } -fn validate_array_element( +pub(crate) fn validate_array_element( expected: &TypeHandle, value: &WinRTValue, index: usize, @@ -1365,6 +1365,9 @@ fn validate_array_element( if matches!(expected.kind(), TypeKind::Char16) && matches!(value, WinRTValue::U16(_)) { return Ok(()); } + if matches!(expected.kind(), TypeKind::HResult) && matches!(value, WinRTValue::I32(_)) { + return Ok(()); + } if value.get_type_kind() != expected.kind() { return Err(windows_core::Error::new( windows_core::HRESULT(0x80070057u32 as i32), diff --git a/crates/dynwinrt/src/value.rs b/crates/dynwinrt/src/value.rs index 9ab4241b..868e3e91 100644 --- a/crates/dynwinrt/src/value.rs +++ b/crates/dynwinrt/src/value.rs @@ -217,6 +217,17 @@ impl WinRTValue { matches!(self, WinRTValue::Null) } + /// Whether this owned value contains a COM reference, including nested + /// array elements or struct fields. + pub fn contains_com_references(&self) -> bool { + match self { + Self::Object(_) | Self::Async(_) | Self::ArrayOfIUnknown(_) => true, + Self::Struct(data) => data.type_handle().contains_com_references(), + Self::Array(data) => data.contains_com_references(), + _ => false, + } + } + /// If this is an Object wrapping a null IUnknown, replace with Null to prevent /// crash on clone/drop (IUnknown::from_raw(null) is invalid). pub fn sanitize_null_object(&mut self) { From c9c933efbee4995093876cc318ad66c3e1eaf717 Mon Sep 17 00:00:00 2001 From: Leilei Zhang Date: Wed, 30 Sep 2026 10:11:00 +0800 Subject: [PATCH 12/15] Isolate stock WinRT array boundary tests within one apartment Use the owned local IStringable fixture for deterministic parallel core tests instead of repeatedly activating cached stock Uri factories across test apartments. Retain stock Uri typed-QI and mismatched-array coverage in an isolated integration executable and Python subprocess, each repeating both paths inside one balanced apartment. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../tests/test_native_container_lifetime.py | 65 +++++++++++++++++++ crates/dynwinrt/src/array.rs | 51 ++++++++++----- .../tests/stock_uri_array_boundary.rs | 60 +++++++++++++++++ 3 files changed, 161 insertions(+), 15 deletions(-) create mode 100644 crates/dynwinrt/tests/stock_uri_array_boundary.rs diff --git a/bindings/py/tests/test_native_container_lifetime.py b/bindings/py/tests/test_native_container_lifetime.py index d9bbf68c..f113dd3a 100644 --- a/bindings/py/tests/test_native_container_lifetime.py +++ b/bindings/py/tests/test_native_container_lifetime.py @@ -283,6 +283,55 @@ def dispatch(index, slot, args): print('array-contract-rejected-before-owning', mode, flush=True) """ +_STOCK_URI_ARRAY = r""" +from dynwinrt import ( + DynWinRTArray, DynWinRTMethodSig, DynWinRTType, DynWinRTValue, + RoApartment, WinGUID, projected_lifetime_scope, +) + +factory_iid = WinGUID.parse('44a9796f-723e-4fdf-a218-033e75b0c084') +stringable_iid = WinGUID.parse('96369f54-8eb6-48f0-abce-c1b211e627c3') +factory_type = DynWinRTType.register_interface( + 'Tests.IUriRuntimeClassFactoryArrayBoundary', factory_iid, +).add_method( + 'CreateUri', + DynWinRTMethodSig() + .add_in(DynWinRTType.hstring()) + .add_out(DynWinRTType.object()), +) + +with RoApartment(), projected_lifetime_scope(): + def exercise(url): + factory = DynWinRTValue.activation_factory( + 'Windows.Foundation.Uri' + ).cast(factory_iid) + uri = factory_type.method(6).invoke(factory, [DynWinRTValue.from_hstring(url)]) + identity = uri.identity_raw() + try: + DynWinRTArray.from_values([uri], DynWinRTType.i32_type()) + except OSError as error: + assert error.winerror == -2147024809 and 'Array element 0' in str(error) + else: + raise AssertionError('stock Uri pointer was stored in an I32 array') + + checked = DynWinRTArray.from_values( + [uri, DynWinRTValue.null_value()], + DynWinRTType.interface(stringable_iid), + ) + typed = checked.get(0) + assert typed.identity_raw() == identity + assert typed.as_raw() != uri.as_raw(), 'typed element did not QueryInterface' + assert checked.get(1).is_null() + assert uri.identity_raw() == identity and not uri.is_released() + return uri, checked + + first_uri, first_array = exercise('https://example.com/first') + second_uri, second_array = exercise('https://example.com/second') +assert first_uri.is_released() and first_array.is_released() +assert second_uri.is_released() and second_array.is_released() +print('stock-uri-array-one-apartment', flush=True) +""" + @pytest.mark.parametrize( "mode", @@ -361,6 +410,22 @@ def test_invalid_array_contract_fails_before_retaining_native_references(mode): assert f"array-contract-rejected-before-owning {mode}" in result.stdout +def test_stock_uri_checked_arrays_repeat_within_one_apartment(): + result = subprocess.run( + [sys.executable, "-B", "-c", _STOCK_URI_ARRAY], + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, ( + hex(result.returncode & 0xFFFFFFFF), + result.stdout, + result.stderr, + ) + assert "stock-uri-array-one-apartment" in result.stdout + + def test_checked_array_contracts_keep_valid_null_scalars_and_struct_owners(): with RoApartment(), projected_lifetime_scope(): source = DynWinRTValue.activation_factory("Windows.Foundation.Uri") diff --git a/crates/dynwinrt/src/array.rs b/crates/dynwinrt/src/array.rs index b6ab54fe..8f8e891b 100644 --- a/crates/dynwinrt/src/array.rs +++ b/crates/dynwinrt/src/array.rs @@ -677,18 +677,41 @@ fn serialize_to_buffer(element_type: &TypeHandle, values: &[WinRTValue]) -> Vec< mod tests { use super::*; use crate::metadata_table::MetadataTable; - use windows::Foundation::{IStringable, IUriRuntimeClass, Uri}; - use windows::Win32::System::WinRT::{RO_INIT_MULTITHREADED, RoInitialize}; - use windows_core::h; + use crate::{ + MethodSignature, WinRtImplementation, WinRtImplementationPlan, WinRtInterfaceDefinition, + WinRtMethodDefinition, WinRtThreadingPolicy, + }; + use std::sync::Arc; + use windows::Foundation::IStringable; + + fn stringable_owner(table: &Arc) -> windows_core::Result { + let signature = MethodSignature::new(table).add_out(table.hstring()); + let plan = WinRtImplementationPlan::new( + vec![WinRtInterfaceDefinition { + name: "Windows.Foundation.IStringable".into(), + interface_type: table.interface(IStringable::IID), + required_iids: vec![], + methods: vec![WinRtMethodDefinition { + name: "ToString".into(), + vtable_index: 6, + signature, + }], + }], + WinRtThreadingPolicy::OwnerThread, + )?; + WinRtImplementation::new( + plan, + Arc::new(|_, _, _| Ok(vec![WinRTValue::HString("array".into())])), + Some("DynWinRt.Tests.Array"), + ) + } #[test] fn checked_elements_reject_mismatched_and_nested_payloads_before_ownership() -> windows_core::Result<()> { - let _ = unsafe { RoInitialize(RO_INIT_MULTITHREADED) }; - let uri = Uri::CreateUri(h!("https://example.com"))?; - let source: IUriRuntimeClass = uri.cast()?; - let source = WinRTValue::Object(source.cast()?); let table = MetadataTable::new(); + let owner = stringable_owner(&table)?; + let source = owner.to_value()?; let wrong = ArrayData::try_from_values(table.i32_type(), &[source.clone()]) .expect_err("COM input cannot be stored as I32"); assert_eq!(wrong.code().0, 0x80070057u32 as i32); @@ -722,12 +745,11 @@ mod tests { #[test] fn checked_elements_preserve_typed_qi_null_enum_char16_and_structs() -> windows_core::Result<()> { - let _ = unsafe { RoInitialize(RO_INIT_MULTITHREADED) }; - let uri = Uri::CreateUri(h!("https://example.com"))?; - let default: IUriRuntimeClass = uri.cast()?; - let expected: IStringable = uri.cast()?; - let source = WinRTValue::Object(default.cast()?); let table = MetadataTable::new(); + let owner = stringable_owner(&table)?; + let source = owner.to_value()?; + let expected: IStringable = source.as_object().unwrap().cast()?; + assert_ne!(source.as_object().unwrap().as_raw(), expected.as_raw()); let typed = ArrayData::try_from_values( table.interface(IStringable::IID), @@ -763,10 +785,9 @@ mod tests { #[test] fn actual_payloads_keep_mislabeled_arrays_visible_to_scope_tracking() -> windows_core::Result<()> { - let _ = unsafe { RoInitialize(RO_INIT_MULTITHREADED) }; - let uri = Uri::CreateUri(h!("https://example.com"))?; - let source = WinRTValue::Object(uri.cast()?); let table = MetadataTable::new(); + let owner = stringable_owner(&table)?; + let source = owner.to_value()?; let mislabeled = ArrayData::from_values(table.i32_type(), &[source.clone()]); assert!(mislabeled.contains_com_references()); assert!(WinRTValue::Array(mislabeled).contains_com_references()); diff --git a/crates/dynwinrt/tests/stock_uri_array_boundary.rs b/crates/dynwinrt/tests/stock_uri_array_boundary.rs new file mode 100644 index 00000000..4bcea302 --- /dev/null +++ b/crates/dynwinrt/tests/stock_uri_array_boundary.rs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use dynwinrt::{ArrayData, MetadataTable, WinRTValue}; +use windows::Foundation::{IStringable, IUriRuntimeClass, Uri}; +use windows::Win32::System::WinRT::{RO_INIT_MULTITHREADED, RoInitialize, RoUninitialize}; +use windows_core::{Interface, h}; + +struct Apartment; + +impl Apartment { + fn enter() -> windows_core::Result { + unsafe { RoInitialize(RO_INIT_MULTITHREADED) }?; + Ok(Self) + } +} + +impl Drop for Apartment { + fn drop(&mut self) { + unsafe { RoUninitialize() }; + } +} + +// A separate test executable keeps stock activation inside one apartment. +#[test] +fn stock_uri_array_ownership_and_typed_qi_share_one_apartment() -> windows_core::Result<()> { + let _apartment = Apartment::enter()?; + let table = MetadataTable::new(); + + { + let uri = Uri::CreateUri(h!("https://example.com/first"))?; + let default: IUriRuntimeClass = uri.cast()?; + let source = WinRTValue::Object(default.cast()?); + let mislabeled = ArrayData::from_values(table.i32_type(), &[source.clone()]); + assert!(mislabeled.contains_com_references()); + assert!(WinRTValue::Array(mislabeled).contains_com_references()); + assert!(ArrayData::try_from_values(table.i32_type(), &[source.clone()]).is_err()); + let inner = WinRTValue::Array(ArrayData::from_values(table.object(), &[source])); + assert!(ArrayData::try_from_values(table.array(&table.object()), &[inner]).is_err()); + } + + { + let uri = Uri::CreateUri(h!("https://example.com/second"))?; + let default: IUriRuntimeClass = uri.cast()?; + let stringable: IStringable = uri.cast()?; + assert_ne!(default.as_raw(), stringable.as_raw()); + let source = WinRTValue::Object(default.cast()?); + let checked = ArrayData::try_from_values( + table.interface(IStringable::IID), + &[source.clone(), WinRTValue::Null], + )?; + assert_eq!( + checked.get(0).as_object().unwrap().as_raw(), + stringable.as_raw() + ); + assert!(checked.get(1).is_null_object()); + assert!(source.as_object().is_some()); + } + Ok(()) +} From cb3a88a92a1ae33331edbd5956368167f256c990 Mon Sep 17 00:00:00 2001 From: Leilei Zhang Date: Wed, 30 Sep 2026 16:59:54 +0800 Subject: [PATCH 13/15] Close Python COM owners before managed apartment teardown Weakly track raw, container, callback, implementation and async owners so the final managed RoUninitialize releases only their references on the owner thread. Guard cross-thread access and add an explicit callback shutdown gate for embedding hosts, with isolated real-finalization and generated-code regressions. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- bindings/py/README.md | 122 +- bindings/py/dynwinrt.pyi | 6 + bindings/py/src/async_runtime.rs | 351 +++- bindings/py/src/delegate_method.rs | 10 +- bindings/py/src/errors.rs | 6 +- bindings/py/src/implementation.rs | 136 +- bindings/py/src/lib.rs | 103 +- bindings/py/src/runtime.rs | 1687 +++++++++++++++-- bindings/py/tests/test_error_messages.py | 3 +- bindings/py/tests/test_implementation.py | 5 +- .../tests/test_native_container_lifetime.py | 1325 +++++++++++++ bindings/py/tests/test_object_value_view.py | 3 +- bindings/py/tests/test_phase1.py | 29 + tests/e2e/e2e_specs.json | 11 +- tests/e2e/e2e_specs.schema.json | 3 + tests/e2e/runners/implementation_py.py | 3 +- tests/e2e/runners/py_runner.py | 183 +- .../tests/python_raw_lifetime_test.rs | 286 ++- .../python_released_implementation_test.rs | 4 +- 19 files changed, 4005 insertions(+), 271 deletions(-) diff --git a/bindings/py/README.md b/bindings/py/README.md index bf83ca09..95128ad6 100644 --- a/bindings/py/README.md +++ b/bindings/py/README.md @@ -531,7 +531,9 @@ with RoApartment(RO_INIT_SINGLETHREADED): model are supported. Requesting a conflicting model raises `OSError` with `RPC_E_CHANGED_MODE`. The low-level `ro_initialize()` API remains available, but each successful call, including `S_FALSE`, must be paired with one -`ro_uninitialize()` call on the same thread. +`ro_uninitialize()` call on the same thread. Only this library's successful +initializations count toward its managed apartment depth; a third-party COM +initialization is not counted. WinRT is never initialized implicitly. A call on a thread without an apartment raises `OSError` with `CO_E_NOTINITIALIZED` in `error.winerror`; its message @@ -756,9 +758,8 @@ stay on the caller's native thread but release the Python GIL while WinUI pumps messages. WinRT callbacks reacquire the GIL, and worker threads can use `DispatcherQueue.try_enqueue()` to return to the UI thread. -Use a projection lifetime scope inside the COM apartment so projected wrappers -and raw native `DynWinRTValue` outputs release their owned COM references -before `RoUninitialize`: +Use a projection lifetime scope for deterministic early cleanup inside the COM +apartment: ```python from dynwinrt import RO_INIT_SINGLETHREADED, RoApartment, projected_lifetime_scope @@ -770,53 +771,102 @@ with RoApartment(RO_INIT_SINGLETHREADED), projected_lifetime_scope(): Native factory and method outputs are tracked automatically, even when a generated factory returns a bare `DynWinRTValue` (for example, -`PropertyValue.create_uint32(8080)`). A raw result can escape the Python -function that created it, but after the scope closes it reports -`is_released() == True`; it cannot be used outside the apartment. Without a -scope, explicitly call `release()` on every retained native result before -leaving `RoApartment`. Pure scalar results do not own COM references and remain -usable after a scope closes. The scope retains projected wrappers strongly but -observes raw native outputs weakly: temporary casts and callback inputs can -drop normally instead of accumulating references until the scope closes. -Raw results still held by Python at scope exit are released there. +`PropertyValue.create_uint32(8080)`). Independently owned COM references made +inside `RoApartment` or after `ro_initialize()` are also observed weakly +without an explicit scope: any still-live owners release their own references +on the creating thread before the final *dynwinrt-managed* `RoUninitialize`. +The raw value or generated wrapper remains a Python object, but calls after +that boundary raise `RuntimeError` instead of releasing a native pointer in an +uninitialized apartment. Earlier `projected_lifetime_scope()` disposal remains +idempotent with apartment cleanup. It retains projected wrappers strongly, +but observes raw native outputs weakly, so ordinary temporary casts still drop +early. Pure scalar results do not own COM references and remain usable after +either boundary. An object created under an external COM initialization alone +is not automatically tied to a dynwinrt-managed apartment. `DynWinRTArray` and `DynWinRTStruct` can independently own COM references, including after `DynWinRTValue.as_array()` or `.as_struct()` clones an already -tracked value. COM-bearing containers created or extracted inside a scope are -also observed weakly and released before the apartment exits. Retained -containers then report `is_released()` and reject reads, writes and +tracked value. COM-bearing containers created or extracted inside a managed +apartment are observed weakly and released before its final exit, even without +an explicit scope. Retained containers report `is_released()` and reject reads, writes and `to_value()` with the released-object `RuntimeError`, rather than exposing -silently emptied storage. Scalar-only containers remain usable after the -scope. Without a scope, call `release()` on COM-bearing arrays and structs -inside their apartment; repeated release is safe. +silently emptied storage. Scalar-only containers remain usable. Calling +`release()` early is repeat-safe and drops only the container's own references. +Cross-thread access and explicit release require the creating thread unless +every contained COM reference is agile. This also applies after mutating a +struct's object or nested-struct fields. An implicit foreign-thread Drop of a +non-agile container quarantines its own reference with a native diagnostic +rather than calling COM `Release` from the wrong apartment; agile containers +release normally on either thread. `DynWinRTArray.from_values()` and `from_object_values()` validate every element against its declared native type before retaining an independent -reference. Mismatched scalar/object or struct identities and unsupported -nested array elements raise `OSError` instead of storing a value that cannot -be marshaled safely; nullable interface elements still accept native null. -For arrays produced by lower-level native paths, scope tracking also checks +reference. Array, vector, map, delegate, and method inputs first validate +borrowed Python handles, then clone native COM references; a foreign-thread +non-agile input cannot cause `AddRef` before its thread error. Mismatched +scalar/object or struct identities and unsupported nested array elements +raise `OSError` instead of storing a value that cannot be marshaled safely; +nullable interface elements still accept native null. +For arrays produced by lower-level native paths, lifetime tracking also checks the **actual owned elements**, not only the declared array element type. Scopes nest in LIFO order. Wrappers and raw native results that survive a -closed scope remain Python objects, but their COM references are released: -using one afterwards, as the -object of a call, as an argument, or inside a sequence, mapping, array, or -struct input, raises `RuntimeError` explaining that it was released, as it -does after `release_projected(wrapper)` or `DynWinRTValue.release()`. +closed scope or final managed apartment exit remain Python objects, but their +COM references are released. Using one afterwards as the object of a call, as +an argument, or inside a sequence, mapping, array, or struct input raises +`RuntimeError` explaining that it was released, as it does after +`release_projected(wrapper)` or `DynWinRTValue.release()`. Returning one from an interface implementation handler fails the native call like any other handler error. `DynWinRTValue.is_released()` tells a released value apart from a WinRT null reference: both report `is_null()`, but only the null can still be passed. Each scope is thread-affine: enter, use, and close it inside that thread's `RoApartment`. Same-thread asyncio tasks inherit the -active scope, while worker threads must open their own ordered -`with RoApartment(...), projected_lifetime_scope():`. -Native callbacks invoked -on a foreign thread preserve other captured context but do not inherit the -creator thread's lifetime scope. This includes generated delegates, raw progress -handlers, and element-factory callbacks. Retained callback values remain -user-owned; open an explicit callback-local scope for deterministic temporary -cleanup. +active scope; worker threads use their own `RoApartment` and optionally their +own earlier-cleanup scope. Native callbacks invoked on a foreign thread +preserve other captured context but do not inherit the subscriber's lifetime +scope. Independently cloned non-agile callback inputs created on a managed +thread stay usable after an explicit scope, but are released when that thread's +managed apartment finally closes. Agile callback inputs can be retained and +released normally; a non-agile progress input cannot cross callback threads +and reports an error instead. + +An unfinished async future prevents the final apartment close without +silently cancelling work. Settle or explicitly cancel it, then call the +owner-thread guard's `close()` again. If an unnamed `RoApartment` context's +close failed, use `retry_pending_apartment_close()` on the same thread. A +non-agile WinRT async operation awaited from asyncio checks completion on its +own apartment thread rather than passing its native reference to a worker. +Explicitly cancelling that asyncio task calls native `IAsyncInfo::Cancel` on +the owner thread; ordinary apartment cleanup never cancels an external task. +Calls to `RoApartment.close()` on another OS thread raise without changing the +apartment state. Only an implicit wrong-thread finalizer or interpreter +shutdown without a usable GIL can force a diagnostic and retain unsafe native +references until process exit; normal close never treats a leak as success. + +### Embedded host callback shutdown + +If an embedded host retains native aliases to Python-backed delegates, +element factories, implementations, or progress handlers, it must close their +Python callback entry **while Python is still alive**, before `Py_FinalizeEx`: + +1. Stop other native threads from invoking those aliases and settle callbacks + already in flight (including queued progress delivery). +2. Call `dynwinrt.shutdown_python_callbacks()` on the live interpreter. + This gate is idempotent; if a native callback is still active it raises + `RuntimeError` without closing, so settle that callback and retry. +3. Finalize Python only after the gate succeeds. Release external COM aliases + according to their own ownership contract; closing an apartment never + disconnects them or cancels external work. + +After the gate closes, late native delegate, element-factory, XAML, and +implementation entrypoints return `RO_E_CLOSED` **without attaching to +Python**. Python-backed callback creation also fails explicitly. A void +progress callback cannot return an HRESULT and instead issues a best-effort +native diagnostic. Hosts that skip the explicit gate have only PyO3's +best-effort shutdown detection: a foreign callback during the early, +unobservable part of `Py_FinalizeEx` cannot be guaranteed deadlock-free across +all supported Python versions. This external-callback limitation is separate +from the managed-apartment owner-release guarantee for #189. Normal construction remains unavailable for protected-only composable classes and system-returned classes without public activation metadata. Named Python diff --git a/bindings/py/dynwinrt.pyi b/bindings/py/dynwinrt.pyi index 0b2501a9..f029fb7e 100644 --- a/bindings/py/dynwinrt.pyi +++ b/bindings/py/dynwinrt.pyi @@ -57,6 +57,8 @@ __all__ = [ "init_winappsdk", "ro_initialize", "ro_uninitialize", + "retry_pending_apartment_close", + "shutdown_python_callbacks", "register_xaml_runtime_class", "has_package_identity", "get_winappsdk_resource_pri_path", @@ -681,6 +683,8 @@ class DynWinRtDelegate: callback: Callable[..., object], ) -> DynWinRtDelegate: ... def to_value(self) -> DynWinRTValue: ... + def is_released(self) -> bool: ... + def release(self) -> None: ... @final @@ -700,6 +704,8 @@ class DynWinRtElementFactory: def init_winappsdk(major: int, minor: int) -> WinAppSDKContext: ... def ro_initialize(apartment_type: Optional[int]) -> None: ... def ro_uninitialize() -> None: ... +def retry_pending_apartment_close() -> None: ... +def shutdown_python_callbacks() -> None: ... def has_package_identity() -> bool: ... def get_winappsdk_resource_pri_path() -> str: ... def get_computer_name() -> str: ... diff --git a/bindings/py/src/async_runtime.rs b/bindings/py/src/async_runtime.rs index 3e36c275..642f6ba1 100644 --- a/bindings/py/src/async_runtime.rs +++ b/bindings/py/src/async_runtime.rs @@ -4,11 +4,17 @@ use std::cell::Cell; use std::future::IntoFuture; use std::sync::{Arc, Mutex, MutexGuard}; +use std::thread::{self, ThreadId}; use crate::errors::{ map_dynwinrt_error, map_dynwinrt_error_with_context, map_windows_error_with_context, }; -use crate::runtime::{DynWinRTValue, tracked_native_value}; +use crate::runtime::{ + DynWinRTValue, callback_native_argument, current_native_owner_thread, + ensure_native_owner_thread, ensure_python_callbacks_open, log_unsafe_native_owner_drop, + must_quarantine_owner, native_value_is_agile, track_native_owner, tracked_native_value, + with_python_callback, +}; use pyo3::exceptions::{PyRuntimeError, PyTypeError}; use pyo3::prelude::*; use pyo3::types::PyList; @@ -18,6 +24,7 @@ use windows::Win32::System::Com::{ CoGetApartmentType, }; use windows::Win32::System::WinRT::{RO_INIT_MULTITHREADED, RoInitialize, RoUninitialize}; +use windows::core::Interface; thread_local! { static TOKIO_RO_INITIALIZED: Cell = const { Cell::new(false) }; @@ -178,6 +185,17 @@ impl CoroutineProtocol { } } + fn can_drop_on_foreign_thread(&self) -> bool { + self.state.try_lock().is_ok_and(|state| { + matches!( + &*state, + CoroutineExecutionState::New + | CoroutineExecutionState::Finished + | CoroutineExecutionState::Closed + ) + }) + } + fn lock_state(&self) -> PyResult> { self.state .lock() @@ -458,6 +476,7 @@ struct AsyncOperation { struct ProgressDispatcher { event_loop: Py, + owner_thread: ThreadId, // Loop handles retain this list, which is cleared to release captures and disable queued work. dispatch_state: Py, dispatch_progress: Py, @@ -474,7 +493,19 @@ impl ProgressDispatcher { return Ok(()); } - let raw = Py::new(py, DynWinRTValue::new(value))?; + if thread::current().id() != self.owner_thread + && value.contains_com_references() + && !native_value_is_agile(&value)? + { + return Err(PyRuntimeError::new_err( + "non-agile WinRT progress arguments cannot cross an apartment thread", + )); + } + let raw = if thread::current().id() == self.owner_thread { + callback_native_argument(py, value)? + } else { + Py::new(py, DynWinRTValue::new(value))? + }; let context = self.callback_context.call_method0(py, "copy")?; let context_run = context.getattr(py, "run")?; self.event_loop.call_method1( @@ -508,6 +539,16 @@ impl AsyncOperation { .map_err(|_| PyRuntimeError::new_err("async operation state lock was poisoned")) } + fn can_drop_on_foreign_thread(&self) -> bool { + self.state + .try_lock() + .is_ok_and(|state| matches!(&*state, ExecutionState::Idle)) + && self + .progress_dispatcher + .try_lock() + .is_ok_and(|dispatcher| dispatcher.is_none()) + } + fn stop(&self, py: Python<'_>) -> PyResult<()> { let progress_result = self.clear_progress_dispatcher(py); let cancel_result = self.cancel(); @@ -568,7 +609,50 @@ impl AsyncOperation { } } + fn ensure_apartment_release_safe(&self, py: Python<'_>) -> PyResult<()> { + let future = { + let state = self.lock_state()?; + match &*state { + ExecutionState::Blocking => { + return Err(PyRuntimeError::new_err( + "cannot close the COM apartment while wait() is running; retry on the owner thread", + )); + } + ExecutionState::Future(future) => Some(future.clone_ref(py)), + ExecutionState::Idle => None, + } + }; + if let Some(future) = &future + && !future.call_method0(py, "done")?.extract::(py)? + { + return Err(PyRuntimeError::new_err( + "cannot close the COM apartment while a WinRT async future is pending; await or explicitly cancel it, then retry on the owner thread", + )); + } + let dynwinrt::WinRTValue::Async(info) = &self.value else { + return Err(PyRuntimeError::new_err( + "cannot close the COM apartment: the async owner has no native operation", + )); + }; + let started = info.is_started().map_err(map_dynwinrt_error)?; + if started { + let agile = info.info.cast::().is_ok(); + if !agile { + return Err(PyRuntimeError::new_err( + "cannot close the COM apartment while a non-agile WinRT async reference may outlive it; release it on this thread or retry after completion", + )); + } + if started && self.lock_progress_dispatcher()?.is_some() { + return Err(PyRuntimeError::new_err( + "cannot close the COM apartment while a WinRT progress callback is pending; settle the operation and retry on the owner thread", + )); + } + } + Ok(()) + } + fn future<'py>(&self, py: Python<'py>) -> PyResult> { + ensure_python_callbacks_open()?; let mut state = self.lock_state()?; match &*state { ExecutionState::Future(future) => return Ok(future.clone_ref(py).into_bound(py)), @@ -580,13 +664,38 @@ impl AsyncOperation { ExecutionState::Idle => {} } - let value = self.value.clone(); - let winrt_future = value.into_future().defer_get_results().cancel_on_drop(); - let raw_future = pyo3_async_runtimes::tokio::future_into_py(py, async move { - let result = winrt_future.await; - let result = result.map_err(map_dynwinrt_error)?; - Ok(DynWinRTValue::new(result)) - })?; + let agile = matches!( + &self.value, + dynwinrt::WinRTValue::Async(info) + if info.info.cast::().is_ok() + ); + let (raw_future, cancellation_owner) = if agile { + let value = self.value.clone(); + let winrt_future = value.into_future().defer_get_results().cancel_on_drop(); + let raw_future = pyo3_async_runtimes::tokio::future_into_py(py, async move { + let result = winrt_future.await; + let result = result.map_err(map_dynwinrt_error)?; + Ok(DynWinRTValue::new(result)) + })?; + (raw_future, None) + } else { + let loop_ = py.import("asyncio")?.call_method0("get_running_loop")?; + let raw_future = loop_.call_method0("create_future")?; + let native = tracked_native_value(py, self.value.clone())?; + let poll = py + .import("dynwinrt.dynwinrt")? + .getattr("_dynwinrt_poll_nonagile_async")?; + loop_.call_method1( + "call_soon", + ( + poll, + loop_.clone(), + raw_future.clone(), + native.clone_ref(py), + ), + )?; + (raw_future, Some(native)) + }; let converter = self.converter.clone_ref(py); let convert_future = py @@ -599,7 +708,7 @@ impl AsyncOperation { .call_method1("create_task", (coroutine,))?; py.import("dynwinrt.dynwinrt")? .getattr("_dynwinrt_link_cancellation")? - .call1((future.clone(), raw_future))?; + .call1((future.clone(), raw_future, cancellation_owner))?; let future = future.unbind(); let result = future.clone_ref(py).into_bound(py); *state = ExecutionState::Future(future); @@ -664,10 +773,31 @@ pub(crate) fn finish_progress_registration( } } -#[pyclass(name = "_DynWinRTAsync")] +#[pyclass(name = "_DynWinRTAsync", weakref)] pub struct DynWinRTAsync { operation: Option>, coroutine: CoroutineProtocol, + owner_thread: Option, + release_any_thread: bool, +} + +impl Drop for DynWinRTAsync { + fn drop(&mut self) { + let safe_foreign = self.release_any_thread + && self.coroutine.can_drop_on_foreign_thread() + && self + .operation + .as_ref() + .is_none_or(|operation| operation.can_drop_on_foreign_thread()); + if must_quarantine_owner(self.owner_thread, safe_foreign) { + std::mem::forget(self.operation.take()); + std::mem::forget(std::mem::replace( + &mut self.coroutine, + CoroutineProtocol::new(), + )); + log_unsafe_native_owner_drop(); + } + } } impl DynWinRTAsync { @@ -676,16 +806,42 @@ impl DynWinRTAsync { .as_ref() .ok_or_else(|| PyRuntimeError::new_err("the WinRT async operation has been released")) } + + fn release_apartment_owner(&mut self, py: Python<'_>) -> PyResult<()> { + if let Some(operation) = &self.operation { + ensure_native_owner_thread(self.owner_thread, false, "_DynWinRTAsync")?; + operation.ensure_apartment_release_safe(py)?; + } + drop(self.operation.take()); + Ok(()) + } } #[pymethods] impl DynWinRTAsync { #[new] - fn new(value: &DynWinRTValue, result_converter: Py) -> PyResult { - Ok(Self { - operation: Some(Arc::new(AsyncOperation::new(value, result_converter)?)), - coroutine: CoroutineProtocol::new(), - }) + fn new( + py: Python<'_>, + value: &DynWinRTValue, + result_converter: Py, + ) -> PyResult> { + let operation = Arc::new(AsyncOperation::new(value, result_converter)?); + let release_any_thread = matches!( + &operation.value, + dynwinrt::WinRTValue::Async(info) + if info.info.cast::().is_ok() + ); + let output = Py::new( + py, + Self { + operation: Some(operation), + coroutine: CoroutineProtocol::new(), + owner_thread: current_native_owner_thread(true), + release_any_thread, + }, + )?; + track_native_owner(py, output.clone_ref(py).into_any())?; + Ok(output) } fn __await__<'py>(&self, py: Python<'py>) -> PyResult> { @@ -725,6 +881,7 @@ impl DynWinRTAsync { } fn release(&mut self, py: Python<'_>) -> PyResult<()> { + ensure_native_owner_thread(self.owner_thread, false, "_DynWinRTAsync")?; if let Some(operation) = &self.operation { operation.stop(py)?; } @@ -732,16 +889,50 @@ impl DynWinRTAsync { Ok(()) } + fn _check_apartment_release(&self, py: Python<'_>) -> PyResult<()> { + if let Some(operation) = &self.operation { + ensure_native_owner_thread(self.owner_thread, false, "_DynWinRTAsync")?; + operation.ensure_apartment_release_safe(py)?; + } + Ok(()) + } + + fn _release_apartment_owner(&mut self, py: Python<'_>) -> PyResult<()> { + self.release_apartment_owner(py) + } + fn __repr__(&self) -> &'static str { "_DynWinRTAsync(...)" } } -#[pyclass(name = "_DynWinRTAsyncWithProgress")] +#[pyclass(name = "_DynWinRTAsyncWithProgress", weakref)] pub struct DynWinRTAsyncWithProgress { operation: Option>, coroutine: CoroutineProtocol, - progress_converter: Py, + progress_converter: Option>, + owner_thread: Option, + release_any_thread: bool, +} + +impl Drop for DynWinRTAsyncWithProgress { + fn drop(&mut self) { + let safe_foreign = self.release_any_thread + && self.coroutine.can_drop_on_foreign_thread() + && self + .operation + .as_ref() + .is_none_or(|operation| operation.can_drop_on_foreign_thread()); + if must_quarantine_owner(self.owner_thread, safe_foreign) { + std::mem::forget(self.operation.take()); + std::mem::forget(self.progress_converter.take()); + std::mem::forget(std::mem::replace( + &mut self.coroutine, + CoroutineProtocol::new(), + )); + log_unsafe_native_owner_drop(); + } + } } impl DynWinRTAsyncWithProgress { @@ -750,16 +941,26 @@ impl DynWinRTAsyncWithProgress { .as_ref() .ok_or_else(|| PyRuntimeError::new_err("the WinRT async operation has been released")) } + + fn release_apartment_owner(&mut self, py: Python<'_>) -> PyResult<()> { + if let Some(operation) = &self.operation { + ensure_native_owner_thread(self.owner_thread, false, "_DynWinRTAsyncWithProgress")?; + operation.ensure_apartment_release_safe(py)?; + } + drop(self.operation.take()); + Ok(()) + } } #[pymethods] impl DynWinRTAsyncWithProgress { #[new] fn new( + py: Python<'_>, value: &DynWinRTValue, result_converter: Py, progress_converter: Py, - ) -> PyResult { + ) -> PyResult> { let operation = Arc::new(AsyncOperation::new(value, result_converter)?); let has_progress = match &operation.value { dynwinrt::WinRTValue::Async(info) => info.progress_type().is_some(), @@ -770,11 +971,23 @@ impl DynWinRTAsyncWithProgress { "value is not a WinRT async operation with progress", )); } - Ok(Self { - operation: Some(operation), - coroutine: CoroutineProtocol::new(), - progress_converter, - }) + let release_any_thread = matches!( + &operation.value, + dynwinrt::WinRTValue::Async(info) + if info.info.cast::().is_ok() + ); + let output = Py::new( + py, + Self { + operation: Some(operation), + coroutine: CoroutineProtocol::new(), + progress_converter: Some(progress_converter), + owner_thread: current_native_owner_thread(true), + release_any_thread, + }, + )?; + track_native_owner(py, output.clone_ref(py).into_any())?; + Ok(output) } fn __await__<'py>(&self, py: Python<'py>) -> PyResult> { @@ -814,6 +1027,7 @@ impl DynWinRTAsyncWithProgress { } fn release(&mut self, py: Python<'_>) -> PyResult<()> { + ensure_native_owner_thread(self.owner_thread, false, "_DynWinRTAsyncWithProgress")?; if let Some(operation) = &self.operation { operation.stop(py)?; } @@ -821,7 +1035,20 @@ impl DynWinRTAsyncWithProgress { Ok(()) } + fn _check_apartment_release(&self, py: Python<'_>) -> PyResult<()> { + if let Some(operation) = &self.operation { + ensure_native_owner_thread(self.owner_thread, false, "_DynWinRTAsyncWithProgress")?; + operation.ensure_apartment_release_safe(py)?; + } + Ok(()) + } + + fn _release_apartment_owner(&mut self, py: Python<'_>) -> PyResult<()> { + self.release_apartment_owner(py) + } + fn progress(&self, py: Python<'_>, callback: Py) -> PyResult<()> { + ensure_python_callbacks_open()?; let loop_ = py .import("asyncio")? .call_method0("get_running_loop") @@ -853,9 +1080,21 @@ impl DynWinRTAsyncWithProgress { })?; let dispatcher = Arc::new(ProgressDispatcher { event_loop: loop_, - dispatch_state: PyList::new(py, [callback, self.progress_converter.clone_ref(py)])? - .into_any() - .unbind(), + owner_thread: std::thread::current().id(), + dispatch_state: PyList::new( + py, + [ + callback, + self.progress_converter + .as_ref() + .ok_or_else(|| { + PyRuntimeError::new_err("the WinRT async operation has been released") + })? + .clone_ref(py), + ], + )? + .into_any() + .unbind(), dispatch_progress: py .import("dynwinrt.dynwinrt")? .getattr("_dynwinrt_dispatch_progress")? @@ -870,7 +1109,7 @@ impl DynWinRTAsyncWithProgress { let weak_dispatcher = Arc::downgrade(&dispatcher); let progress_callback: dynwinrt::ProgressCallback = Box::new(move |value| { - Python::attach(|py| { + let _ = with_python_callback(|py| { let Some(dispatcher) = weak_dispatcher.upgrade() else { return; }; @@ -968,4 +1207,60 @@ mod tests { assert!(ensure_progress_type_supported(&unsupported).is_err()); } } + + #[test] + fn real_python_finalization_quarantines_an_async_arc_without_cancelling() { + if std::env::var("DYNWINRT_ASYNC_FINALIZE_CHILD").as_deref() != Ok("1") { + let child = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "async_runtime::tests::real_python_finalization_quarantines_an_async_arc_without_cancelling", + "--nocapture", + ]) + .env("DYNWINRT_ASYNC_FINALIZE_CHILD", "1") + .output() + .unwrap(); + assert!( + child.status.success(), + "{}\n{}", + String::from_utf8_lossy(&child.stdout), + String::from_utf8_lossy(&child.stderr) + ); + assert!(String::from_utf8_lossy(&child.stdout).contains("async-Py_FinalizeEx-safe")); + return; + } + + use windows::System::Threading::{ThreadPool, WorkItemHandler}; + + Python::initialize(); + unsafe { RoInitialize(RO_INIT_MULTITHREADED) }.unwrap(); + let handler = WorkItemHandler::new(|_| Ok(())); + let native_operation = ThreadPool::RunAsync(&handler).unwrap(); + let value = DynWinRTValue::new(dynwinrt::WinRTValue::Async(dynwinrt::AsyncInfo { + info: native_operation.cast().unwrap(), + async_type: dynwinrt::MetadataTable::new().async_action(), + })); + let converter = Python::attach(|py| { + py.eval(c"lambda value: value", None, None) + .unwrap() + .unbind() + }); + let shared = Arc::new(AsyncOperation::new(&value, converter).unwrap()); + let observed = Arc::downgrade(&shared); + let owner = DynWinRTAsync { + operation: Some(shared), + coroutine: CoroutineProtocol::new(), + owner_thread: Some(thread::current().id()), + release_any_thread: true, + }; + drop((value, handler)); + std::mem::forget(native_operation); + + unsafe { pyo3::ffi::PyGILState_Ensure() }; + assert_eq!(unsafe { pyo3::ffi::Py_FinalizeEx() }, 0); + assert_eq!(unsafe { pyo3::ffi::Py_IsInitialized() }, 0); + drop(owner); + assert_eq!(observed.strong_count(), 1); + println!("async-Py_FinalizeEx-safe"); + } } diff --git a/bindings/py/src/delegate_method.rs b/bindings/py/src/delegate_method.rs index 4817f0a5..a5e809f0 100644 --- a/bindings/py/src/delegate_method.rs +++ b/bindings/py/src/delegate_method.rs @@ -51,18 +51,20 @@ impl DynWinRTDelegateMethod { pub(crate) fn invoke( &self, value: &Bound<'_, DynWinRTValue>, - args: Vec, + args: Vec>, ) -> PyResult>> { // Keep native pins, not a Python value borrow, across reentrant Invoke. let py = value.py(); - let value = value.try_borrow()?.clone(); - let delegate = value.query(&self.iid, "delegate Invoke()")?; + let delegate = { + let value = value.try_borrow()?; + value.query(&self.iid, "delegate Invoke()")? + }; let dynwinrt::WinRTValue::Object(object) = &delegate else { return Err(PyTypeError::new_err( "delegate invocation requires a managed WinRT delegate value", )); }; - let args = native_arguments("delegate Invoke()", args)?; + let args = native_arguments(py, "delegate Invoke()", args)?; (self.call.0)(object, &args) .map_err(map_windows_error)? .into_iter() diff --git a/bindings/py/src/errors.rs b/bindings/py/src/errors.rs index 0b341304..7aae3730 100644 --- a/bindings/py/src/errors.rs +++ b/bindings/py/src/errors.rs @@ -24,8 +24,8 @@ fn hresult_hint(code: HRESULT) -> Option<&'static str> { .find_map(|&(hinted, hint)| (hinted == code).then_some(hint)) } -const RELEASED_REASON: &str = "has been released (its projected_lifetime_scope() exited, or \ - release_projected() / DynWinRTValue.release() was called) and can no longer be used."; +const RELEASED_REASON: &str = "has been released (its projected_lifetime_scope() or managed COM \ + apartment exited, or release_projected() / DynWinRTValue.release() was called) and can no longer be used."; /// A call on a value after `release()`, including release by its lifetime scope. pub(crate) fn released_receiver_error() -> PyErr { @@ -34,7 +34,7 @@ pub(crate) fn released_receiver_error() -> PyErr { pub(crate) fn released_native_container_error(name: &str) -> PyErr { PyRuntimeError::new_err(format!( - "This {name} has been released (its projected_lifetime_scope() exited, or {name}.release() was called) and can no longer be used." + "This {name} has been released (its projected_lifetime_scope() or managed COM apartment exited, or {name}.release() was called) and can no longer be used." )) } diff --git a/bindings/py/src/implementation.rs b/bindings/py/src/implementation.rs index 8805aa27..ef885b66 100644 --- a/bindings/py/src/implementation.rs +++ b/bindings/py/src/implementation.rs @@ -20,7 +20,9 @@ use windows::core::{Error, HRESULT}; use crate::errors::map_windows_error; use crate::runtime::{ DynWinRTMethodSig, DynWinRTType, DynWinRTValue, PYWINRT_E_UNRAISABLE_PYTHON_EXCEPTION, WinGUID, - native_outputs, tracked_native_value, wrap_python_callback_context, + callback_native_argument, ensure_python_callbacks_open, log_unsafe_native_owner_drop, + native_outputs, python_gil_usable, track_native_owner, tracked_native_value_with_policy, + with_python_callback, wrap_python_callback_context, }; const RO_E_CLOSED: HRESULT = HRESULT(0x80000013_u32 as i32); @@ -192,7 +194,7 @@ impl CallbackCell { if self.interpreter.stopping.load(Ordering::Acquire) { return Err(closed_error()); } - Python::try_attach(|py| { + with_python_callback(|py| { if self.interpreter.stopping.load(Ordering::Acquire) { return Err(closed_error()); } @@ -206,11 +208,11 @@ impl CallbackCell { let result = (|| -> PyResult> { let inputs = args .iter() - .map(|value| Py::new(py, DynWinRTValue::new(value.clone()))) + .map(|value| callback_native_argument(py, value.clone())) .collect::>>()?; let inputs = PyList::new(py, inputs)?; let outputs = callback.call1(py, (interface_index, vtable_index, inputs))?; - native_outputs("implementation callback", outputs.extract(py)?) + native_outputs(py, "implementation callback", outputs.extract(py)?) })(); result.map_err(|error| { let message = format!( @@ -230,6 +232,23 @@ pub struct DynWinRTImplementation { native: Mutex>, callback: Weak, interpreter: Arc, + created_with_managed_apartment: bool, +} + +impl Drop for DynWinRTImplementation { + fn drop(&mut self) { + if self.created_with_managed_apartment && !python_gil_usable() { + let native = self + .native + .get_mut() + .unwrap_or_else(|error| error.into_inner()) + .take(); + if native.is_some() { + std::mem::forget(native); + log_unsafe_native_owner_drop(); + } + } + } } struct NativeLease<'a> { @@ -297,7 +316,8 @@ impl DynWinRTImplementation { interfaces: Vec, callback: Py, runtime_class_name: Option<&str>, - ) -> PyResult { + ) -> PyResult> { + ensure_python_callbacks_open()?; // PyGILState attachment targets the main interpreter. Do not accept a // subinterpreter-owned callable and later attach to the wrong one. if unsafe { pyo3::ffi::PyInterpreterState_Get() != pyo3::ffi::PyInterpreterState_Main() } { @@ -338,11 +358,18 @@ impl DynWinRTImplementation { let native = WinRtImplementation::new(plan, native_callback, runtime_class_name) .map_err(map_windows_error)?; interpreter.register(&callback)?; - Ok(Self { - native: Mutex::new(Some(native)), - callback: Arc::downgrade(&callback), - interpreter, - }) + ensure_python_callbacks_open()?; + let output = Py::new( + py, + Self { + native: Mutex::new(Some(native)), + callback: Arc::downgrade(&callback), + interpreter, + created_with_managed_apartment: crate::runtime::_managed_apartment_depth() > 0, + }, + )?; + track_native_owner(py, output.clone_ref(py).into_any())?; + Ok(output) } fn to_value(&self, py: Python<'_>) -> PyResult> { @@ -350,7 +377,7 @@ impl DynWinRTImplementation { native .to_value() .map_err(map_windows_error) - .and_then(|value| tracked_native_value(py, value)) + .and_then(|value| tracked_native_value_with_policy(py, value, true)) }) } @@ -471,6 +498,7 @@ pub(crate) fn init(module: &Bound<'_, PyModule>) -> PyResult<()> { #[cfg(test)] mod tests { use std::panic::{AssertUnwindSafe, catch_unwind}; + use std::sync::atomic::AtomicUsize; use std::sync::{Barrier, mpsc}; use std::time::Duration; @@ -510,6 +538,7 @@ mod tests { native: Mutex::new(Some(native)), callback: Weak::new(), interpreter: Arc::new(InterpreterState::default()), + created_with_managed_apartment: false, }, ) .unwrap(); @@ -604,6 +633,7 @@ mod tests { native: Mutex::new(Some(native)), callback: Arc::downgrade(&cell), interpreter, + created_with_managed_apartment: false, }, ) .unwrap(); @@ -654,4 +684,88 @@ mod tests { view.call_method0("release").unwrap(); }); } + + #[test] + fn real_finalization_quarantines_the_native_implementation_controller() { + if std::env::var("DYNWINRT_IMPLEMENTATION_FINALIZE_CHILD").as_deref() != Ok("1") { + let child = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "implementation::tests::real_finalization_quarantines_the_native_implementation_controller", + "--nocapture", + ]) + .env("DYNWINRT_IMPLEMENTATION_FINALIZE_CHILD", "1") + .output() + .unwrap(); + assert!( + child.status.success(), + "{}\n{}", + String::from_utf8_lossy(&child.stdout), + String::from_utf8_lossy(&child.stderr) + ); + assert!( + String::from_utf8_lossy(&child.stdout) + .contains("implementation-Py_FinalizeEx-safe") + ); + return; + } + + struct CallbackDrop(Arc); + impl Drop for CallbackDrop { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + + Python::initialize(); + let table = dynwinrt::MetadataTable::new(); + let iid = windows::Foundation::IStringable::IID; + let signature = dynwinrt::MethodSignature::new(&table).add_out(table.hstring()); + let plan = WinRtImplementationPlan::new( + vec![WinRtInterfaceDefinition { + name: "Tests.IStringableFinalizeController".into(), + interface_type: table.interface(iid), + required_iids: vec![], + methods: vec![WinRtMethodDefinition { + name: "ToString".into(), + vtable_index: 6, + signature, + }], + }], + WinRtThreadingPolicy::OwnerThread, + ) + .unwrap(); + let dropped = Arc::new(AtomicUsize::new(0)); + let sentinel = CallbackDrop(dropped.clone()); + let native = WinRtImplementation::new( + plan, + Arc::new(move |_, _, _| { + let _ = &sentinel; + Ok(vec![dynwinrt::WinRTValue::HString("alive".into())]) + }), + None, + ) + .unwrap(); + let owner = DynWinRTImplementation { + native: Mutex::new(Some(native)), + callback: Weak::new(), + interpreter: Arc::new(InterpreterState::default()), + created_with_managed_apartment: true, + }; + let cell = Arc::new(CallbackCell { + callback: Mutex::new(Some(Python::attach(|py| { + py.eval(c"lambda *_args: []", None, None).unwrap().unbind() + }))), + interpreter: Arc::new(InterpreterState::default()), + }); + + unsafe { pyo3::ffi::PyGILState_Ensure() }; + assert_eq!(unsafe { pyo3::ffi::Py_FinalizeEx() }, 0); + assert_eq!(unsafe { pyo3::ffi::Py_IsInitialized() }, 0); + assert_eq!(cell.invoke(0, 6, &[]).unwrap_err().code(), RO_E_CLOSED); + std::mem::forget(cell); + drop(owner); + assert_eq!(dropped.load(Ordering::SeqCst), 0); + println!("implementation-Py_FinalizeEx-safe"); + } } diff --git a/bindings/py/src/lib.rs b/bindings/py/src/lib.rs index 14cb9a6c..8bba4e28 100644 --- a/bindings/py/src/lib.rs +++ b/bindings/py/src/lib.rs @@ -18,7 +18,7 @@ mod dynwinrt { use pyo3::prelude::*; #[pymodule_init] - fn init(m: &Bound<'_, PyModule>) -> PyResult<()> { + pub(super) fn init(m: &Bound<'_, PyModule>) -> PyResult<()> { super::async_runtime::init_async_runtime(); m.py().run( c" @@ -35,7 +35,7 @@ from datetime import datetime as _datetime, timedelta as _timedelta, timezone as from itertools import count as _count from contextvars import ContextVar as _ContextVar, copy_context as _copy_context from operator import index as _index -from threading import current_thread as _thread_current_thread, get_ident as _thread_get_ident +from threading import current_thread as _thread_current_thread, get_ident as _thread_get_ident, local as _thread_local from types import TracebackType as _TracebackType from typing import Any as _Any, Awaitable as _Awaitable, Callable as _Callable from typing import Protocol as _Protocol, TypeVar as _TypeVar @@ -78,6 +78,37 @@ _active_projected_lifetime_scope = _ContextVar( ) _projected_wrapper_cache = _WeakValueDictionary() _projected_scope_serial = _count(1) +_apartment_owned_values = _thread_local() + +def _dynwinrt_apartment_registry(): + registry = getattr(_apartment_owned_values, 'registry', None) + if registry is None: + registry = _WeakValueDictionary() + _apartment_owned_values.registry = registry + return registry + +def _dynwinrt_release_owned_native(native): + release_owner = getattr(native, '_release_apartment_owner', None) + if release_owner is None: + native.release() + else: + release_owner() + +def _dynwinrt_drain_apartment_owners(): + registry = getattr(_apartment_owned_values, 'registry', None) + if registry is None: + return + for _ in range(1024): + if not registry: + return + for native in list(registry.values()): + check = getattr(native, '_check_apartment_release', None) + if check is not None: + check() + for key, native in reversed(list(registry.items())): + _dynwinrt_release_owned_native(native) + registry.pop(key, None) + raise RuntimeError('COM apartment cleanup did not settle; retry on the owner thread.') def _dynwinrt_projected_native_values(value): native_values = [] @@ -252,17 +283,29 @@ class ProjectedLifetimeScope: self._token = None self._active = False + for native in ( + [native for native, _ in self._registry.values()] + + list(self._native_refs.values()) + ): + check = getattr(native, '_check_apartment_release', None) + if check is not None: + try: + check() + except Exception: + self._retry_pending = True + raise + first_error = None for key, (native, _) in reversed(list(self._registry.items())): try: - native.release() + _dynwinrt_release_owned_native(native) del self._registry[key] except BaseException as error: if first_error is None: first_error = error for key, native in reversed(list(self._native_refs.items())): try: - native.release() + _dynwinrt_release_owned_native(native) del self._native_refs[key] except BaseException as error: if first_error is None: @@ -316,11 +359,18 @@ def _dynwinrt_track_projected(value, type_name=None): return value def _dynwinrt_track_native(value): + if _managed_apartment_depth() > 0: + _dynwinrt_apartment_registry()[id(value)] = value scope = _active_projected_lifetime_scope.get() if scope is not None and scope._active and not scope._disposed: scope.track_native(value) return value +def _dynwinrt_track_apartment_callback_copy(value): + if _managed_apartment_depth() > 0: + _dynwinrt_apartment_registry()[id(value)] = value + return value + def project_as(value, wrapper_type): '''Borrow a projected value and expose it as a generated runtime class. @@ -361,7 +411,7 @@ def release_projected(value): if not native_values: raise TypeError('release_projected requires a generated projected wrapper.') for native in reversed(native_values): - native.release() + _dynwinrt_release_owned_native(native) def _dynwinrt_guid(value): if isinstance(value, WinGUID): @@ -577,6 +627,17 @@ class _WinRTMutableMappingMixin(_MutableMapping): _dynwinrt_preflight_non_null_collection(self, (value for _, value in entries)) return super().update(entries) +def _dynwinrt_poll_nonagile_async(loop, future, native): + if future.done(): + return + try: + if native._async_is_started(): + loop.call_later(0.025, _dynwinrt_poll_nonagile_async, loop, future, native) + else: + future.set_result(native) + except Exception as error: + future.set_exception(error) + async def _dynwinrt_convert_future(future, converter): try: completed = _dynwinrt_track_projected(await future, 'WinRTAsync completion') @@ -603,10 +664,15 @@ def _dynwinrt_validate_throw(typ, value, traceback): if traceback is not None and not isinstance(traceback, _TracebackType): raise TypeError('throw() third argument must be a traceback object') -def _dynwinrt_link_cancellation(task, future): +def _dynwinrt_link_cancellation(task, future, native=None): def cancel_inner(completed): - if completed.cancelled() and not future.done(): - future.cancel() + if completed.cancelled(): + try: + if native is not None and not native.is_released(): + native.cancel() + finally: + if not future.done(): + future.cancel() task.add_done_callback(cancel_inner) def _dynwinrt_dispatch_progress(dispatch_state, value): @@ -654,6 +720,25 @@ _Coroutine.register(_DynWinRTAsyncWithProgress) m.add_function(wrap_pyfunction!(super::runtime::init_winappsdk, m)?)?; m.add_function(wrap_pyfunction!(super::runtime::ro_initialize, m)?)?; m.add_function(wrap_pyfunction!(super::runtime::ro_uninitialize, m)?)?; + m.add_function(wrap_pyfunction!( + super::runtime::retry_pending_apartment_close, + m + )?)?; + m.add_function(wrap_pyfunction!( + super::runtime::shutdown_python_callbacks, + m + )?)?; + m.add_function(wrap_pyfunction!( + super::runtime::_dynwinrt_close_callback_gate, + m + )?)?; + m.py() + .import("atexit")? + .call_method1("register", (m.getattr("_dynwinrt_close_callback_gate")?,))?; + m.add_function(wrap_pyfunction!( + super::runtime::_managed_apartment_depth, + m + )?)?; m.add_function(wrap_pyfunction!(super::object_value::unbox_object, m)?)?; m.add_function(wrap_pyfunction!(super::object_value::to_winrt_object, m)?)?; m.add_function(wrap_pyfunction!( @@ -693,6 +778,8 @@ for _name in ( 'DynWinRTImplementationHandle', 'ProjectedLifetimeScope', 'projected_lifetime_scope', + 'retry_pending_apartment_close', + 'shutdown_python_callbacks', 'project_as', 'release_projected', 'unbox_object', diff --git a/bindings/py/src/runtime.rs b/bindings/py/src/runtime.rs index 39302c75..21ebf6ff 100644 --- a/bindings/py/src/runtime.rs +++ b/bindings/py/src/runtime.rs @@ -1,7 +1,14 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use std::sync::{Arc, Mutex}; +use std::{ + cell::Cell, + sync::{ + Arc, Mutex, + atomic::{AtomicBool, Ordering}, + }, + thread::{self, ThreadId}, +}; use dynwinrt; use pyo3::exceptions::{PyIndexError, PyOverflowError, PyRuntimeError, PyTypeError}; @@ -25,6 +32,114 @@ static TABLE: std::sync::LazyLock> = pub(crate) static WINUI_MODULES: dynwinrt::WinUiProcessModules = dynwinrt::WinUiProcessModules::new(); +static PYTHON_SHUTTING_DOWN: AtomicBool = AtomicBool::new(false); +static CALLBACKS_IN_FLIGHT: Mutex = Mutex::new(0); +#[cfg(test)] +static CALLBACK_ATTACH_ATTEMPTS: std::sync::atomic::AtomicUsize = + std::sync::atomic::AtomicUsize::new(0); + +struct NativeCallbackPermit; + +impl Drop for NativeCallbackPermit { + fn drop(&mut self) { + match CALLBACKS_IN_FLIGHT.lock() { + Ok(mut active) if *active > 0 => *active -= 1, + _ => { + PYTHON_SHUTTING_DOWN.store(true, Ordering::Release); + log_unavailable_python_callback(); + } + } + } +} + +pub(crate) fn python_callback_available() -> bool { + !PYTHON_SHUTTING_DOWN.load(Ordering::Acquire) && unsafe { pyo3::ffi::Py_IsInitialized() != 0 } +} + +pub(crate) fn ensure_python_callbacks_open() -> PyResult<()> { + if !python_callback_available() { + return Err(PyRuntimeError::new_err( + "Python WinRT callbacks have been shut down; register them before shutdown_python_callbacks()", + )); + } + Ok(()) +} + +pub(crate) fn log_unavailable_python_callback() { + unsafe { + OutputDebugStringA(b"dynwinrt: rejecting a WinRT callback after Python shutdown\0".as_ptr()) + }; +} + +pub(crate) fn with_python_callback( + callback: impl for<'py> FnOnce(Python<'py>) -> R, +) -> Option { + if !python_callback_available() { + log_unavailable_python_callback(); + return None; + } + let permit = { + let mut active = match CALLBACKS_IN_FLIGHT.lock() { + Ok(active) => active, + Err(_) => { + log_unavailable_python_callback(); + return None; + } + }; + if !python_callback_available() { + log_unavailable_python_callback(); + return None; + } + let Some(next) = active.checked_add(1) else { + log_unavailable_python_callback(); + return None; + }; + *active = next; + NativeCallbackPermit + }; + #[cfg(test)] + CALLBACK_ATTACH_ATTEMPTS.fetch_add(1, Ordering::SeqCst); + let result = Python::try_attach(callback); + drop(permit); + match result { + Some(result) => Some(result), + None => { + log_unavailable_python_callback(); + None + } + } +} + +#[pyfunction] +pub(crate) fn _dynwinrt_close_callback_gate() { + PYTHON_SHUTTING_DOWN.store(true, Ordering::Release); +} + +fn close_native_callback_gate() -> PyResult<()> { + let active = CALLBACKS_IN_FLIGHT.try_lock().map_err(|error| { + PyRuntimeError::new_err(format!( + "cannot shut down Python WinRT callbacks while native callback bookkeeping is busy: {error}" + )) + })?; + if *active != 0 { + return Err(PyRuntimeError::new_err(format!( + "cannot shut down Python WinRT callbacks while {} callback(s) are in flight; settle them and retry", + *active + ))); + } + PYTHON_SHUTTING_DOWN.store(true, Ordering::Release); + Ok(()) +} + +#[pyfunction] +pub fn shutdown_python_callbacks(py: Python<'_>) -> PyResult<()> { + let runtime = py + .import("dynwinrt.dynwinrt")? + .getattr("_dynwinrt_implementation_runtime")?; + close_native_callback_gate()?; + runtime.call_method0("shutdown")?; + Ok(()) +} pub(crate) fn wrap_python_callback_context( py: Python<'_>, @@ -130,10 +245,108 @@ impl WinAppSDKContext { } } -#[pyclass(unsendable)] +#[pyclass] pub struct RoApartment { apartment_type: i32, active: bool, + owner_thread: Option, + cleanup_failed: bool, +} + +thread_local! { + static MANAGED_APARTMENT_DEPTH: Cell = const { Cell::new(0) }; + static MANUAL_APARTMENT_DEPTH: Cell = const { Cell::new(0) }; + static PENDING_APARTMENT_CLOSES: Cell = const { Cell::new(0) }; +} + +#[link(name = "kernel32")] +unsafe extern "system" { + fn OutputDebugStringA(message: *const u8); +} + +fn log_unsafe_apartment_teardown() { + unsafe { + OutputDebugStringA( + b"dynwinrt: retaining an apartment that cannot be safely closed on its owner thread\0" + .as_ptr(), + ) + }; +} + +pub(crate) fn log_unsafe_native_owner_drop() { + unsafe { + OutputDebugStringA( + b"dynwinrt: quarantining a native COM owner without its apartment or Python GIL\0" + .as_ptr(), + ) + }; +} + +pub(crate) fn current_native_owner_thread(owns_com: bool) -> Option { + owns_com.then(|| thread::current().id()) +} + +fn ensure_native_access_thread( + owner: Option, + release_any_thread: bool, + name: &str, +) -> PyResult<()> { + if !release_any_thread && owner.is_some_and(|thread| thread != thread::current().id()) { + return Err(PyRuntimeError::new_err(format!( + "{name} requires its owning COM apartment thread" + ))); + } + Ok(()) +} + +pub(crate) fn ensure_native_owner_thread( + owner: Option, + release_any_thread: bool, + name: &str, +) -> PyResult<()> { + if !release_any_thread && owner.is_some_and(|thread| thread != thread::current().id()) { + return Err(PyRuntimeError::new_err(format!( + "{name}.release() must run on its owning COM apartment thread" + ))); + } + Ok(()) +} + +pub(crate) fn python_gil_usable() -> bool { + (unsafe { pyo3::ffi::Py_IsInitialized() != 0 && pyo3::ffi::PyGILState_Check() != 0 }) + && Python::try_attach(|_| ()).is_some() +} + +pub(crate) fn must_quarantine_owner(owner: Option, release_any_thread: bool) -> bool { + owner.is_some() + && ((!release_any_thread && owner != Some(thread::current().id())) || !python_gil_usable()) +} + +fn managed_apartment_depth() -> usize { + MANAGED_APARTMENT_DEPTH.with(Cell::get) +} + +fn enter_managed_apartment(apartment_type: i32) -> PyResult<()> { + unsafe { RoInitialize(ro_init_type(apartment_type)) }.map_err(map_windows_error)?; + MANAGED_APARTMENT_DEPTH.with(|depth| depth.set(depth.get() + 1)); + Ok(()) +} + +fn leave_managed_apartment(py: Python<'_>) -> PyResult<()> { + let depth = managed_apartment_depth(); + if depth == 0 { + return Err(PyRuntimeError::new_err( + "no successful dynwinrt RoInitialize call remains on this thread", + )); + } + if depth == 1 { + py.import("dynwinrt.dynwinrt")? + .getattr("_dynwinrt_drain_apartment_owners")? + .call0()?; + } + MANAGED_APARTMENT_DEPTH.with(|state| state.set(depth - 1)); + unsafe { windows::Win32::System::WinRT::RoUninitialize() }; + Ok(()) } /// `apartment_type` used when Python omits it: the multithreaded apartment. @@ -163,22 +376,59 @@ impl RoApartment { "the COM apartment context is already active", )); } - unsafe { RoInitialize(ro_init_type(self.apartment_type)) }.map_err(map_windows_error)?; + enter_managed_apartment(self.apartment_type)?; self.active = true; + self.owner_thread = Some(thread::current().id()); Ok(()) } - fn uninitialize(&mut self) { - if self.active { - unsafe { windows::Win32::System::WinRT::RoUninitialize() }; - self.active = false; + fn finish(&mut self, py: Python<'_>) -> PyResult<()> { + if !self.active { + return Ok(()); + } + if self.owner_thread != Some(thread::current().id()) { + return Err(PyRuntimeError::new_err( + "RoApartment.close() must run on its initializing thread", + )); } + if let Err(error) = leave_managed_apartment(py) { + self.cleanup_failed = true; + return Err(error); + } + self.active = false; + self.cleanup_failed = false; + Ok(()) } } impl Drop for RoApartment { fn drop(&mut self) { - self.uninitialize(); + if !self.active { + return; + } + if self.owner_thread != Some(thread::current().id()) || !python_gil_usable() { + log_unsafe_apartment_teardown(); + return; + } + if self.cleanup_failed { + if PENDING_APARTMENT_CLOSES + .try_with(|pending| pending.set(pending.get() + 1)) + .is_err() + { + log_unsafe_apartment_teardown(); + } + return; + } + if Python::try_attach(|py| { + if let Err(error) = self.finish(py) { + PENDING_APARTMENT_CLOSES.with(|pending| pending.set(pending.get() + 1)); + error.write_unraisable(py, None); + } + }) + .is_none() + { + log_unsafe_apartment_teardown(); + } } } @@ -190,6 +440,8 @@ impl RoApartment { Self { apartment_type: apartment_type.unwrap_or(DEFAULT_APARTMENT_TYPE), active: false, + owner_thread: None, + cleanup_failed: false, } } @@ -203,13 +455,22 @@ impl RoApartment { _exc_type: &Bound<'_, PyAny>, _exc_value: &Bound<'_, PyAny>, _traceback: &Bound<'_, PyAny>, - ) -> bool { - self.uninitialize(); - false + ) -> PyResult { + let py = _exc_type.py(); + match self.finish(py) { + Ok(()) => Ok(false), + Err(cleanup_error) if !_exc_value.is_none() => { + py.import("dynwinrt.dynwinrt")? + .getattr("_dynwinrt_append_exception_cause")? + .call1((_exc_value, cleanup_error.value(py)))?; + Ok(false) + } + Err(cleanup_error) => Err(cleanup_error), + } } - fn close(&mut self) { - self.uninitialize(); + fn close(&mut self, py: Python<'_>) -> PyResult<()> { + self.finish(py) } fn __repr__(&self) -> String { @@ -229,14 +490,40 @@ pub fn init_winappsdk(major: u32, minor: u32) -> PyResult { #[pyfunction] pub fn ro_initialize(apartment_type: Option) -> PyResult<()> { - let init_type = ro_init_type(apartment_type.unwrap_or(DEFAULT_APARTMENT_TYPE)); - unsafe { RoInitialize(init_type) }.map_err(map_windows_error) + enter_managed_apartment(apartment_type.unwrap_or(DEFAULT_APARTMENT_TYPE))?; + MANUAL_APARTMENT_DEPTH.with(|depth| depth.set(depth.get() + 1)); + Ok(()) +} + +#[pyfunction] +pub fn ro_uninitialize(py: Python<'_>) -> PyResult<()> { + let depth = MANUAL_APARTMENT_DEPTH.with(Cell::get); + if depth == 0 { + return Err(PyRuntimeError::new_err( + "ro_uninitialize() requires a successful ro_initialize() on this thread", + )); + } + leave_managed_apartment(py)?; + MANUAL_APARTMENT_DEPTH.with(|state| state.set(depth - 1)); + Ok(()) +} + +#[pyfunction] +pub fn retry_pending_apartment_close(py: Python<'_>) -> PyResult<()> { + let pending = PENDING_APARTMENT_CLOSES.with(Cell::get); + if pending == 0 { + return Err(PyRuntimeError::new_err( + "no failed RoApartment close is pending on this thread", + )); + } + leave_managed_apartment(py)?; + PENDING_APARTMENT_CLOSES.with(|state| state.set(pending - 1)); + Ok(()) } #[pyfunction] -pub fn ro_uninitialize() { - use windows::Win32::System::WinRT::RoUninitialize; - unsafe { RoUninitialize() }; +pub(crate) fn _managed_apartment_depth() -> usize { + managed_apartment_depth() } // ====================================================================== @@ -326,6 +613,7 @@ pub fn register_xaml_runtime_class( constructor: Py, supported_overrides: Option>, ) -> PyResult { + ensure_python_callbacks_open()?; if !constructor.bind(py).is_callable() { return Err(PyRuntimeError::new_err( "register_xaml_runtime_class: constructor must be callable", @@ -350,7 +638,7 @@ pub fn register_xaml_runtime_class( 0x8001010Eu32 as i32, ))); } - Python::attach(|py| { + with_python_callback(|py| { let result = (|| -> PyResult { let invocation_context = context.call_method0(py, "copy")?; let instance = invocation_context.call_method1(py, "run", (callback.bind(py),))?; @@ -387,6 +675,11 @@ pub fn register_xaml_runtime_class( } } }) + .unwrap_or_else(|| { + Err(windows::core::Error::from_hresult( + PYWINRT_E_INTERPRETER_CLOSED, + )) + }) }); let registration = dynwinrt::register_xaml_runtime_class( &runtime_class_name, @@ -726,7 +1019,7 @@ impl DynWinRTOverrideInterface { if std::thread::current().id() != thread_id { return windows::core::HRESULT(0x8001010Eu32 as i32); } - Python::attach(|py| { + with_python_callback(|py| { let result = (|| -> PyResult<()> { let invocation_context = context.call_method0(py, "copy")?; invocation_context.call_method1(py, "run", (callback.bind(py),))?; @@ -740,6 +1033,7 @@ impl DynWinRTOverrideInterface { } } }) + .unwrap_or(PYWINRT_E_INTERPRETER_CLOSED) }); interface = interface .with_void_callback(*vtable_index, callback) @@ -754,7 +1048,7 @@ impl DynWinRTOverrideInterface { if std::thread::current().id() != thread_id { return windows::core::HRESULT(0x8001010Eu32 as i32); } - Python::attach(|py| { + with_python_callback(|py| { let result = (|| -> PyResult<(f32, f32)> { let invocation_context = context.call_method0(py, "copy")?; let result = invocation_context.call_method1( @@ -783,6 +1077,7 @@ impl DynWinRTOverrideInterface { } } }) + .unwrap_or(PYWINRT_E_INTERPRETER_CLOSED) }, ); interface = interface @@ -807,6 +1102,7 @@ impl DynWinRTOverrideInterface { abi_shapes: Vec, callbacks: &Bound<'_, PyDict>, ) -> PyResult { + ensure_python_callbacks_open()?; let methods = abi_shapes .iter() .enumerate() @@ -869,20 +1165,35 @@ impl DynWinRTOverrideInterface { } } +fn pin_method_receiver( + py: Python<'_>, + obj: &Py, + operation: &str, + accepts_async: bool, +) -> PyResult { + let obj = obj.try_borrow(py)?; + if accepts_async { + obj.com_receiver(operation) + } else { + obj.receiver(operation).cloned() + } +} + #[pymethods] impl DynWinRTMethodHandle { /// Invoke this method on a COM object. fn invoke( &self, py: Python<'_>, - obj: DynWinRTValue, - args: Vec, + obj: Py, + args: Vec>, ) -> PyResult> { - // Extraction retains the native object without holding a Python borrow - // while an implementation callback may release the original wrapper. - let raw = obj.receiver("invoke()")?.as_raw(); - let wrt_args = native_arguments("invoke()", args)?; - let results = self.0.invoke(raw, &wrt_args).map_err(map_dynwinrt_error)?; + let object = pin_method_receiver(py, &obj, "invoke()", false)?; + let wrt_args = native_arguments(py, "invoke()", args)?; + let results = self + .0 + .invoke(object.as_raw(), &wrt_args) + .map_err(map_dynwinrt_error)?; let value = results .into_iter() .next() @@ -895,8 +1206,8 @@ impl DynWinRTMethodHandle { fn invoke_detached( &self, py: Python<'_>, - obj: DynWinRTValue, - args: Vec, + obj: Py, + args: Vec>, ) -> PyResult> { struct SameThreadCall { method: dynwinrt::MethodHandle, @@ -918,14 +1229,13 @@ impl DynWinRTMethodHandle { unsafe impl Send for SameThreadCall {} unsafe impl Send for SameThreadResult {} - // Owned extraction ends the Python receiver borrow before dispatch. - // Move its native pin into the call so reentrant disposal can release - // the original wrapper without shortening the in-flight call lifetime. - let object = obj.into_receiver("invoke_detached()")?; + // Validate a borrowed Python handle before cloning the native pin. + // Only the pin and validated arguments cross the detached GIL boundary. + let object = pin_method_receiver(py, &obj, "invoke_detached()", false)?; let call = SameThreadCall { method: self.0.clone(), object, - args: native_arguments("invoke_detached()", args)?, + args: native_arguments(py, "invoke_detached()", args)?, }; let results = py .detach(move || call.run()) @@ -943,12 +1253,15 @@ impl DynWinRTMethodHandle { fn invoke_all( &self, py: Python<'_>, - obj: DynWinRTValue, - args: Vec, + obj: Py, + args: Vec>, ) -> PyResult>> { - let raw = obj.receiver("invoke_all()")?.as_raw(); - let wrt_args = native_arguments("invoke_all()", args)?; - let results = self.0.invoke(raw, &wrt_args).map_err(map_dynwinrt_error)?; + let object = pin_method_receiver(py, &obj, "invoke_all()", false)?; + let wrt_args = native_arguments(py, "invoke_all()", args)?; + let results = self + .0 + .invoke(object.as_raw(), &wrt_args) + .map_err(map_dynwinrt_error)?; results .into_iter() .map(|value| tracked_native_value(py, value)) @@ -960,14 +1273,14 @@ impl DynWinRTMethodHandle { &self, py: Python<'_>, factory: &DynWinRTValue, - args: Vec, + args: Vec>, outer_index: usize, inner_output_index: usize, instance_output_index: usize, agile: bool, ) -> PyResult> { let factory = factory.com_receiver("invoke_composed() factory")?; - let args = native_arguments("invoke_composed()", args)?; + let args = native_arguments(py, "invoke_composed()", args)?; dynwinrt::compose_winrt( &factory, &self.0, @@ -987,7 +1300,7 @@ impl DynWinRTMethodHandle { &self, py: Python<'_>, factory: &DynWinRTValue, - args: Vec, + args: Vec>, outer_index: usize, inner_output_index: usize, instance_output_index: usize, @@ -1006,7 +1319,7 @@ impl DynWinRTMethodHandle { ); } let factory = factory.com_receiver("invoke_composed_with_overrides() factory")?; - let args = native_arguments("invoke_composed_with_overrides()", args)?; + let args = native_arguments(py, "invoke_composed_with_overrides()", args)?; let overrides = override_interfaces .iter() .map(|interface| interface.to_core(py)) @@ -1028,32 +1341,36 @@ impl DynWinRTMethodHandle { // --- Fast paths: skip Vec alloc for common getter patterns --- /// Getter → string (0 args, zero Vec allocation) - fn get_string(&self, obj: DynWinRTValue) -> PyResult { - let raw = obj.com_receiver("get_string()")?.as_raw(); + fn get_string(&self, py: Python<'_>, obj: Py) -> PyResult { + let object = pin_method_receiver(py, &obj, "get_string()", true)?; let hs = self .0 - .call_getter_hstring(raw) + .call_getter_hstring(object.as_raw()) .map_err(map_dynwinrt_error)?; Ok(hs.to_string()) } /// Getter → i32 (0 args, zero Vec allocation) - fn get_i32(&self, obj: DynWinRTValue) -> PyResult { - let raw = obj.com_receiver("get_i32()")?.as_raw(); - self.0.call_getter_i32(raw).map_err(map_dynwinrt_error) + fn get_i32(&self, py: Python<'_>, obj: Py) -> PyResult { + let object = pin_method_receiver(py, &obj, "get_i32()", true)?; + self.0 + .call_getter_i32(object.as_raw()) + .map_err(map_dynwinrt_error) } /// Getter → bool (0 args, zero Vec allocation) - fn get_bool(&self, obj: DynWinRTValue) -> PyResult { - let raw = obj.com_receiver("get_bool()")?.as_raw(); - self.0.call_getter_bool(raw).map_err(map_dynwinrt_error) + fn get_bool(&self, py: Python<'_>, obj: Py) -> PyResult { + let object = pin_method_receiver(py, &obj, "get_bool()", true)?; + self.0 + .call_getter_bool(object.as_raw()) + .map_err(map_dynwinrt_error) } /// Getter → DynWinRTValue object (0 args, zero Vec allocation) - fn get_obj(&self, py: Python<'_>, obj: DynWinRTValue) -> PyResult> { - let raw = obj.com_receiver("get_obj()")?.as_raw(); + fn get_obj(&self, py: Python<'_>, obj: Py) -> PyResult> { + let object = pin_method_receiver(py, &obj, "get_obj()", true)?; self.0 - .call_getter_object(raw) + .call_getter_object(object.as_raw()) .map_err(map_dynwinrt_error) .and_then(|value| tracked_native_value(py, value)) } @@ -1062,13 +1379,16 @@ impl DynWinRTMethodHandle { fn invoke_hstring( &self, py: Python<'_>, - obj: DynWinRTValue, + obj: Py, arg: String, ) -> PyResult> { - let raw = obj.com_receiver("invoke_hstring()")?.as_raw(); + let object = pin_method_receiver(py, &obj, "invoke_hstring()", true)?; let results = self .0 - .invoke(raw, &[dynwinrt::WinRTValue::HString(HSTRING::from(arg))]) + .invoke( + object.as_raw(), + &[dynwinrt::WinRTValue::HString(HSTRING::from(arg))], + ) .map_err(map_dynwinrt_error)?; tracked_native_value( py, @@ -1083,13 +1403,13 @@ impl DynWinRTMethodHandle { fn invoke_i32( &self, py: Python<'_>, - obj: DynWinRTValue, + obj: Py, arg: i32, ) -> PyResult> { - let raw = obj.com_receiver("invoke_i32()")?.as_raw(); + let object = pin_method_receiver(py, &obj, "invoke_i32()", true)?; let results = self .0 - .invoke(raw, &[dynwinrt::WinRTValue::I32(arg)]) + .invoke(object.as_raw(), &[dynwinrt::WinRTValue::I32(arg)]) .map_err(map_dynwinrt_error)?; tracked_native_value( py, @@ -1105,20 +1425,46 @@ impl DynWinRTMethodHandle { // DynWinRTValue — main value container // ====================================================================== -#[pyclass(from_py_object, weakref)] +#[pyclass(weakref, skip_from_py_object)] #[derive(Clone)] -pub struct DynWinRTValue(pub(crate) dynwinrt::WinRTValue, Lifecycle); +pub struct DynWinRTValue( + pub(crate) dynwinrt::WinRTValue, + Lifecycle, + Option, + bool, +); + +impl Drop for DynWinRTValue { + fn drop(&mut self) { + if matches!(self.1, Lifecycle::Live) + && self.0.contains_com_references() + && must_quarantine_owner(self.2, self.3) + { + std::mem::forget(std::mem::replace(&mut self.0, dynwinrt::WinRTValue::Null)); + log_unsafe_native_owner_drop(); + } + } +} static TRACK_NATIVE: PyOnceLock> = PyOnceLock::new(); +static TRACK_APARTMENT_CALLBACK_COPY: PyOnceLock> = PyOnceLock::new(); pub(crate) fn init_native_tracking(module: &Bound<'_, PyModule>) -> PyResult<()> { + PYTHON_SHUTTING_DOWN.store(false, Ordering::Release); TRACK_NATIVE.get_or_try_init(module.py(), || { Ok::, PyErr>(module.getattr("_dynwinrt_track_native")?.unbind()) })?; + TRACK_APARTMENT_CALLBACK_COPY.get_or_try_init(module.py(), || { + Ok::, PyErr>( + module + .getattr("_dynwinrt_track_apartment_callback_copy")? + .unbind(), + ) + })?; Ok(()) } -fn track_native_owner(py: Python<'_>, owner: Py) -> PyResult<()> { +pub(crate) fn track_native_owner(py: Python<'_>, owner: Py) -> PyResult<()> { if let Some(track) = TRACK_NATIVE.get(py) { track.call1(py, (owner,))?; } @@ -1130,18 +1476,104 @@ fn track_native_owner(py: Python<'_>, owner: Py) -> PyResult<()> { pub(crate) fn tracked_native_value( py: Python<'_>, value: dynwinrt::WinRTValue, +) -> PyResult> { + tracked_native_value_with_policy(py, value, false) +} + +pub(crate) fn tracked_native_value_with_policy( + py: Python<'_>, + value: dynwinrt::WinRTValue, + release_any_thread: bool, ) -> PyResult> { let owns_native = value.contains_com_references(); - let output = Py::new(py, DynWinRTValue::new(value))?; + let agile = release_any_thread || (owns_native && native_value_is_agile(&value)?); + let output = Py::new(py, DynWinRTValue::new_managed(value, agile))?; if owns_native { track_native_owner(py, output.clone_ref(py).into_any())?; } Ok(output) } +fn native_array_is_agile(array: &dynwinrt::ArrayData) -> PyResult { + for index in 0..array.len() { + if !native_value_is_agile(&array.try_get(index).map_err(map_dynwinrt_error)?)? { + return Ok(false); + } + } + Ok(true) +} + +fn native_struct_is_agile(data: &dynwinrt::ValueTypeData) -> PyResult { + for index in 0..data.type_handle().field_count() { + let kind = data.field_kind_checked(index).map_err(map_dynwinrt_error)?; + if kind.is_com_pointer() { + if let Some(object) = data.get_field_object(index).map_err(map_dynwinrt_error)? + && object.cast::().is_err() + { + return Ok(false); + } + } else if matches!(kind, dynwinrt::TypeKind::Struct(_)) { + let nested = data + .get_field_struct_checked(index) + .map_err(map_dynwinrt_error)?; + if !native_struct_is_agile(&nested)? { + return Ok(false); + } + } else if data + .type_handle() + .field_type(index) + .contains_com_references() + { + return Ok(false); + } + } + Ok(true) +} + +pub(crate) fn native_value_is_agile(value: &dynwinrt::WinRTValue) -> PyResult { + match value { + dynwinrt::WinRTValue::Object(object) => { + Ok(object.cast::().is_ok()) + } + dynwinrt::WinRTValue::Async(info) => { + Ok(info.info.cast::().is_ok()) + } + dynwinrt::WinRTValue::ArrayOfIUnknown(values) => Ok((0..values.0.len()).all(|index| { + values.0[index] + .as_ref() + .is_none_or(|object| object.cast::().is_ok()) + })), + dynwinrt::WinRTValue::Array(array) => native_array_is_agile(array), + dynwinrt::WinRTValue::Struct(data) => native_struct_is_agile(data), + _ => Ok(true), + } +} + +pub(crate) fn callback_native_argument( + py: Python<'_>, + value: dynwinrt::WinRTValue, +) -> PyResult> { + if value.contains_com_references() + && managed_apartment_depth() > 0 + && !native_value_is_agile(&value)? + { + let output = Py::new(py, DynWinRTValue::new_managed(value, false))?; + let track = TRACK_APARTMENT_CALLBACK_COPY.get(py).ok_or_else(|| { + PyRuntimeError::new_err("native callback lifetime tracker is missing") + })?; + track.call1(py, (output.clone_ref(py).into_any(),))?; + return Ok(output); + } + Py::new(py, DynWinRTValue::new(value)) +} + fn tracked_native_array(py: Python<'_>, array: dynwinrt::ArrayData) -> PyResult> { let owns_com = array.contains_com_references(); - let output = Py::new(py, DynWinRTArray(Some(array)))?; + let agile = owns_com && native_array_is_agile(&array)?; + let output = Py::new( + py, + DynWinRTArray(Some(array), current_native_owner_thread(owns_com), agile), + )?; if owns_com { track_native_owner(py, output.clone_ref(py).into_any())?; } @@ -1153,7 +1585,11 @@ fn tracked_native_struct( data: dynwinrt::ValueTypeData, ) -> PyResult> { let owns_com = data.type_handle().contains_com_references(); - let output = Py::new(py, DynWinRTStruct(Some(data)))?; + let agile = owns_com && native_struct_is_agile(&data)?; + let output = Py::new( + py, + DynWinRTStruct(Some(data), current_native_owner_thread(owns_com), agile), + )?; if owns_com { track_native_owner(py, output.clone_ref(py).into_any())?; } @@ -1171,28 +1607,27 @@ enum Lifecycle { impl DynWinRTValue { pub(crate) fn new(value: dynwinrt::WinRTValue) -> Self { - Self(value, Lifecycle::Live) + Self(value, Lifecycle::Live, None, false) + } + + fn new_managed(value: dynwinrt::WinRTValue, release_any_thread: bool) -> Self { + let owner = current_native_owner_thread(value.contains_com_references()); + Self(value, Lifecycle::Live, owner, release_any_thread) } /// The WinRT object receiving `operation`. fn receiver(&self, operation: &str) -> PyResult<&IUnknown> { + self.ensure_live()?; match &self.0 { dynwinrt::WinRTValue::Object(object) => Ok(object), _ => Err(self.receiver_error(operation)), } } - /// Like `receiver`, but moves the object out of this value. - fn into_receiver(self, operation: &str) -> PyResult { - match self.0 { - dynwinrt::WinRTValue::Object(object) => Ok(object), - _ => Err(self.receiver_error(operation)), - } - } - /// The COM identity receiving `operation`. Unlike `receiver`, this also /// accepts async operations, as the legacy convenience entry points do. fn com_receiver(&self, operation: &str) -> PyResult { + self.ensure_live()?; self.0 .as_object() .ok_or_else(|| self.receiver_error(operation)) @@ -1200,6 +1635,7 @@ impl DynWinRTValue { /// QueryInterface this value for `operation`. pub(crate) fn query(&self, iid: &GUID, operation: &str) -> PyResult { + self.ensure_live()?; self.0.cast(iid).map_err(|error| match error { dynwinrt::Error::ExpectObjectTypeError(_) => self.receiver_error(operation), error => map_dynwinrt_error(error), @@ -1218,7 +1654,7 @@ impl DynWinRTValue { /// payload errors, such as IBuffer access. fn ensure_live(&self) -> PyResult<()> { match self.1 { - Lifecycle::Live => Ok(()), + Lifecycle::Live => ensure_native_access_thread(self.2, self.3, "DynWinRTValue"), Lifecycle::Released => Err(released_receiver_error()), } } @@ -1247,7 +1683,7 @@ impl DynWinRTValue { /// Reject this value if released; `slot` names where `operation` received it. pub(crate) fn check_input(&self, operation: &str, slot: InputSlot) -> PyResult<()> { match self.1 { - Lifecycle::Live => Ok(()), + Lifecycle::Live => ensure_native_access_thread(self.2, self.3, "DynWinRTValue"), Lifecycle::Released => Err(released_input_error(operation, slot)), } } @@ -1256,35 +1692,39 @@ impl DynWinRTValue { /// The native values `operation` received, rejecting released values. `slot` /// maps each position to where it was passed, such as an argument or element. fn native_inputs( + py: Python<'_>, operation: &str, - values: Vec, + values: Vec>, slot: fn(usize) -> InputSlot, ) -> PyResult> { values .into_iter() .enumerate() .map(|(index, value)| { + let value = value.try_borrow(py)?; value.check_input(operation, slot(index))?; - Ok(value.0) + Ok(value.0.clone()) }) .collect() } /// The native arguments of `operation`, rejecting released values. pub(crate) fn native_arguments( + py: Python<'_>, operation: &str, - args: Vec, + args: Vec>, ) -> PyResult> { - native_inputs(operation, args, InputSlot::Argument) + native_inputs(py, operation, args, InputSlot::Argument) } /// The native values a Python `operation` callback returned, rejecting /// released values instead of returning them as WinRT null. pub(crate) fn native_outputs( + py: Python<'_>, operation: &str, - outputs: Vec, + outputs: Vec>, ) -> PyResult> { - native_inputs(operation, outputs, InputSlot::Output) + native_inputs(py, operation, outputs, InputSlot::Output) } fn value_kind(value: &dynwinrt::WinRTValue) -> &'static str { @@ -1354,11 +1794,13 @@ impl DynWinRTValue { metadata_provider: &DynWinRTValue, launched_callback: Option<&DynWinRTValue>, ) -> PyResult> { + metadata_provider.check_input("create_xaml_application()", InputSlot::Argument(0))?; let provider = metadata_provider.0.as_object().ok_or_else(|| { PyRuntimeError::new_err("create_xaml_application: metadata_provider must be an Object") })?; let callback = launched_callback .map(|value| { + value.check_input("create_xaml_application()", InputSlot::Argument(1))?; value.0.as_object().ok_or_else(|| { PyRuntimeError::new_err( "create_xaml_application: launched_callback must be an Object", @@ -1484,10 +1926,15 @@ impl DynWinRTValue { #[staticmethod] fn create_vector( py: Python<'_>, - items: Vec, + items: Vec>, element_type: &DynWinRTType, ) -> PyResult> { - let wrt_items = native_inputs("DynWinRTValue.create_vector()", items, InputSlot::Element)?; + let wrt_items = native_inputs( + py, + "DynWinRTValue.create_vector()", + items, + InputSlot::Element, + )?; let iids = TABLE.vector_iids(&element_type.0); let vector = dynwinrt::vector::create_vector_from_values(&wrt_items, &element_type.0, iids) .map_err(map_dynwinrt_error)?; @@ -1498,8 +1945,8 @@ impl DynWinRTValue { #[staticmethod] fn create_map( py: Python<'_>, - keys: Vec, - values: Vec, + keys: Vec>, + values: Vec>, key_type: &DynWinRTType, value_type: &DynWinRTType, ) -> PyResult> { @@ -1509,8 +1956,8 @@ impl DynWinRTValue { )); } const OPERATION: &str = "DynWinRTValue.create_map()"; - let keys = native_inputs(OPERATION, keys, InputSlot::Key)?; - let values = native_inputs(OPERATION, values, InputSlot::Value)?; + let keys = native_inputs(py, OPERATION, keys, InputSlot::Key)?; + let values = native_inputs(py, OPERATION, values, InputSlot::Value)?; let iids = TABLE.map_iids(&key_type.0, &value_type.0); let entries: Vec<(dynwinrt::WinRTValue, dynwinrt::WinRTValue)> = keys.into_iter().zip(values).collect(); @@ -1531,6 +1978,31 @@ impl DynWinRTValue { .and_then(|value| tracked_native_value(py, value)) } + fn _async_is_started(&self) -> PyResult { + self.ensure_live()?; + match &self.0 { + dynwinrt::WinRTValue::Async(info) => info.is_started().map_err(map_dynwinrt_error), + _ => Err(PyRuntimeError::new_err( + "value is not a WinRT async operation", + )), + } + } + + fn _check_apartment_release(&self) -> PyResult<()> { + if let dynwinrt::WinRTValue::Async(info) = &self.0 + && info.is_started().map_err(map_dynwinrt_error)? + && info + .info + .cast::() + .is_err() + { + return Err(PyRuntimeError::new_err( + "cannot close the COM apartment while a non-agile WinRT async reference is pending; settle it and retry on its owner thread", + )); + } + Ok(()) + } + /// Cancel the underlying WinRT async operation (calls `IAsyncInfo::Cancel`). /// Safe to call multiple times or on already-completed operations. /// @@ -1547,6 +2019,7 @@ impl DynWinRTValue { /// Register a progress callback on an async-with-progress operation. fn on_progress(&self, py: Python<'_>, callback: Py) -> PyResult<()> { + ensure_python_callbacks_open()?; let async_info = match &self.0 { dynwinrt::WinRTValue::Async(a) => a, _ => return Err(PyRuntimeError::new_err("on_progress: not an async value")), @@ -1566,9 +2039,9 @@ impl DynWinRTValue { let callback = wrap_python_callback_context(py, callback)?; let progress_cb: dynwinrt::ProgressCallback = Box::new(move |val: dynwinrt::WinRTValue| { - Python::attach(|py| { + let _ = with_python_callback(|py| { let result = (|| -> PyResult<()> { - let py_val = Py::new(py, DynWinRTValue::new(val))?; + let py_val = callback_native_argument(py, val)?; callback.call1(py, (py_val,))?; Ok(()) })(); @@ -1740,23 +2213,32 @@ impl DynWinRTValue { /// DynWinRTArray or a raw DynWinRTValue. fn _validate_non_null_collection_input( &self, - value: DynWinRTValue, + py: Python<'_>, + value: Py, iid: &WinGUID, name: &str, - ) -> PyResult { - value.check_input("collection input", InputSlot::Argument(0))?; - let contains_null = match &value.0 { - dynwinrt::WinRTValue::Array(data) => { - (0..data.len()).any(|index| data.get(index).is_null_object()) + ) -> PyResult> { + let contains_null = { + let value = value.try_borrow(py)?; + value.check_input("collection input", InputSlot::Argument(0))?; + match &value.0 { + dynwinrt::WinRTValue::Array(data) => { + (0..data.len()).any(|index| data.get(index).is_null_object()) + } + other => other.is_null_object(), } - other => other.is_null_object(), }; if contains_null && self.matches_runtime_class(&iid.0, name)? { return Err(PyTypeError::new_err(format!( "{name} requires a non-null IJsonValue; use JsonValue.create_null_value() for JSON null" ))); } - Ok(value) + let (native, release_any_thread) = { + let value = value.try_borrow(py)?; + value.check_input("collection input", InputSlot::Argument(0))?; + (value.0.clone(), value.3) + }; + tracked_native_value_with_policy(py, native, release_any_thread) } /// Guard-only QueryInterface probe; never treats a native failure as a non-match. @@ -1795,10 +2277,15 @@ impl DynWinRTValue { /// /// This is idempotent so projected lifetime scopes can safely retry /// cleanup without double-releasing COM references. - fn release(&mut self) { + fn release(&mut self) -> PyResult<()> { + if matches!(self.1, Lifecycle::Released) { + return Ok(()); + } + ensure_native_owner_thread(self.2, self.3, "DynWinRTValue")?; let value = std::mem::replace(&mut self.0, dynwinrt::WinRTValue::Null); self.1 = Lifecycle::Released; drop(value); + Ok(()) } fn as_raw(&self) -> PyResult { @@ -1814,7 +2301,7 @@ impl DynWinRTValue { /// COM QueryInterface — cast to a different interface. fn cast(&self, py: Python<'_>, iid: &WinGUID) -> PyResult> { - tracked_native_value(py, self.query(&iid.0, "cast()")?) + tracked_native_value_with_policy(py, self.query(&iid.0, "cast()")?, self.3) } /// Invoke metadata-described Invoke on an IUnknown-rooted WinRT delegate. @@ -1822,7 +2309,7 @@ impl DynWinRTValue { slf: &Bound<'_, Self>, iid: &WinGUID, signature: &DynWinRTMethodSig, - args: Vec, + args: Vec>, ) -> PyResult>> { crate::delegate_method::DynWinRTDelegateMethod::create(iid, signature)?.invoke(slf, args) } @@ -1897,7 +2384,7 @@ impl DynWinRTValue { method_index: usize, return_type: &DynWinRTType, in_types: Vec, - args: Vec, + args: Vec>, ) -> PyResult> { let mut method = dynwinrt::MethodSignature::new(&*TABLE); for t in &in_types { @@ -1906,7 +2393,7 @@ impl DynWinRTValue { method = method.add_out(return_type.0.clone()); let obj = self.receiver("call()")?.as_raw(); - let winrt_args = native_arguments("call()", args)?; + let winrt_args = native_arguments(py, "call()", args)?; let mut iface = dynwinrt::InterfaceSignature::define_from_iinspectable("", Default::default(), &*TABLE); @@ -1958,27 +2445,52 @@ impl DynWinRTValue { // DynWinRTArray — array container with blittable fast paths // ====================================================================== -#[pyclass(unsendable, from_py_object, weakref)] -#[derive(Clone)] -pub struct DynWinRTArray(Option); +#[pyclass(weakref)] +pub struct DynWinRTArray(Option, Option, bool); + +// PyO3 enforces exclusive mutable borrows even on free-threaded Python. +// Shared borrows only read the owned buffer, and off-thread COM reads require +// every contained reference to have passed IAgileObject QI. +unsafe impl Send for DynWinRTArray {} +unsafe impl Sync for DynWinRTArray {} + +impl Drop for DynWinRTArray { + fn drop(&mut self) { + if self + .0 + .as_ref() + .is_some_and(|data| data.contains_com_references()) + && must_quarantine_owner(self.1, self.2) + { + std::mem::forget(self.0.take()); + log_unsafe_native_owner_drop(); + } + } +} impl DynWinRTArray { fn data(&self) -> PyResult<&dynwinrt::ArrayData> { + ensure_native_access_thread(self.1, self.2, "DynWinRTArray")?; self.0 .as_ref() .ok_or_else(|| released_native_container_error("DynWinRTArray")) } fn scalar_array(typ: dynwinrt::TypeHandle, values: &[dynwinrt::WinRTValue]) -> Self { - Self(Some(dynwinrt::ArrayData::from_values(typ, values))) + Self( + Some(dynwinrt::ArrayData::from_values(typ, values)), + None, + false, + ) } fn from_elements( + py: Python<'_>, operation: &str, - values: Vec, + values: Vec>, element_type: &DynWinRTType, ) -> PyResult { - let values = native_inputs(operation, values, InputSlot::Element)?; + let values = native_inputs(py, operation, values, InputSlot::Element)?; dynwinrt::ArrayData::try_from_values(element_type.0.clone(), &values) .map_err(map_windows_error) } @@ -2191,12 +2703,12 @@ impl DynWinRTArray { #[staticmethod] fn from_values( py: Python<'_>, - values: Vec, + values: Vec>, element_type: &DynWinRTType, ) -> PyResult> { tracked_native_array( py, - Self::from_elements("DynWinRTArray.from_values()", values, element_type)?, + Self::from_elements(py, "DynWinRTArray.from_values()", values, element_type)?, ) } @@ -2209,12 +2721,17 @@ impl DynWinRTArray { #[staticmethod] fn from_object_values( py: Python<'_>, - values: Vec, + values: Vec>, element_type: &DynWinRTType, ) -> PyResult> { tracked_native_array( py, - Self::from_elements("DynWinRTArray.from_object_values()", values, element_type)?, + Self::from_elements( + py, + "DynWinRTArray.from_object_values()", + values, + element_type, + )?, ) } @@ -2261,8 +2778,10 @@ impl DynWinRTArray { self.0.is_none() } - fn release(&mut self) { + fn release(&mut self) -> PyResult<()> { + ensure_native_owner_thread(self.1, self.2, "DynWinRTArray")?; drop(self.0.take()); + Ok(()) } fn __repr__(&self) -> String { @@ -2277,18 +2796,39 @@ impl DynWinRTArray { // DynWinRTStruct — typed field access by index // ====================================================================== -#[pyclass(unsendable, from_py_object, weakref)] -#[derive(Clone)] -pub struct DynWinRTStruct(Option); +#[pyclass(weakref)] +pub struct DynWinRTStruct(Option, Option, bool); + +// PyO3 serializes mutable borrows of the owned struct allocation. Shared +// reads cannot race setters, and non-agile COM field access stays on its +// creating thread; a foreign Drop quarantines such fields without Release. +unsafe impl Send for DynWinRTStruct {} +unsafe impl Sync for DynWinRTStruct {} + +impl Drop for DynWinRTStruct { + fn drop(&mut self) { + if self + .0 + .as_ref() + .is_some_and(|data| data.type_handle().contains_com_references()) + && must_quarantine_owner(self.1, self.2) + { + std::mem::forget(self.0.take()); + log_unsafe_native_owner_drop(); + } + } +} impl DynWinRTStruct { fn data(&self) -> PyResult<&dynwinrt::ValueTypeData> { + ensure_native_access_thread(self.1, self.2, "DynWinRTStruct")?; self.0 .as_ref() .ok_or_else(|| released_native_container_error("DynWinRTStruct")) } fn data_mut(&mut self) -> PyResult<&mut dynwinrt::ValueTypeData> { + ensure_native_access_thread(self.1, self.2, "DynWinRTStruct")?; self.0 .as_mut() .ok_or_else(|| released_native_container_error("DynWinRTStruct")) @@ -2507,9 +3047,21 @@ impl DynWinRTStruct { fn set_struct(&mut self, index: i64, value: &DynWinRTStruct) -> PyResult<()> { let index = checked_index(index)?; + self.data()?; + let nested = value.data()?; + if self.1.is_some_and(|owner| owner != thread::current().id()) + && !native_struct_is_agile(nested)? + { + return Err(PyRuntimeError::new_err( + "cannot store non-agile COM fields in a struct from another apartment thread", + )); + } + self.2 = false; self.data_mut()? - .set_field_struct_checked(index, value.data()?) - .map_err(map_dynwinrt_error) + .set_field_struct_checked(index, nested) + .map_err(map_dynwinrt_error)?; + self.2 = native_struct_is_agile(self.data()?)?; + Ok(()) } fn get_object(&self, py: Python<'_>, index: i64) -> PyResult> { @@ -2527,19 +3079,31 @@ impl DynWinRTStruct { fn set_object(&mut self, index: i64, value: &DynWinRTValue) -> PyResult<()> { let index = checked_index(index)?; - let data = self.data_mut()?; + self.data()?; value.check_input("DynWinRTStruct.set_object()", InputSlot::Field(index))?; - match &value.0 { - dynwinrt::WinRTValue::Object(obj) => data - .set_field_object(index, Some(obj)) - .map_err(map_dynwinrt_error), - dynwinrt::WinRTValue::Null => data - .set_field_object(index, None) - .map_err(map_dynwinrt_error), - _ => Err(PyTypeError::new_err( - "set_object requires a WinRT object or null value", - )), + let object = match &value.0 { + dynwinrt::WinRTValue::Object(obj) => Some(obj), + dynwinrt::WinRTValue::Null => None, + _ => { + return Err(PyTypeError::new_err( + "set_object requires a WinRT object or null value", + )); + } + }; + if self.1.is_some_and(|owner| owner != thread::current().id()) + && object.is_some() + && !native_value_is_agile(&value.0)? + { + return Err(PyRuntimeError::new_err( + "cannot store a non-agile COM field from another apartment thread", + )); } + self.2 = false; + self.data_mut()? + .set_field_object(index, object) + .map_err(map_dynwinrt_error)?; + self.2 = native_struct_is_agile(self.data()?)?; + Ok(()) } /// Wrap as DynWinRTValue::Struct for passing to call(). @@ -2551,8 +3115,10 @@ impl DynWinRTStruct { self.0.is_none() } - fn release(&mut self) { + fn release(&mut self) -> PyResult<()> { + ensure_native_owner_thread(self.1, self.2, "DynWinRTStruct")?; drop(self.0.take()); + Ok(()) } fn __repr__(&self) -> String { @@ -2568,11 +3134,22 @@ impl DynWinRTStruct { // DynWinRtDelegate — dynamic WinRT delegate (callback) binding // ====================================================================== -#[pyclass] -pub struct DynWinRtDelegate(dynwinrt::WinRTValue); +#[pyclass(weakref)] +pub struct DynWinRtDelegate(Option, Option); + +impl Drop for DynWinRtDelegate { + fn drop(&mut self) { + if self.0.is_some() && must_quarantine_owner(self.1, true) { + std::mem::forget(self.0.take()); + log_unsafe_native_owner_drop(); + } + } +} pub(crate) const PYWINRT_E_UNRAISABLE_PYTHON_EXCEPTION: windows::core::HRESULT = windows::core::HRESULT(0xA0EE4005_u32 as i32); +pub(crate) const PYWINRT_E_INTERPRETER_CLOSED: windows::core::HRESULT = + windows::core::HRESULT(0x80000013_u32 as i32); fn create_python_delegate( iid: GUID, @@ -2581,16 +3158,11 @@ fn create_python_delegate( ) -> PyResult { let delegate_callback: dynwinrt::delegate::DelegateCallback = Box::new(move |args: &[dynwinrt::WinRTValue]| { - Python::attach(|py| { + with_python_callback(|py| { let result = (|| -> PyResult<()> { let py_args = args .iter() - .map(|arg| { - Ok(DynWinRTValue::new(arg.clone()) - .into_pyobject(py)? - .into_any() - .unbind()) - }) + .map(|arg| Ok(callback_native_argument(py, arg.clone())?.into_any())) .collect::>>>()?; let py_tuple = pyo3::types::PyTuple::new(py, &py_args)?; callback.call1(py, py_tuple)?; @@ -2604,6 +3176,7 @@ fn create_python_delegate( } } }) + .unwrap_or(PYWINRT_E_INTERPRETER_CLOSED) }); dynwinrt::delegate::try_create_delegate_value(iid, type_handles, delegate_callback) .map_err(|error| map_dynwinrt_error_with_context(error, "DynWinRtDelegate.create failed")) @@ -2618,19 +3191,41 @@ impl DynWinRtDelegate { /// - `callback`: Python callable invoked when WinRT fires the event #[staticmethod] fn create( + py: Python<'_>, iid: &WinGUID, param_types: Vec>, callback: Py, - ) -> PyResult { + ) -> PyResult> { + ensure_python_callbacks_open()?; let type_handles: Vec = param_types.iter().map(|t| t.0.clone()).collect(); let value = create_python_delegate(iid.0, type_handles, callback)?; - Ok(DynWinRtDelegate(value)) + ensure_python_callbacks_open()?; + let output = Py::new( + py, + DynWinRtDelegate(Some(value), current_native_owner_thread(true)), + )?; + track_native_owner(py, output.clone_ref(py).into_any())?; + Ok(output) } /// Get the delegate as a DynWinRTValue for passing to WinRT methods. fn to_value(&self, py: Python<'_>) -> PyResult> { - tracked_native_value(py, self.0.clone()) + let value = self + .0 + .as_ref() + .ok_or_else(|| PyRuntimeError::new_err("DynWinRtDelegate has been released"))?; + tracked_native_value_with_policy(py, value.clone(), true) + } + + fn is_released(&self) -> bool { + self.0.is_none() + } + + fn release(&mut self) -> PyResult<()> { + ensure_native_owner_thread(self.1, true, "DynWinRtDelegate")?; + drop(self.0.take()); + Ok(()) } fn __repr__(&self) -> String { @@ -2652,10 +3247,20 @@ struct ElementFactoryCallbacks { recycle_element: Option, } -#[pyclass] +#[pyclass(weakref)] pub struct DynWinRtElementFactory { - value: dynwinrt::WinRTValue, + value: Option, callbacks: Arc>, + owner_thread: Option, +} + +impl Drop for DynWinRtElementFactory { + fn drop(&mut self) { + if self.value.is_some() && must_quarantine_owner(self.owner_thread, true) { + std::mem::forget(self.value.take()); + log_unsafe_native_owner_drop(); + } + } } impl DynWinRtElementFactory { @@ -2682,10 +3287,11 @@ impl DynWinRtElementFactory { element_iid: &WinGUID, get_element: Py, recycle_element: Py, - ) -> PyResult { + ) -> PyResult> { const E_FAIL: windows::core::HRESULT = windows::core::HRESULT(0x80004005_u32 as i32); const RO_E_CLOSED: windows::core::HRESULT = windows::core::HRESULT(0x80000013_u32 as i32); + ensure_python_callbacks_open()?; let element_iid = element_iid.0; let get_element = ElementFactoryCallback { error_target: get_element.clone_ref(py), @@ -2702,7 +3308,7 @@ impl DynWinRtElementFactory { let get_callbacks = callbacks.clone(); let get_callback: dynwinrt::ElementFactoryGetCallback = Box::new(move |args| { - Python::attach(|py| { + with_python_callback(|py| { let (callback, error_target) = { let callbacks = get_callbacks.lock().map_err(|_| E_FAIL)?; let callback = callbacks.get_element.as_ref().ok_or(RO_E_CLOSED)?; @@ -2712,7 +3318,7 @@ impl DynWinRtElementFactory { ) }; let result = (|| -> PyResult { - let argument = Py::new(py, DynWinRTValue::new(args.clone()))?; + let argument = callback_native_argument(py, args.clone())?; let result = callback.call1(py, (argument,))?; let value = result.extract::>(py)?; value.0.cast(&element_iid).map_err(map_dynwinrt_error) @@ -2725,11 +3331,12 @@ impl DynWinRtElementFactory { } } }) + .unwrap_or(Err(PYWINRT_E_INTERPRETER_CLOSED)) }); let recycle_callbacks = callbacks.clone(); let recycle_callback: dynwinrt::ElementFactoryRecycleCallback = Box::new(move |args| { - Python::attach(|py| { + with_python_callback(|py| { let (callback, error_target) = { let callbacks = match recycle_callbacks.lock() { Ok(callbacks) => callbacks, @@ -2744,7 +3351,7 @@ impl DynWinRtElementFactory { ) }; let result = (|| -> PyResult<()> { - let argument = Py::new(py, DynWinRTValue::new(args.clone()))?; + let argument = callback_native_argument(py, args.clone())?; callback.call1(py, (argument,))?; Ok(()) })(); @@ -2756,16 +3363,31 @@ impl DynWinRtElementFactory { } } }) + .unwrap_or(PYWINRT_E_INTERPRETER_CLOSED) }); - Ok(Self { - value: dynwinrt::create_element_factory_value(get_callback, recycle_callback), - callbacks, - }) + ensure_python_callbacks_open()?; + let output = Py::new( + py, + Self { + value: Some(dynwinrt::create_element_factory_value( + get_callback, + recycle_callback, + )), + callbacks, + owner_thread: current_native_owner_thread(true), + }, + )?; + track_native_owner(py, output.clone_ref(py).into_any())?; + Ok(output) } fn to_value(&self, py: Python<'_>) -> PyResult> { - tracked_native_value(py, self.value.clone()) + let value = self + .value + .as_ref() + .ok_or_else(|| PyRuntimeError::new_err("DynWinRtElementFactory has been released"))?; + tracked_native_value_with_policy(py, value.clone(), true) } fn release_callbacks(&self) -> PyResult<()> { @@ -2773,9 +3395,15 @@ impl DynWinRtElementFactory { } fn release(&mut self) -> PyResult<()> { + ensure_native_owner_thread(self.owner_thread, true, "DynWinRtElementFactory")?; self.clear_callbacks()?; - let value = std::mem::replace(&mut self.value, dynwinrt::WinRTValue::Null); - drop(value); + drop(self.value.take()); + Ok(()) + } + + fn _release_apartment_owner(&mut self) -> PyResult<()> { + ensure_native_owner_thread(self.owner_thread, true, "DynWinRtElementFactory")?; + drop(self.value.take()); Ok(()) } @@ -2828,17 +3456,27 @@ mod tests { use std::ffi::c_void; use std::sync::atomic::{AtomicU32, Ordering}; + #[test] + fn guarded_python_native_containers_are_send_and_sync() { + fn assert_send_sync() {} + assert_send_sync::(); + assert_send_sync::(); + } + #[derive(Default)] struct QueryCounts { queries: AtomicU32, addrefs: AtomicU32, releases: AtomicU32, + wrong_thread_addrefs: AtomicU32, + wrong_thread_releases: AtomicU32, } #[repr(C)] struct QueryProbe { vtable: *const windows::core::IUnknown_Vtbl, references: AtomicU32, + owner_thread: ThreadId, counts: Arc, } @@ -2857,6 +3495,7 @@ mod tests { let object = Box::new(Self { vtable: &Self::VTABLE, references: AtomicU32::new(1), + owner_thread: thread::current().id(), counts: counts.clone(), }); ( @@ -2887,14 +3526,94 @@ mod tests { } } + fn received_finalization_array(source: &IUnknown, nested: bool) -> DynWinRTArray { + use dynwinrt::{ + WinRtImplementation, WinRtImplementationPlan, WinRtInterfaceDefinition, + WinRtMethodDefinition, WinRtThreadingPolicy, + }; + + let table = dynwinrt::MetadataTable::new(); + let element = if nested { + let inner = + table.struct_type("Tests.FinalizationInner", &[table.interface(IUnknown::IID)]); + table.struct_type("Tests.FinalizationOuter", &[inner]) + } else { + table.interface(IUnknown::IID) + }; + let item = if nested { + let inner_type = element.field_type(0); + let mut inner = inner_type.default_value(); + inner.set_field_object(0, Some(source)).unwrap(); + let mut outer = element.default_value(); + outer.set_field_struct_checked(0, &inner).unwrap(); + dynwinrt::WinRTValue::Struct(outer) + } else { + dynwinrt::WinRTValue::Object(source.clone()) + }; + let signature = dynwinrt::MethodSignature::new(&table).add_out(table.array(&element)); + let iid = if nested { + GUID::from_u128(0x38684d40_bab3_42de_998d_26e4cce87c52) + } else { + GUID::from_u128(0x38684d40_bab3_42de_998d_26e4cce87c51) + }; + let plan = WinRtImplementationPlan::new( + vec![WinRtInterfaceDefinition { + name: "Tests.IFinalizationReceivedArray".into(), + interface_type: table.interface(iid), + required_iids: vec![], + methods: vec![WinRtMethodDefinition { + name: "GetItems".into(), + vtable_index: 6, + signature: signature.clone(), + }], + }], + WinRtThreadingPolicy::OwnerThread, + ) + .unwrap(); + let outputs = Mutex::new(Some(dynwinrt::WinRTValue::Array( + dynwinrt::ArrayData::from_values(element, &[item]), + ))); + let mut host = WinRtImplementation::new( + plan, + Arc::new(move |_, _, _| Ok(vec![outputs.lock().unwrap().take().unwrap()])), + None, + ) + .unwrap(); + let receiver = host.to_value().unwrap().cast(&iid).unwrap(); + let receiver_object = receiver.as_object().unwrap(); + let mut results = signature + .build(6) + .call_dynamic(receiver_object.as_raw(), &[]) + .unwrap(); + drop((receiver_object, receiver)); + host.release(); + let dynwinrt::WinRTValue::Array(array) = results.remove(0) else { + panic!("expected a received native array"); + }; + assert!(format!("{array:?}").contains("CoTaskMem(")); + DynWinRTArray(Some(array), Some(thread::current().id()), false) + } + unsafe extern "system" fn add_ref(this: *mut c_void) -> u32 { let object = unsafe { &*this.cast::() }; + if thread::current().id() != object.owner_thread { + object + .counts + .wrong_thread_addrefs + .fetch_add(1, Ordering::SeqCst); + } object.counts.addrefs.fetch_add(1, Ordering::SeqCst); object.references.fetch_add(1, Ordering::SeqCst) + 1 } unsafe extern "system" fn release(this: *mut c_void) -> u32 { let object = unsafe { &*this.cast::() }; + if thread::current().id() != object.owner_thread { + object + .counts + .wrong_thread_releases + .fetch_add(1, Ordering::SeqCst); + } object.counts.releases.fetch_add(1, Ordering::SeqCst); let remaining = object.references.fetch_sub(1, Ordering::SeqCst) - 1; if remaining == 0 { @@ -2983,7 +3702,7 @@ mod tests { } assert_eq!(counts.queries.load(Ordering::SeqCst), 3); - value.release(); + value.release().unwrap(); assert_eq!(counts.releases.load(Ordering::SeqCst), 2); let released = value ._try_query_interface(&WinGUID(QueryProbe::SUPPORTED)) @@ -3044,9 +3763,13 @@ mod tests { None, ) .unwrap(); - let receiver = DynWinRTValue::new(owner.to_value().unwrap().cast(&iid).unwrap()); + let receiver = Py::new( + py, + DynWinRTValue::new(owner.to_value().unwrap().cast(&iid).unwrap()), + ) + .unwrap(); let method = DynWinRTMethodHandle(interface.method(6).unwrap()); - let direct = method.invoke(py, receiver.clone(), vec![]).unwrap(); + let direct = method.invoke(py, receiver.clone_ref(py), vec![]).unwrap(); let detached = method.invoke_detached(py, receiver, vec![]).unwrap(); for result in [&direct, &detached] { assert!(matches!( @@ -3055,7 +3778,11 @@ mod tests { )); } let invalid = method - .invoke_detached(py, DynWinRTValue::new(dynwinrt::WinRTValue::I32(0)), vec![]) + .invoke_detached( + py, + Py::new(py, DynWinRTValue::new(dynwinrt::WinRTValue::I32(0))).unwrap(), + vec![], + ) .err() .expect("non-object receiver must be rejected"); assert!(invalid.is_instance_of::(py)); @@ -3137,4 +3864,636 @@ mod tests { ); }); } + + #[test] + fn managed_native_owner_never_releases_com_on_foreign_thread_or_without_gil() { + Python::initialize(); + + for foreign in [false, true] { + let (source, counts) = QueryProbe::new(); + let previous = MANAGED_APARTMENT_DEPTH.with(|depth| depth.replace(1)); + assert_eq!(previous, 0); + let mut owned = + DynWinRTValue::new_managed(dynwinrt::WinRTValue::Object(source.clone()), false); + MANAGED_APARTMENT_DEPTH.with(|depth| depth.set(previous)); + if foreign { + std::thread::spawn(move || { + assert!(owned.release().is_err()); + assert!(!owned.is_released()); + drop(owned); + }) + .join() + .unwrap(); + } else { + assert_eq!(unsafe { pyo3::ffi::PyGILState_Check() }, 0); + drop(owned); + } + assert_eq!(counts.releases.load(Ordering::SeqCst), 0); + drop(source); + assert_eq!(counts.releases.load(Ordering::SeqCst), 1); + assert_eq!(counts.addrefs.load(Ordering::SeqCst), 1); + } + } + + #[test] + fn no_gil_apartment_finalizer_retains_initialization_for_owner_thread_cleanup() { + Python::initialize(); + std::thread::spawn(|| { + assert_eq!(unsafe { pyo3::ffi::PyGILState_Check() }, 0); + let mut apartment = RoApartment::new(Some(1)); + apartment.initialize().unwrap(); + assert_eq!(managed_apartment_depth(), 1); + drop(apartment); + assert_eq!(managed_apartment_depth(), 1); + // This Rust-only test has no initialized Python owner registry. + unsafe { windows::Win32::System::WinRT::RoUninitialize() }; + MANAGED_APARTMENT_DEPTH.with(|depth| depth.set(0)); + assert_eq!(managed_apartment_depth(), 0); + }) + .join() + .unwrap(); + } + + #[test] + fn foreign_python_array_and_vector_inputs_reject_before_com_addref() { + Python::initialize(); + let (source, counts) = QueryProbe::new(); + let (value, element_type, key_type) = Python::attach(|py| { + ( + Py::new( + py, + DynWinRTValue::new_managed(dynwinrt::WinRTValue::Object(source.clone()), false), + ) + .unwrap(), + Py::new(py, DynWinRTType(TABLE.interface(QueryProbe::SUPPORTED))).unwrap(), + Py::new(py, DynWinRTType(TABLE.hstring())).unwrap(), + ) + }); + let (value, element_type, key_type) = thread::spawn(move || { + Python::attach(|py| { + let module = PyModule::new(py, "native_input_probe").unwrap(); + module.add_class::().unwrap(); + module.add_class::().unwrap(); + module.add_class::().unwrap(); + for name in ["from_values", "from_object_values"] { + let error = module + .getattr("DynWinRTArray") + .unwrap() + .call_method1( + name, + (vec![value.clone_ref(py)], element_type.clone_ref(py)), + ) + .unwrap_err(); + assert!(error.is_instance_of::(py)); + assert!(error.to_string().contains("owning COM apartment thread")); + } + let error = module + .getattr("DynWinRTValue") + .unwrap() + .call_method1( + "create_vector", + (vec![value.clone_ref(py)], element_type.clone_ref(py)), + ) + .unwrap_err(); + assert!(error.is_instance_of::(py)); + assert!(error.to_string().contains("owning COM apartment thread")); + let key = Py::new( + py, + DynWinRTValue::new(dynwinrt::WinRTValue::HString("key".into())), + ) + .unwrap(); + let error = module + .getattr("DynWinRTValue") + .unwrap() + .call_method1( + "create_map", + ( + vec![key], + vec![value.clone_ref(py)], + key_type.clone_ref(py), + element_type.clone_ref(py), + ), + ) + .unwrap_err(); + assert!(error.is_instance_of::(py)); + assert!(error.to_string().contains("owning COM apartment thread")); + let error = + native_outputs(py, "implementation callback", vec![value.clone_ref(py)]) + .unwrap_err(); + assert!(error.is_instance_of::(py)); + assert!(error.to_string().contains("owning COM apartment thread")); + }); + (value, element_type, key_type) + }) + .join() + .unwrap(); + assert_eq!( + counts.wrong_thread_addrefs.load(Ordering::SeqCst), + 0, + "a foreign Python argument was cloned before its thread check" + ); + assert_eq!(counts.wrong_thread_releases.load(Ordering::SeqCst), 0); + Python::attach(|py| value.borrow_mut(py).release().unwrap()); + drop((value, element_type, key_type, source)); + } + + #[test] + fn real_interpreter_shutdown_quarantines_native_owner_and_apartment() { + if std::env::var("DYNWINRT_FINALIZE_CHILD").as_deref() != Ok("1") { + for mode in ["explicit", "skipped", "explicit-last-alias"] { + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "runtime::tests::real_interpreter_shutdown_quarantines_native_owner_and_apartment", + "--nocapture", + ]) + .env("DYNWINRT_FINALIZE_CHILD", "1") + .env("DYNWINRT_FINALIZE_GATE_MODE", mode) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(15); + while child.try_wait().unwrap().is_none() { + if std::time::Instant::now() >= deadline { + child.kill().unwrap(); + let output = child.wait_with_output().unwrap(); + panic!( + "Python finalization child deadlocked ({mode}):\n{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + thread::sleep(std::time::Duration::from_millis(25)); + } + let child = child.wait_with_output().unwrap(); + assert!( + child.status.success(), + "{mode}:\n{}\n{}", + String::from_utf8_lossy(&child.stdout), + String::from_utf8_lossy(&child.stderr) + ); + assert!( + String::from_utf8_lossy(&child.stdout) + .contains(&format!("real-Py_FinalizeEx-quarantined-{mode}")) + ); + } + return; + } + + Python::initialize(); + let mut apartment = RoApartment::new(Some(1)); + apartment.initialize().unwrap(); + let (source, counts) = QueryProbe::new(); + let owned = DynWinRTValue::new_managed(dynwinrt::WinRTValue::Object(source.clone()), false); + let values = DynWinRTArray( + Some(dynwinrt::ArrayData::from_values( + TABLE.interface(IUnknown::IID), + &[dynwinrt::WinRTValue::Object(source.clone())], + )), + Some(thread::current().id()), + false, + ); + let inner_type = TABLE.struct_type( + "Tests.FinalizationNested", + &[TABLE.interface(IUnknown::IID)], + ); + let outer_type = TABLE.struct_type("Tests.FinalizationRecord", &[inner_type.clone()]); + let mut inner = inner_type.default_value(); + inner.set_field_object(0, Some(&source)).unwrap(); + let mut record = outer_type.default_value(); + record.set_field_struct_checked(0, &inner).unwrap(); + drop(inner); + let structured = DynWinRTStruct(Some(record), Some(thread::current().id()), false); + let cotaskmem = QueryProbe::received_finalization_array(&source, false); + let cotaskmem_nested = QueryProbe::received_finalization_array(&source, true); + let callback = + Python::attach(|py| py.eval(c"lambda *args: None", None, None).unwrap().unbind()); + let mut delegate = DynWinRtDelegate( + Some(create_python_delegate(GUID::zeroed(), vec![], callback).unwrap()), + Some(thread::current().id()), + ); + let (mut element_factory, factory_interface) = Python::attach(|py| { + let callback = py.eval(c"lambda args: None", None, None).unwrap().unbind(); + let native = DynWinRtElementFactory::create( + py, + &WinGUID(QueryProbe::SUPPORTED), + callback.clone_ref(py), + callback, + ) + .unwrap(); + let (value, callbacks, owner_thread) = { + let mut factory = native.borrow_mut(py); + ( + factory.value.take(), + factory.callbacks.clone(), + factory.owner_thread, + ) + }; + drop(native); + let interface = value + .as_ref() + .unwrap() + .cast(&dynwinrt::element_factory::IID_IELEMENT_FACTORY) + .unwrap() + .as_object() + .unwrap(); + ( + DynWinRtElementFactory { + value, + callbacks, + owner_thread, + }, + interface, + ) + }); + let recycle = dynwinrt::MethodSignature::new(&*TABLE) + .add_in(TABLE.object()) + .build(7); + let releases_before = counts.releases.load(Ordering::SeqCst); + + let mode = std::env::var("DYNWINRT_FINALIZE_GATE_MODE").unwrap(); + let delegate_alias = if mode == "explicit-last-alias" { + let alias = delegate.0.as_ref().unwrap().clone(); + delegate.release().unwrap(); + element_factory._release_apartment_owner().unwrap(); + Some(alias) + } else { + None + }; + let active_delegate = delegate_alias.as_ref().or(delegate.0.as_ref()).unwrap(); + if mode != "skipped" { + close_native_callback_gate().unwrap(); + assert_eq!(unsafe { pyo3::ffi::Py_IsInitialized() }, 1); + assert_eq!( + unsafe { invoke_delegate(active_delegate) }, + PYWINRT_E_INTERPRETER_CLOSED + ); + let error = recycle + .call_dynamic( + factory_interface.as_raw(), + &[dynwinrt::WinRTValue::Object(factory_interface.clone())], + ) + .unwrap_err(); + assert_eq!(error.code(), PYWINRT_E_INTERPRETER_CLOSED); + } + unsafe { pyo3::ffi::PyGILState_Ensure() }; + assert_eq!(unsafe { pyo3::ffi::Py_FinalizeEx() }, 0); + assert_eq!(unsafe { pyo3::ffi::Py_IsInitialized() }, 0); + assert_eq!( + unsafe { invoke_delegate(active_delegate) }, + windows::core::HRESULT(0x80000013u32 as i32), + "a native callback must fail closed once Python has finalized" + ); + let error = recycle + .call_dynamic( + factory_interface.as_raw(), + &[dynwinrt::WinRTValue::Object(factory_interface.clone())], + ) + .unwrap_err(); + assert_eq!(error.code(), PYWINRT_E_INTERPRETER_CLOSED); + if mode == "explicit-last-alias" { + drop(delegate_alias); + drop(factory_interface); + assert_eq!(Arc::strong_count(&element_factory.callbacks), 1); + } else { + std::mem::forget(factory_interface); + } + drop(( + owned, + values, + structured, + cotaskmem, + cotaskmem_nested, + delegate, + element_factory, + )); + assert_eq!(counts.releases.load(Ordering::SeqCst), releases_before); + assert_eq!(counts.wrong_thread_releases.load(Ordering::SeqCst), 0); + drop(apartment); + assert_eq!(managed_apartment_depth(), 1); + drop(source); + assert_eq!(counts.releases.load(Ordering::SeqCst), releases_before + 1); + println!("real-Py_FinalizeEx-quarantined-{mode}"); + } + + #[test] + fn embedded_public_callback_gate_survives_real_finalization() { + fn native_reference_count(object: &IUnknown) -> u32 { + let raw = object.as_raw(); + let vtable = unsafe { *(raw as *const *const windows::core::IUnknown_Vtbl) }; + let added = unsafe { ((*vtable).AddRef)(raw) }; + let remaining = unsafe { ((*vtable).Release)(raw) }; + assert_eq!(added, remaining + 1); + remaining + } + + fn assert_native_callbacks_closed(aliases: &[IUnknown; 3]) { + let delegate = dynwinrt::WinRTValue::Object(aliases[0].clone()); + assert_eq!( + unsafe { invoke_delegate(&delegate) }, + PYWINRT_E_INTERPRETER_CLOSED + ); + drop(delegate); + + let recycle = dynwinrt::MethodSignature::new(&*TABLE) + .add_in(TABLE.object()) + .build(7); + let error = recycle + .call_dynamic( + aliases[1].as_raw(), + &[dynwinrt::WinRTValue::Object(aliases[1].clone())], + ) + .unwrap_err(); + assert_eq!(error.code(), PYWINRT_E_INTERPRETER_CLOSED); + + let to_string = dynwinrt::MethodSignature::new(&*TABLE) + .add_out(TABLE.hstring()) + .build(6); + let error = to_string + .call_dynamic(aliases[2].as_raw(), &[]) + .unwrap_err(); + assert_eq!(error.code(), PYWINRT_E_INTERPRETER_CLOSED); + } + + if std::env::var("DYNWINRT_EMBEDDED_GATE_CHILD").as_deref() != Ok("1") { + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "runtime::tests::embedded_public_callback_gate_survives_real_finalization", + "--nocapture", + ]) + .env("DYNWINRT_EMBEDDED_GATE_CHILD", "1") + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + while child.try_wait().unwrap().is_none() { + if std::time::Instant::now() >= deadline { + child.kill().unwrap(); + let output = child.wait_with_output().unwrap(); + panic!( + "embedded host child deadlocked:\n{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + thread::sleep(std::time::Duration::from_millis(25)); + } + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "embedded host child exited {:?}:\n{}\n{}", + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!( + String::from_utf8_lossy(&output.stdout).contains("embedded-public-gate-complete") + ); + return; + } + + Python::initialize(); + let owner_thread = thread::current().id(); + unsafe { RoInitialize(RO_INIT_MULTITHREADED) }.unwrap(); + let (aliases, attempts_after_gate) = Python::attach(|py| { + let package_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("python") + .join("dynwinrt"); + let package = PyModule::new(py, "dynwinrt").unwrap(); + package + .setattr("__path__", vec![package_path.to_string_lossy().to_string()]) + .unwrap(); + package.setattr("__package__", "dynwinrt").unwrap(); + let native = PyModule::new(py, "dynwinrt.dynwinrt").unwrap(); + native.setattr("__package__", "dynwinrt").unwrap(); + let spec = py + .import("importlib.machinery") + .unwrap() + .getattr("ModuleSpec") + .unwrap() + .call1(("dynwinrt.dynwinrt", py.None())) + .unwrap(); + spec.setattr( + "origin", + package_path + .join("dynwinrt.pyd") + .to_string_lossy() + .to_string(), + ) + .unwrap(); + native.setattr("__spec__", spec).unwrap(); + let modules = py.import("sys").unwrap().getattr("modules").unwrap(); + modules.set_item("dynwinrt", &package).unwrap(); + modules.set_item("dynwinrt.dynwinrt", &native).unwrap(); + package.setattr("dynwinrt", &native).unwrap(); + crate::dynwinrt::init(&native).unwrap(); + + let source = std::fs::read_to_string(package_path.join("__init__.py")).unwrap(); + let source = std::ffi::CString::new(source).unwrap(); + py.run(source.as_c_str(), Some(&package.dict()), None) + .unwrap(); + + let locals = PyDict::new(py); + let script = std::ffi::CString::new( + r#" +import threading +from dynwinrt import ( + DynWinRTImplementation, DynWinRTImplementationMethod, DynWinRTInterfacePlan, + DynWinRTMethodSig, DynWinRTType, DynWinRTValue, DynWinRtDelegate, + DynWinRtElementFactory, RoApartment, WinGUID, shutdown_python_callbacks, +) + +apartment = RoApartment(1) +apartment.__enter__() +calls = [] +errors = [] +started = threading.Event() +proceed = threading.Event() +object_type = DynWinRTType.object() +stringable_iid = WinGUID.parse('96369f54-8eb6-48f0-abce-c1b211e627c3') +delegate_iid = WinGUID.parse('13fd99ec-a997-4497-aabc-247345013f26') +factory_iid = WinGUID.parse('75faba47-2cf2-54ae-91e6-0581556fddaa') +delegate_sig = DynWinRTMethodSig().add_in(object_type).add_in(object_type) +string_sig = DynWinRTMethodSig().add_out(DynWinRTType.hstring()) +string_type = DynWinRTType.register_interface( + 'Tests.IEmbeddedHostStringable', stringable_iid +).add_method('ToString', string_sig) +string_plan = DynWinRTInterfacePlan.create( + 'Tests.IEmbeddedHostStringable', string_type, + [DynWinRTImplementationMethod('ToString', 6, string_sig)], +) +factory_type = DynWinRTType.register_interface( + 'Tests.IEmbeddedHostElementFactory', factory_iid +).add_method( + 'GetElement', DynWinRTMethodSig().add_in(object_type).add_out(object_type) +).add_method('RecycleElement', DynWinRTMethodSig().add_in(object_type)) + +def delegate_callback(_first, _second): + calls.append('delegate') + +def implementation_callback(*_args): + calls.append('implementation') + return [DynWinRTValue.from_hstring('alive')] + +def recycle(_args): + calls.append('factory') + started.set() + assert proceed.wait(8), 'host did not settle its native callback' + +delegate = DynWinRtDelegate.create( + delegate_iid, [object_type, object_type], delegate_callback +) +delegate_view = delegate.to_value().cast(delegate_iid) +factory = DynWinRtElementFactory.create( + stringable_iid, lambda _args: DynWinRTValue.null_value(), recycle +) +factory_view = factory.to_value().cast(factory_iid) +implementation = DynWinRTImplementation.create( + [string_plan], implementation_callback +) +implementation_view = implementation.to_value().cast(stringable_iid) + +def invoke_delegate(): + return delegate_view.invoke_delegate( + delegate_iid, delegate_sig, + [DynWinRTValue.null_value(), DynWinRTValue.null_value()], + ) + +assert invoke_delegate() == [] +assert string_type.method(6).invoke(implementation_view, []).to_string() == 'alive' + +def invoke_factory(): + with RoApartment(1): + factory_type.method(7).invoke(factory_view, [factory_view]) + +def worker(): + try: + invoke_factory() + except BaseException as error: + errors.append(error) + +thread = threading.Thread(target=worker) +thread.start() +assert started.wait(5), 'native factory callback did not start' +try: + shutdown_python_callbacks() +except RuntimeError as error: + assert 'callback(s) are in flight' in str(error), error +else: + raise AssertionError('public gate closed while a callback was in flight') +assert invoke_delegate() == [], 'failed gate attempt closed unrelated callbacks' +proceed.set() +thread.join(10) +assert not thread.is_alive() and not errors, errors + +delegate.release() +factory._release_apartment_owner() +implementation.release() +assert invoke_delegate() == [] +assert string_type.method(6).invoke(implementation_view, []).to_string() == 'alive' +factory_type.method(7).invoke(factory_view, [factory_view]) +assert calls == [ + 'delegate', 'implementation', 'factory', 'delegate', + 'delegate', 'implementation', 'factory', +] +shutdown_python_callbacks() +shutdown_python_callbacks() +state = { + 'apartment': apartment, + 'views': (delegate_view, factory_view, implementation_view), + 'scalar': DynWinRTValue.from_u32(77), + 'calls': calls, +} +"#, + ) + .unwrap(); + py.run(script.as_c_str(), Some(&locals), None).unwrap(); + let state = locals.get_item("state").unwrap().unwrap(); + let views = state.get_item("views").unwrap(); + // The host AddRefs each borrowed Python view before apartment cleanup. + let aliases: [IUnknown; 3] = std::array::from_fn(|index| { + let raw = views + .get_item(index) + .unwrap() + .call_method0("as_raw") + .unwrap() + .extract::() + .unwrap() as usize as *mut c_void; + unsafe { IUnknown::from_raw_borrowed(&raw) } + .expect("a live native alias") + .clone() + }); + state + .get_item("apartment") + .unwrap() + .call_method0("close") + .unwrap(); + assert_eq!(managed_apartment_depth(), 0); + for index in 0..3 { + assert!( + views + .get_item(index) + .unwrap() + .call_method0("is_released") + .unwrap() + .extract::() + .unwrap() + ); + } + assert_eq!( + state + .get_item("scalar") + .unwrap() + .call_method0("to_u32") + .unwrap() + .extract::() + .unwrap(), + 77 + ); + for alias in &aliases { + assert_eq!(native_reference_count(alias), 1); + } + let attempts_after_gate = CALLBACK_ATTACH_ATTEMPTS.load(Ordering::SeqCst); + assert_native_callbacks_closed(&aliases); + assert_eq!( + CALLBACK_ATTACH_ATTEMPTS.load(Ordering::SeqCst), + attempts_after_gate + ); + assert_eq!(state.get_item("calls").unwrap().len().unwrap(), 7); + (aliases, attempts_after_gate) + }); + + assert_eq!(thread::current().id(), owner_thread); + unsafe { pyo3::ffi::PyGILState_Ensure() }; + assert_eq!(unsafe { pyo3::ffi::Py_FinalizeEx() }, 0); + assert_eq!(unsafe { pyo3::ffi::Py_IsInitialized() }, 0); + assert_native_callbacks_closed(&aliases); + assert_eq!( + CALLBACK_ATTACH_ATTEMPTS.load(Ordering::SeqCst), + attempts_after_gate + ); + for alias in &aliases { + assert_eq!(native_reference_count(alias), 1); + } + drop(aliases); + assert_eq!(thread::current().id(), owner_thread); + unsafe { windows::Win32::System::WinRT::RoUninitialize() }; + let mut apartment_type = windows::Win32::System::Com::APTTYPE_CURRENT; + let mut qualifier = windows::Win32::System::Com::APTTYPEQUALIFIER_NONE; + let error = unsafe { + windows::Win32::System::Com::CoGetApartmentType(&mut apartment_type, &mut qualifier) + } + .unwrap_err(); + assert_eq!( + error.code(), + windows::Win32::Foundation::CO_E_NOTINITIALIZED + ); + println!( + "embedded-public-gate-complete: {attempts_after_gate} pre-gate Python attachments, \ + none after gate, three native aliases released on their owner thread" + ); + } } diff --git a/bindings/py/tests/test_error_messages.py b/bindings/py/tests/test_error_messages.py index 28b5288c..511e9e55 100644 --- a/bindings/py/tests/test_error_messages.py +++ b/bindings/py/tests/test_error_messages.py @@ -56,7 +56,8 @@ "calling WinRT APIs." ) RELEASED_REASON = re.escape( - "has been released (its projected_lifetime_scope() exited, or " + "has been released (its projected_lifetime_scope() or managed COM " + "apartment exited, or " "release_projected() / DynWinRTValue.release() was called) and can no longer " "be used." ) + "$" diff --git a/bindings/py/tests/test_implementation.py b/bindings/py/tests/test_implementation.py index 0260c87d..2e5af05d 100644 --- a/bindings/py/tests/test_implementation.py +++ b/bindings/py/tests/test_implementation.py @@ -1265,7 +1265,10 @@ def late_native_call(): try: state["factory"]() except RuntimeError as error: - assert "shutting down" in str(error) + assert str(error) == ( + "Python WinRT callbacks have been shut down; register them " + "before shutdown_python_callbacks()" + ) else: raise AssertionError("created implementation during shutdown") owner.dispose() diff --git a/bindings/py/tests/test_native_container_lifetime.py b/bindings/py/tests/test_native_container_lifetime.py index f113dd3a..5400d052 100644 --- a/bindings/py/tests/test_native_container_lifetime.py +++ b/bindings/py/tests/test_native_container_lifetime.py @@ -332,6 +332,1019 @@ def exercise(url): print('stock-uri-array-one-apartment', flush=True) """ +_UNSCOPED_APARTMENT = r""" +import gc +import sys +import weakref +from dynwinrt import ( + DynWinRTArray, DynWinRTStruct, DynWinRTType, DynWinRTValue, + RoApartment, ro_initialize, ro_uninitialize, +) + +mode = sys.argv[1] +if mode == 'manual': + ro_initialize(1) +with RoApartment(1): + if mode == 'nested': + with RoApartment(1): + source = DynWinRTValue.activation_factory('Windows.Foundation.Uri') + assert not source.is_released() + else: + source = DynWinRTValue.activation_factory('Windows.Foundation.Uri') + object_type = DynWinRTType.object() + array = DynWinRTArray.from_object_values([source], object_type) + extracted = array.to_value().as_array() + shape = DynWinRTType.struct_type('Tests.UnscopedApartment', [object_type]) + record = DynWinRTStruct.create(shape) + record.set_object(0, source) + nested = record.to_value().as_struct() + scalars = DynWinRTArray.from_i32_values([7]) + + transient = DynWinRTArray.from_object_values([source], object_type) + observed = weakref.ref(transient) + del transient + gc.collect() + assert observed() is None, 'implicit apartment registry rooted a temporary array' + +if mode == 'manual': + assert not source.is_released() and not array.is_released() + ro_uninitialize() +assert source.is_released() +assert array.is_released() and extracted.is_released() +assert record.is_released() and nested.is_released() +assert scalars.to_i32_list() == [7] +for operation in (source.identity_raw, lambda: array.get(0), lambda: record.get_object(0)): + try: + operation() + except RuntimeError as error: + assert 'released' in str(error) + else: + raise AssertionError('unscoped COM owner remained callable outside apartment') +print('unscoped-owner-safe', mode, flush=True) +if mode != 'shutdown': + del source, array, extracted, record, nested + gc.collect() + print('after-del', mode, flush=True) +""" + +_EXTERNAL_RO_INITIALIZE = r""" +import ctypes +from dynwinrt import DynWinRTValue, RoApartment, WinGUID + +runtimeobject = ctypes.WinDLL('combase.dll') +runtimeobject.RoInitialize.argtypes = (ctypes.c_int,) +runtimeobject.RoInitialize.restype = ctypes.c_long +runtimeobject.RoUninitialize.argtypes = () +runtimeobject.RoUninitialize.restype = None +assert runtimeobject.RoInitialize(1) >= 0 +try: + external = DynWinRTValue.activation_factory('Windows.Foundation.Uri') + with RoApartment(1): + owned = external.cast( + WinGUID.parse('44a9796f-723e-4fdf-a218-033e75b0c084') + ) + assert not external.is_released() and not owned.is_released() + assert owned.is_released() and not external.is_released() + assert external.identity_raw() != 0 + assert runtimeobject.RoInitialize(1) == 1 + runtimeobject.RoUninitialize() + external.release() +finally: + runtimeobject.RoUninitialize() +print('external-host-initialization-preserved', flush=True) +""" + +_INVERTED_PROJECTION_SCOPE = r""" +from dynwinrt import ( + DynWinRTArray, DynWinRTType, DynWinRTValue, RoApartment, + projected_lifetime_scope, +) + +with projected_lifetime_scope() as scope: + with RoApartment(1): + source = DynWinRTValue.activation_factory('Windows.Foundation.Uri') + array = DynWinRTArray.from_object_values( + [source], DynWinRTType.object() + ) + assert not source.is_released() and not array.is_released() + assert source.is_released() and array.is_released() +assert scope.disposed +source.release() +array.release() +print('inverted-scope-deduplicated', flush=True) +""" + +_UNSCOPED_CLOSE_FAILURE = r""" +import gc +import sys +from dynwinrt import RoApartment, retry_pending_apartment_close +from dynwinrt.dynwinrt import _dynwinrt_track_native + +class FailingOwner: + def __init__(self): + self.attempts = 0 + def release(self): + self.attempts += 1 + if self.attempts == 1: + raise RuntimeError('owner release failed') + +mode = sys.argv[1] +try: + with RoApartment(1): + owner = FailingOwner() + _dynwinrt_track_native(owner) + if mode == 'body': + raise ValueError('original body failure') +except ValueError as error: + assert mode == 'body' and str(error) == 'original body failure' + assert isinstance(error.__cause__, RuntimeError) + assert str(error.__cause__) == 'owner release failed' +except RuntimeError as error: + assert mode == 'cleanup' and str(error) == 'owner release failed' +else: + raise AssertionError('final apartment close hid owner cleanup failure') +gc.collect() +assert owner.attempts == 1 +retry_pending_apartment_close() +assert owner.attempts == 2 +print('failed-apartment-close-retried', mode, flush=True) +""" + +_REENTRANT_APARTMENT_CLOSE = r""" +from dynwinrt import DynWinRTValue, RoApartment +from dynwinrt.dynwinrt import _dynwinrt_track_native + +created = [] +class ReentrantOwner: + def __init__(self): + self.releases = 0 + def release(self): + self.releases += 1 + if self.releases == 1: + created.append(DynWinRTValue.activation_factory('Windows.Foundation.Uri')) + +with RoApartment(1): + owner = ReentrantOwner() + _dynwinrt_track_native(owner) +assert owner.releases == 1 +assert len(created) == 1 and created[0].is_released() +print('reentrant-apartment-owners-drained', flush=True) +""" + +_UNBALANCED_MANAGED_APARTMENT = r""" +from dynwinrt import retry_pending_apartment_close, ro_uninitialize + +for invalid in (ro_uninitialize, retry_pending_apartment_close): + try: + invalid() + except RuntimeError as error: + assert 'requires a successful' in str(error) or 'no failed RoApartment close' in str(error) + else: + raise AssertionError('unbalanced managed apartment call succeeded') +print('unbalanced-managed-apartment-rejected', flush=True) +""" + +_WRONG_THREAD_APARTMENT = r""" +import threading +from dynwinrt import DynWinRTValue, RoApartment + +with RoApartment(1) as apartment: + source = DynWinRTValue.activation_factory('Windows.Foundation.Uri') + errors = [] + def close_from_foreign_thread(): + try: + apartment.close() + except RuntimeError as error: + errors.append(str(error)) + worker = threading.Thread(target=close_from_foreign_thread) + worker.start() + worker.join() + assert len(errors) == 1 and 'initializing thread' in errors[0], errors + assert not source.is_released() +assert source.is_released() +print('wrong-thread-close-retryable', flush=True) +""" + +_FOREIGN_GUARD_FINALIZER = r""" +import gc +import threading +from dynwinrt import DynWinRTType, DynWinRTValue, RoApartment +from dynwinrt.dynwinrt import _managed_apartment_depth + +apartment = RoApartment(1) +apartment.__enter__() +native = DynWinRTValue.activation_factory('Windows.Foundation.Uri') +handoff = [apartment] +del apartment +def finalizer_thread(): + handoff.clear() + gc.collect() + DynWinRTType.i32_type() + gc.collect() +worker = threading.Thread(target=finalizer_thread) +worker.start() +worker.join(10) +assert not worker.is_alive() +assert _managed_apartment_depth() == 1, 'foreign Drop uninitialized the owner thread' +assert native.identity_raw() != 0 +native.release() +print('foreign-guard-drop-retained-apartment', flush=True) +""" + +_EXPLICIT_CALLBACK_SHUTDOWN = r""" +import sys +import threading +from dynwinrt import ( + DynWinRTImplementation, DynWinRTImplementationMethod, DynWinRTInterfacePlan, + DynWinRTMethodSig, DynWinRTType, DynWinRTValue, DynWinRtDelegate, + DynWinRtElementFactory, RoApartment, WinGUID, shutdown_python_callbacks, +) + +mode = sys.argv[1] +calls = [] +errors = [] +started = threading.Event() +proceed = threading.Event() +stringable_iid = WinGUID.parse('96369f54-8eb6-48f0-abce-c1b211e627c3') +factory_iid = WinGUID.parse('75faba47-2cf2-54ae-91e6-0581556fddaa') +delegate_iid = WinGUID.parse('13fd99ec-a997-4497-aabc-247345013f26') +object_type = DynWinRTType.object() +delegate_sig = DynWinRTMethodSig().add_in(object_type).add_in(object_type) +signature = DynWinRTMethodSig().add_out(DynWinRTType.hstring()) +string_type = DynWinRTType.register_interface( + 'Tests.IStringableShutdownGate', stringable_iid +).add_method('ToString', signature) +string_plan = DynWinRTInterfacePlan.create( + 'Tests.IStringableShutdownGate', string_type, + [DynWinRTImplementationMethod('ToString', 6, signature)], +) +factory_type = DynWinRTType.register_interface( + 'Tests.IElementFactoryShutdownGate', factory_iid +).add_method( + 'GetElement', DynWinRTMethodSig().add_in(object_type).add_out(object_type) +).add_method('RecycleElement', DynWinRTMethodSig().add_in(object_type)) + +def recycle(_args): + calls.append('factory') + if mode == 'inflight': + started.set() + assert proceed.wait(8), 'host never settled its callback' + +def delegate_callback(_first, _second): + calls.append('delegate') + +def implementation_callback(*_args): + calls.append('implementation') + if mode == 'inflight-implementation': + try: + shutdown_python_callbacks() + except RuntimeError as error: + assert 'callback(s) are in flight' in str(error), error + else: + raise AssertionError('host gate closed during an implementation callback') + return [DynWinRTValue.from_hstring('alive')] + +def invoke_delegate(value): + return value.invoke_delegate( + delegate_iid, delegate_sig, + [DynWinRTValue.null_value(), DynWinRTValue.null_value()], + ) + +with RoApartment(1): + delegate = DynWinRtDelegate.create( + delegate_iid, [object_type, object_type], delegate_callback + ) + delegate_alias = delegate.to_value().cast(delegate_iid) + delegate.release() + factory = DynWinRtElementFactory.create( + stringable_iid, lambda _args: None, recycle + ) + factory_alias = factory.to_value().cast(factory_iid) + factory._release_apartment_owner() + implementation = DynWinRTImplementation.create( + [string_plan], implementation_callback + ) + implementation_alias = implementation.to_value().cast(stringable_iid) + implementation.release() + + assert invoke_delegate(delegate_alias) == [] + assert string_type.method(6).invoke(implementation_alias, []).to_string() == 'alive' + if mode == 'inflight': + def worker(): + try: + with RoApartment(1): + factory_type.method(7).invoke(factory_alias, [factory_alias]) + except BaseException as error: + errors.append(error) + thread = threading.Thread(target=worker) + thread.start() + assert started.wait(5), 'native callback did not start' + try: + shutdown_python_callbacks() + except RuntimeError as error: + assert 'callback(s) are in flight' in str(error) + else: + raise AssertionError('host gate closed while a callback was in flight') + proceed.set() + thread.join(10) + assert not thread.is_alive() and not errors, errors + else: + factory_type.method(7).invoke(factory_alias, [factory_alias]) + assert calls == ['delegate', 'implementation', 'factory'] + if mode == 'gate-preflight': + import dynwinrt.dynwinrt as native_module + implementation_runtime = native_module._dynwinrt_implementation_runtime + del native_module._dynwinrt_implementation_runtime + try: + try: + shutdown_python_callbacks() + except AttributeError as error: + assert '_dynwinrt_implementation_runtime' in str(error), error + else: + raise AssertionError('missing implementation runtime closed the gate') + finally: + native_module._dynwinrt_implementation_runtime = implementation_runtime + assert invoke_delegate(delegate_alias) == [] + assert calls.pop() == 'delegate' + assert string_type.method(6).invoke(implementation_alias, []).to_string() == 'alive' + assert calls.pop() == 'implementation' + if mode == 'ordered': + factory.release_callbacks() + implementation.dispose() + for alias in (delegate_alias, factory_alias, implementation_alias): + alias.release() + shutdown_python_callbacks() + shutdown_python_callbacks() + + if mode != 'ordered': + def expect_closed(call): + try: + call() + except OSError as error: + assert error.winerror == -2147483629, error + else: + raise AssertionError('late native callback reached Python after shutdown gate') + expect_closed(lambda: invoke_delegate(delegate_alias)) + expect_closed(lambda: factory_type.method(7).invoke(factory_alias, [factory_alias])) + expect_closed(lambda: string_type.method(6).invoke(implementation_alias, [])) + factory.release_callbacks() + implementation.dispose() + for alias in (delegate_alias, factory_alias, implementation_alias): + alias.release() + assert calls == ['delegate', 'implementation', 'factory'] + + for constructor in ( + lambda: DynWinRtDelegate.create(delegate_iid, [object_type, object_type], delegate_callback), + lambda: DynWinRtElementFactory.create(stringable_iid, lambda _args: None, recycle), + lambda: DynWinRTImplementation.create([string_plan], implementation_callback), + ): + try: + constructor() + except RuntimeError as error: + assert 'shut down' in str(error) or 'shutting down' in str(error), error + else: + raise AssertionError('Python-backed callback was created after shutdown gate') + del delegate_alias, factory_alias, implementation_alias + del delegate, factory, implementation +if mode == 'ordered': + print('host-ordered-shutdown-safe', flush=True) +else: + print('explicit-native-callback-gate-safe', mode, flush=True) +""" + +_SPECIAL_UNSCOPED_OWNER = r""" +import sys +from dynwinrt import ( + DynWinRTImplementation, DynWinRTImplementationMethod, DynWinRTInterfacePlan, + DynWinRTMethodSig, DynWinRTType, DynWinRTValue, DynWinRtDelegate, + DynWinRtElementFactory, RoApartment, WinGUID, +) + +mode = sys.argv[1] +early = sys.argv[2] == 'early' +stringable = WinGUID.parse('96369f54-8eb6-48f0-abce-c1b211e627c3') +element_factory = WinGUID.parse('75faba47-2cf2-54ae-91e6-0581556fddaa') + +with RoApartment(1): + if mode == 'delegate': + owner = DynWinRtDelegate.create(stringable, [], lambda *args: None) + alias = owner.to_value() + identity = alias.identity_raw() + if early: + owner.release() + assert owner.is_released() and alias.identity_raw() == identity + elif mode == 'element_factory': + calls = [] + typ = DynWinRTType.register_interface('Tests.IApartmentElementFactory', element_factory) + typ = typ.add_method( + 'GetElement', DynWinRTMethodSig() + .add_in(DynWinRTType.object()).add_out(DynWinRTType.object()) + ).add_method('RecycleElement', DynWinRTMethodSig().add_in(DynWinRTType.object())) + owner = DynWinRtElementFactory.create( + stringable, lambda _args: None, lambda _args: calls.append('recycled') + ) + alias = owner.to_value().cast(element_factory) + if early: + owner._release_apartment_owner() + typ.method(7).invoke(alias, [alias]) + assert calls == ['recycled'], 'owner release disconnected an independent alias' + else: + signature = DynWinRTMethodSig().add_out(DynWinRTType.hstring()) + typ = DynWinRTType.register_interface('Tests.IApartmentImplementation', stringable) + typ = typ.add_method('ToString', signature) + plan = DynWinRTInterfacePlan.create( + 'Tests.IApartmentImplementation', typ, + [DynWinRTImplementationMethod('ToString', 6, signature)], + ) + owner = DynWinRTImplementation.create( + [plan], lambda *_: [DynWinRTValue.from_hstring('still alive')], + ) + alias = owner.to_value().cast(stringable) + if early: + owner.release() + assert typ.method(6).invoke(alias, []).to_string() == 'still alive' + + assert not alias.is_released() +assert alias.is_released() +if mode == 'delegate': + assert owner.is_released() +else: + try: + owner.to_value() + except (RuntimeError, OSError) as error: + assert 'released' in str(error) or 'closed' in str(error) + else: + raise AssertionError('special owner kept an independent COM reference after apartment') +print('special-unscoped-owner', mode, 'early' if early else 'automatic', flush=True) +""" + +_CROSS_THREAD_LOCAL_OWNER = r""" +import gc +import sys +import threading +import weakref +from dynwinrt import ( + DynWinRTImplementation, DynWinRTImplementationMethod, DynWinRTInterfacePlan, + DynWinRTMethodSig, DynWinRTType, DynWinRTValue, DynWinRtDelegate, + DynWinRtElementFactory, RoApartment, WinGUID, +) + +mode = sys.argv[1] +owner_thread = threading.get_ident() +dropped = [] +errors = [] +stringable = WinGUID.parse('96369f54-8eb6-48f0-abce-c1b211e627c3') + +class Handler: + def __call__(self, *_args): + return None + def get(self, _args): + return None + def recycle(self, _args): + return None + def dispatch(self, *_args): + return [DynWinRTValue.from_hstring('alive')] + def __del__(self): + dropped.append(threading.get_ident()) + +with RoApartment(1): + handler = Handler() + retained = weakref.ref(handler) + if mode == 'delegate': + owner = DynWinRtDelegate.create(stringable, [], handler) + alias = owner.to_value() + owner.release() + elif mode == 'element_factory': + owner = DynWinRtElementFactory.create( + stringable, handler.get, handler.recycle + ) + alias = owner.to_value() + owner._release_apartment_owner() + else: + signature = DynWinRTMethodSig().add_out(DynWinRTType.hstring()) + typ = DynWinRTType.register_interface( + 'Tests.ICrossThreadOwner', stringable + ).add_method('ToString', signature) + plan = DynWinRTInterfacePlan.create( + 'Tests.ICrossThreadOwner', typ, + [DynWinRTImplementationMethod('ToString', 6, signature)], + ) + owner = DynWinRTImplementation.create([plan], handler.dispatch) + alias = owner.to_value() + owner.release() + del owner, handler + assert retained() is not None + handoff = [alias] + del alias + + def drop_on_worker(): + try: + handoff.clear() + gc.collect() + DynWinRTType.i32_type() + gc.collect() + assert retained() is None, 'an agile/local COM owner leaked on foreign Python Drop' + except BaseException as error: + errors.append(error) + + worker = threading.Thread(target=drop_on_worker) + worker.start() + worker.join(10) + assert not worker.is_alive() and not errors, errors +assert len(dropped) == 1 and dropped[0] != owner_thread +print('local-owner-foreign-drop-balanced', mode, flush=True) +""" + +_AGILE_CONTAINER_FOREIGN_DROP = r""" +import gc +import sys +import threading +import weakref +from dynwinrt import ( + DynWinRTArray, DynWinRTStruct, DynWinRTType, DynWinRtElementFactory, + RoApartment, WinGUID, +) + +mode = sys.argv[1] +disposed = [] +errors = [] + +class Handler: + def get(self, _args): + return None + def recycle(self, _args): + return None + def __del__(self): + disposed.append(threading.get_ident()) + +with RoApartment(1): + handler = Handler() + weak = weakref.ref(handler) + owner = DynWinRtElementFactory.create( + WinGUID.parse('96369f54-8eb6-48f0-abce-c1b211e627c3'), + handler.get, handler.recycle, + ) + source = owner.to_value() + if mode == 'array': + container = DynWinRTArray.from_object_values( + [source], DynWinRTType.object() + ) + elif mode == 'nested': + inner_type = DynWinRTType.struct_type( + 'Tests.AgileInner', [DynWinRTType.object()] + ) + outer_type = DynWinRTType.struct_type('Tests.AgileOuter', [inner_type]) + inner = DynWinRTStruct.create(inner_type) + inner.set_object(0, source) + container = DynWinRTStruct.create(outer_type) + container.set_struct(0, inner) + del inner + else: + shape = DynWinRTType.struct_type( + 'Tests.AgileContainerField', [DynWinRTType.object()] + ) + container = DynWinRTStruct.create(shape) + container.set_object(0, source) + owner._release_apartment_owner() + source.release() + del owner, source, handler + assert weak() is not None + handoff = [container] + del container + + def destroy_on_worker(): + try: + handoff.clear() + gc.collect() + DynWinRTType.i32_type() + gc.collect() + assert weak() is None, 'agile COM container leaked after foreign Drop' + except BaseException as error: + errors.append(error) + + worker = threading.Thread(target=destroy_on_worker) + worker.start() + worker.join(10) + assert not worker.is_alive() and not errors, errors +assert len(disposed) == 1 +print('agile-container-foreign-drop-balanced', mode, flush=True) +""" + +_NONAGILE_CONTAINER_FOREIGN_DROP = r""" +import gc +import sys +import threading +import weakref +from dynwinrt import ( + DynWinRTArray, DynWinRTImplementation, DynWinRTImplementationMethod, + DynWinRTInterfacePlan, DynWinRTMethodSig, DynWinRTStruct, DynWinRTType, + DynWinRTValue, RoApartment, WinGUID, +) + +mode = sys.argv[1] +errors = [] + +class Handler: + def dispatch(self, *_args): + return [DynWinRTValue.from_hstring('owner thread')] + +iid = WinGUID.parse('96369f54-8eb6-48f0-abce-c1b211e627c3') +signature = DynWinRTMethodSig().add_out(DynWinRTType.hstring()) +interface = DynWinRTType.register_interface('Tests.INonAgileForeignContainer', iid) +interface = interface.add_method('ToString', signature) +plan = DynWinRTInterfacePlan.create( + 'Tests.INonAgileForeignContainer', interface, + [DynWinRTImplementationMethod('ToString', 6, signature)], +) +with RoApartment(1): + handler = Handler() + retained = weakref.ref(handler) + owner = DynWinRTImplementation.create([plan], handler.dispatch) + source = owner.to_value() + if mode == 'array': + container = DynWinRTArray.from_object_values( + [source], DynWinRTType.object() + ) + elif mode == 'nested': + inner_type = DynWinRTType.struct_type( + 'Tests.NonAgileForeignInner', [DynWinRTType.object()] + ) + outer_type = DynWinRTType.struct_type( + 'Tests.NonAgileForeignOuter', [inner_type] + ) + inner = DynWinRTStruct.create(inner_type) + inner.set_object(0, source) + container = DynWinRTStruct.create(outer_type) + container.set_struct(0, inner) + del inner + else: + shape = DynWinRTType.struct_type( + 'Tests.NonAgileForeignField', [DynWinRTType.object()] + ) + container = DynWinRTStruct.create(shape) + container.set_object(0, source) + owner.release() + source.release() + del source, handler, owner + assert retained() is not None + handoff = [container] + del container + + def release_on_worker(): + try: + native = handoff[0] + try: + if mode == 'array': + native.get(0) + elif mode == 'nested': + native.get_struct(0) + else: + native.get_object(0) + except RuntimeError as error: + assert 'owning COM apartment thread' in str(error) + else: + raise AssertionError('non-agile COM field was callable on a foreign thread') + del native + handoff.clear() + gc.collect() + DynWinRTType.i32_type() + gc.collect() + assert retained() is not None, 'non-agile COM ref was released off-thread' + except BaseException as error: + errors.append(error) + + worker = threading.Thread(target=release_on_worker) + worker.start() + worker.join(10) + assert not worker.is_alive() and not errors, errors +print('nonagile-container-foreign-drop-quarantined', mode, flush=True) +""" + +_FOREIGN_NONAGILE_STRUCT_MUTATION = r""" +import threading +from dynwinrt import ( + DynWinRTImplementation, DynWinRTImplementationMethod, DynWinRTInterfacePlan, + DynWinRTMethodSig, DynWinRTStruct, DynWinRTType, DynWinRTValue, + RoApartment, WinGUID, +) + +object_type = DynWinRTType.object() +inner_type = DynWinRTType.struct_type('Tests.ForeignInnerField', [object_type]) +outer_type = DynWinRTType.struct_type('Tests.ForeignOuterField', [inner_type]) +stringable = WinGUID.parse('96369f54-8eb6-48f0-abce-c1b211e627c3') +signature = DynWinRTMethodSig().add_out(DynWinRTType.hstring()) +interface = DynWinRTType.register_interface('Tests.IForeignStructOwner', stringable) +interface = interface.add_method('ToString', signature) +plan = DynWinRTInterfacePlan.create( + 'Tests.IForeignStructOwner', interface, + [DynWinRTImplementationMethod('ToString', 6, signature)], +) + +errors = [] +with RoApartment(1): + direct = DynWinRTStruct.create(inner_type) + nested = DynWinRTStruct.create(outer_type) + def worker(): + try: + with RoApartment(1): + owner = DynWinRTImplementation.create( + [plan], lambda *_: [DynWinRTValue.from_hstring('alive')] + ) + source = owner.to_value() + source_record = DynWinRTStruct.create(inner_type) + source_record.set_object(0, source) + for attempt in ( + lambda: direct.set_object(0, source), + lambda: nested.set_struct(0, source_record), + ): + try: + attempt() + except RuntimeError as error: + assert 'non-agile' in str(error), error + else: + raise AssertionError('cross-apartment non-agile COM field was stored') + except BaseException as error: + errors.append(error) + thread = threading.Thread(target=worker) + thread.start() + thread.join(10) + assert not thread.is_alive() and not errors, errors + assert direct.get_object(0).is_null() + assert nested.get_struct(0).get_object(0).is_null() +assert direct.is_released() and nested.is_released() +print('foreign-nonagile-struct-mutation-rejected', flush=True) +""" + +_UNSCOPED_ASYNC_OWNER = r""" +from pathlib import Path +from tempfile import TemporaryDirectory +from dynwinrt import ( + DynWinRTMethodSig, DynWinRTType, DynWinRTValue, RoApartment, WinGUID, +) +from dynwinrt.dynwinrt import _DynWinRTAsync + +with TemporaryDirectory() as folder: + path = Path(folder) / 'owner.txt' + path.write_text('alive') + static_iid = WinGUID.parse('5984c710-daf2-43c8-8bb4-a4d3eacfd03f') + file_iid = WinGUID.parse('fa3f6186-4214-428c-a64c-14c9ac7315ea') + file_type = DynWinRTType.runtime_class( + 'Windows.Storage.StorageFile', DynWinRTType.interface(file_iid) + ) + statics = DynWinRTType.register_interface( + 'Tests.IStorageFileStaticsLifetime', static_iid + ).add_method( + 'GetFileFromPathAsync', + DynWinRTMethodSig() + .add_in(DynWinRTType.hstring()) + .add_out(DynWinRTType.i_async_operation(file_type)), + ) + with RoApartment(1): + factory = DynWinRTValue.activation_factory( + 'Windows.Storage.StorageFile' + ).cast(static_iid) + raw = statics.method(6).invoke( + factory, [DynWinRTValue.from_hstring(str(path))] + ) + operation = _DynWinRTAsync(raw, lambda value: value) + result = operation.wait() + assert result.identity_raw() != 0 + assert raw.is_released() and factory.is_released() and result.is_released() + try: + operation.wait() + except RuntimeError as error: + assert 'released' in str(error) + else: + raise AssertionError('async owner retained COM past apartment exit') +print('unscoped-async-owner-released', flush=True) +""" + +_NONAGILE_CALLBACK_COPY = r""" +from dynwinrt import ( + DynWinRTImplementation, DynWinRTImplementationMethod, DynWinRTInterfacePlan, + DynWinRTMethodSig, DynWinRTType, DynWinRTValue, DynWinRtElementFactory, + RoApartment, WinGUID, projected_lifetime_scope, +) + +stringable = WinGUID.parse('96369f54-8eb6-48f0-abce-c1b211e627c3') +element_factory = WinGUID.parse('75faba47-2cf2-54ae-91e6-0581556fddaa') +signature = DynWinRTMethodSig().add_out(DynWinRTType.hstring()) +interface = DynWinRTType.register_interface('Tests.INonAgileCallbackSource', stringable) +interface = interface.add_method('ToString', signature) +plan = DynWinRTInterfacePlan.create( + 'Tests.INonAgileCallbackSource', interface, + [DynWinRTImplementationMethod('ToString', 6, signature)], +) +factory_type = DynWinRTType.register_interface( + 'Tests.IElementFactoryCallbackLifetime', element_factory +) +factory_type = factory_type.add_method( + 'GetElement', DynWinRTMethodSig() + .add_in(DynWinRTType.object()).add_out(DynWinRTType.object()) +).add_method('RecycleElement', DynWinRTMethodSig().add_in(DynWinRTType.object())) +retained = [] +with RoApartment(1): + with projected_lifetime_scope(): + owner = DynWinRTImplementation.create( + [plan], lambda *_: [DynWinRTValue.from_hstring('alive')] + ) + source = owner.to_value() + factory = DynWinRtElementFactory.create( + stringable, lambda _args: None, retained.append + ) + receiver = factory.to_value().cast(element_factory) + factory_type.method(7).invoke(receiver, [source]) + assert len(retained) == 1 + assert not retained[0].is_released(), 'callback clone inherited an explicit scope' + assert retained[0].identity_raw() != 0 +assert retained[0].is_released(), 'non-agile callback clone escaped its native apartment' +print('nonagile-callback-copy-released', flush=True) +""" + +_UNSCOPED_RECEIVED_ARRAY = r""" +import sys +from dynwinrt import ( + DynWinRTArray, DynWinRTImplementation, DynWinRTImplementationMethod, + DynWinRTInterfacePlan, DynWinRTMethodSig, DynWinRTStruct, DynWinRTType, DynWinRTValue, + RoApartment, WinGUID, +) + +mode = sys.argv[1] +iid = WinGUID.parse('38684d40-bab3-42de-998d-26e4cce87c51') +element = DynWinRTType.object() +if mode == 'nested': + inner_type = DynWinRTType.struct_type('Tests.ReceivedInner', [element]) + element_type = DynWinRTType.struct_type('Tests.ReceivedOuter', [inner_type]) +else: + element_type = element +signature = DynWinRTMethodSig().add_out(DynWinRTType.array_type(element_type)) +interface = DynWinRTType.register_interface('Tests.IReceivedOwnerArray', iid) +interface = interface.add_method('GetItems', signature) +plan = DynWinRTInterfacePlan.create( + 'Tests.IReceivedOwnerArray', interface, + [DynWinRTImplementationMethod('GetItems', 6, signature)], +) + +with RoApartment(1): + source = DynWinRTValue.activation_factory('Windows.Foundation.Uri') + def produce(): + if mode == 'nested': + inner = DynWinRTStruct.create(inner_type) + inner.set_object(0, source) + outer = DynWinRTStruct.create(element_type) + outer.set_struct(0, inner) + values = [outer.to_value()] + else: + values = [source] + return [DynWinRTArray.from_values(values, element_type).to_value()] + owner = DynWinRTImplementation.create( + [plan], + lambda *_: produce(), + ) + receiver = owner.to_value().cast(iid) + raw = interface.method(6).invoke(receiver, []) + extracted = raw.as_array() + clone = extracted.to_value().as_array() + assert len(extracted) == len(clone) == 1 + if mode == 'nested': + nested = extracted.get(0).as_struct().get_struct(0) + assert nested.get_object(0).identity_raw() == source.identity_raw() + else: + assert extracted.get(0).identity_raw() == source.identity_raw() +assert raw.is_released() and extracted.is_released() and clone.is_released() +assert receiver.is_released() and source.is_released() +if mode == 'nested': + assert nested.is_released() +assert owner.is_closed +print('unscoped-received-com-array-safe', mode, flush=True) +""" + +_FOREIGN_RECEIVED_COM_ARRAY = r""" +import gc +import sys +import threading +import weakref +from dynwinrt import ( + DynWinRTArray, DynWinRTImplementation, DynWinRTImplementationMethod, + DynWinRTInterfacePlan, DynWinRTMethodSig, DynWinRTStruct, DynWinRTType, + DynWinRTValue, DynWinRtElementFactory, RoApartment, WinGUID, +) + +agility, shape = sys.argv[1:] +errors = [] +stringable = WinGUID.parse('96369f54-8eb6-48f0-abce-c1b211e627c3') +received_iid = WinGUID.parse('38684d40-bab3-42de-998d-26e4cce87c51') + +class Handler: + def get(self, _args): + return None + def recycle(self, _args): + return None + def dispatch(self, *_args): + return [DynWinRTValue.from_hstring('alive')] + +with RoApartment(1): + handler = Handler() + retained = weakref.ref(handler) + if agility == 'agile': + source_owner = DynWinRtElementFactory.create( + stringable, handler.get, handler.recycle + ) + else: + signature = DynWinRTMethodSig().add_out(DynWinRTType.hstring()) + string_type = DynWinRTType.register_interface( + 'Tests.INonAgileReceivedSource', stringable + ).add_method('ToString', signature) + string_plan = DynWinRTInterfacePlan.create( + 'Tests.INonAgileReceivedSource', string_type, + [DynWinRTImplementationMethod('ToString', 6, signature)], + ) + source_owner = DynWinRTImplementation.create( + [string_plan], handler.dispatch + ) + source = source_owner.to_value() + object_type = DynWinRTType.object() + if shape == 'nested': + inner_type = DynWinRTType.struct_type( + 'Tests.ForeignReceivedInner', [object_type] + ) + element = DynWinRTType.struct_type( + 'Tests.ForeignReceivedOuter', [inner_type] + ) + else: + element = object_type + + def produce(): + if shape == 'nested': + inner = DynWinRTStruct.create(inner_type) + inner.set_object(0, source) + outer = DynWinRTStruct.create(element) + outer.set_struct(0, inner) + values = [outer.to_value()] + else: + values = [source] + return [DynWinRTArray.from_values(values, element).to_value()] + + array_sig = DynWinRTMethodSig().add_out(DynWinRTType.array_type(element)) + receiver_type = DynWinRTType.register_interface( + 'Tests.IForeignReceivedArray', received_iid + ).add_method('GetItems', array_sig) + receive_plan = DynWinRTInterfacePlan.create( + 'Tests.IForeignReceivedArray', receiver_type, + [DynWinRTImplementationMethod('GetItems', 6, array_sig)], + ) + receive_owner = DynWinRTImplementation.create( + [receive_plan], lambda *_: produce() + ) + receiver = receive_owner.to_value().cast(received_iid) + raw = receiver_type.method(6).invoke(receiver, []) + extracted = raw.as_array() + assert len(extracted) == 1 + raw.release() + receiver.release() + receive_owner.release() + source.release() + if agility == 'agile': + source_owner._release_apartment_owner() + else: + source_owner.release() + del handler, source_owner, receive_owner + assert retained() is not None + handoff = [extracted] + del extracted + + def release_on_worker(): + try: + array = handoff[0] + if agility == 'agile': + item = array.get(0) + if shape == 'nested': + item = item.as_struct().get_struct(0).get_object(0) + assert item.identity_raw() != 0 + del item + else: + try: + array.get(0) + except RuntimeError as error: + assert 'owning COM apartment thread' in str(error) + else: + raise AssertionError('non-agile CoTaskMem array read on foreign thread') + del array + handoff.clear() + gc.collect() + DynWinRTType.i32_type() + gc.collect() + assert (retained() is None) == (agility == 'agile') + except BaseException as error: + errors.append(error) + + worker = threading.Thread(target=release_on_worker) + worker.start() + worker.join(10) + assert not worker.is_alive() and not errors, errors +print('foreign-received-com-array', agility, shape, flush=True) +""" + @pytest.mark.parametrize( "mode", @@ -426,6 +1439,298 @@ def test_stock_uri_checked_arrays_repeat_within_one_apartment(): assert "stock-uri-array-one-apartment" in result.stdout +@pytest.mark.parametrize("mode", ["raw", "nested", "manual", "shutdown"]) +def test_unscoped_native_carriers_release_before_last_managed_apartment_exit(mode): + result = subprocess.run( + [sys.executable, "-B", "-c", _UNSCOPED_APARTMENT, mode], + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, ( + mode, + hex(result.returncode & 0xFFFFFFFF), + result.stdout, + result.stderr, + ) + assert f"unscoped-owner-safe {mode}" in result.stdout + + +def test_final_managed_exit_does_not_consume_an_external_com_initialization(): + result = subprocess.run( + [sys.executable, "-B", "-c", _EXTERNAL_RO_INITIALIZE], + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, ( + hex(result.returncode & 0xFFFFFFFF), + result.stdout, + result.stderr, + ) + assert "external-host-initialization-preserved" in result.stdout + + +def test_earlier_managed_exit_deduplicates_an_outer_explicit_scope(): + result = subprocess.run( + [sys.executable, "-B", "-c", _INVERTED_PROJECTION_SCOPE], + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, ( + hex(result.returncode & 0xFFFFFFFF), + result.stdout, + result.stderr, + ) + assert "inverted-scope-deduplicated" in result.stdout + + +@pytest.mark.parametrize("mode", ["cleanup", "body"]) +def test_failed_unnamed_apartment_close_preserves_owner_for_same_thread_retry(mode): + result = subprocess.run( + [sys.executable, "-B", "-c", _UNSCOPED_CLOSE_FAILURE, mode], + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, ( + hex(result.returncode & 0xFFFFFFFF), + result.stdout, + result.stderr, + ) + assert f"failed-apartment-close-retried {mode}" in result.stdout + + +@pytest.mark.parametrize( + ("script", "marker"), + [ + (_REENTRANT_APARTMENT_CLOSE, "reentrant-apartment-owners-drained"), + (_UNBALANCED_MANAGED_APARTMENT, "unbalanced-managed-apartment-rejected"), + ], +) +def test_managed_apartment_drain_is_reentrant_and_unbalanced_calls_fail(script, marker): + result = subprocess.run( + [sys.executable, "-B", "-c", script], + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, ( + hex(result.returncode & 0xFFFFFFFF), + result.stdout, + result.stderr, + ) + assert marker in result.stdout + + +def test_wrong_thread_apartment_close_preserves_native_owners(): + result = subprocess.run( + [sys.executable, "-B", "-c", _WRONG_THREAD_APARTMENT], + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, ( + hex(result.returncode & 0xFFFFFFFF), + result.stdout, + result.stderr, + ) + assert "wrong-thread-close-retryable" in result.stdout + + +def test_implicit_foreign_guard_drop_never_uninitializes_the_owner_thread(): + result = subprocess.run( + [sys.executable, "-B", "-c", _FOREIGN_GUARD_FINALIZER], + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, ( + hex(result.returncode & 0xFFFFFFFF), + result.stdout, + result.stderr, + ) + assert "foreign-guard-drop-retained-apartment" in result.stdout + + +@pytest.mark.parametrize( + "mode", [ + "retained-aliases", "inflight", "inflight-implementation", + "gate-preflight", "ordered", + ] +) +def test_explicit_callback_gate_rejects_late_native_invocations(mode): + result = subprocess.run( + [sys.executable, "-B", "-c", _EXPLICIT_CALLBACK_SHUTDOWN, mode], + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, ( + mode, + hex(result.returncode & 0xFFFFFFFF), + result.stdout, + result.stderr, + ) + marker = ( + "host-ordered-shutdown-safe" + if mode == "ordered" + else f"explicit-native-callback-gate-safe {mode}" + ) + assert marker in result.stdout + + +@pytest.mark.parametrize("mode", ["delegate", "element_factory", "implementation"]) +@pytest.mark.parametrize("early", [False, True], ids=["automatic", "early"]) +def test_special_owners_release_only_their_own_native_reference(mode, early): + result = subprocess.run( + [ + sys.executable, "-B", "-c", _SPECIAL_UNSCOPED_OWNER, + mode, "early" if early else "automatic", + ], + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, ( + mode, + early, + hex(result.returncode & 0xFFFFFFFF), + result.stdout, + result.stderr, + ) + assert f"special-unscoped-owner {mode}" in result.stdout + + +@pytest.mark.parametrize("mode", ["delegate", "element_factory", "implementation"]) +def test_local_com_owners_dropped_on_foreign_thread_do_not_leak_callbacks(mode): + result = subprocess.run( + [sys.executable, "-B", "-c", _CROSS_THREAD_LOCAL_OWNER, mode], + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, ( + mode, + hex(result.returncode & 0xFFFFFFFF), + result.stdout, + result.stderr, + ) + assert f"local-owner-foreign-drop-balanced {mode}" in result.stdout + + +@pytest.mark.parametrize("mode", ["array", "struct", "nested"]) +def test_agile_com_containers_dropped_on_foreign_thread_release_own_references(mode): + result = subprocess.run( + [sys.executable, "-B", "-c", _AGILE_CONTAINER_FOREIGN_DROP, mode], + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, ( + mode, + hex(result.returncode & 0xFFFFFFFF), + result.stdout, + result.stderr, + ) + assert f"agile-container-foreign-drop-balanced {mode}" in result.stdout + + +@pytest.mark.parametrize("mode", ["array", "struct", "nested"]) +def test_nonagile_com_containers_drop_without_off_thread_native_release(mode): + result = subprocess.run( + [sys.executable, "-B", "-c", _NONAGILE_CONTAINER_FOREIGN_DROP, mode], + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, ( + mode, + hex(result.returncode & 0xFFFFFFFF), + result.stdout, + result.stderr, + ) + assert f"nonagile-container-foreign-drop-quarantined {mode}" in result.stdout + + +def test_nonagile_struct_fields_reject_cross_apartment_mutation_before_owning(): + result = subprocess.run( + [sys.executable, "-B", "-c", _FOREIGN_NONAGILE_STRUCT_MUTATION], + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, ( + hex(result.returncode & 0xFFFFFFFF), + result.stdout, + result.stderr, + ) + assert "foreign-nonagile-struct-mutation-rejected" in result.stdout + + +def test_completed_async_owner_drops_its_reference_without_implicit_cancel(): + result = subprocess.run( + [sys.executable, "-B", "-c", _UNSCOPED_ASYNC_OWNER], + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, ( + hex(result.returncode & 0xFFFFFFFF), + result.stdout, + result.stderr, + ) + assert "unscoped-async-owner-released" in result.stdout + + +def test_nonagile_callback_clone_outlives_scope_but_not_its_apartment(): + result = subprocess.run( + [sys.executable, "-B", "-c", _NONAGILE_CALLBACK_COPY], + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, ( + hex(result.returncode & 0xFFFFFFFF), + result.stdout, + result.stderr, + ) + assert "nonagile-callback-copy-released" in result.stdout + + +@pytest.mark.parametrize("mode", ["object", "nested"]) +def test_unscoped_received_com_array_and_clones_release_before_apartment_exit(mode): + result = subprocess.run( + [sys.executable, "-B", "-c", _UNSCOPED_RECEIVED_ARRAY, mode], + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, ( + hex(result.returncode & 0xFFFFFFFF), + result.stdout, + result.stderr, + ) + assert f"unscoped-received-com-array-safe {mode}" in result.stdout + + def test_checked_array_contracts_keep_valid_null_scalars_and_struct_owners(): with RoApartment(), projected_lifetime_scope(): source = DynWinRTValue.activation_factory("Windows.Foundation.Uri") @@ -463,6 +1768,26 @@ def test_checked_array_contracts_keep_valid_null_scalars_and_struct_owners(): assert not source.is_released() and source.identity_raw() == identity +@pytest.mark.parametrize("agility", ["agile", "nonagile"]) +@pytest.mark.parametrize("shape", ["object", "nested"]) +def test_received_cotaskmem_com_array_foreign_drop_preserves_native_contract(agility, shape): + result = subprocess.run( + [sys.executable, "-B", "-c", _FOREIGN_RECEIVED_COM_ARRAY, agility, shape], + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, ( + agility, + shape, + hex(result.returncode & 0xFFFFFFFF), + result.stdout, + result.stderr, + ) + assert f"foreign-received-com-array {agility} {shape}" in result.stdout + + def test_borrowed_callback_array_survives_scope_within_its_apartment(): result = subprocess.run( [sys.executable, "-B", "-c", _BORROWED_CALLBACK], diff --git a/bindings/py/tests/test_object_value_view.py b/bindings/py/tests/test_object_value_view.py index df6436d6..4f2d5a53 100644 --- a/bindings/py/tests/test_object_value_view.py +++ b/bindings/py/tests/test_object_value_view.py @@ -55,7 +55,8 @@ E_NOTIMPL = -2147467263 UTC = timezone.utc RELEASED_REASON = ( - "has been released (its projected_lifetime_scope() exited, or " + "has been released (its projected_lifetime_scope() or managed COM " + "apartment exited, or " "release_projected() / DynWinRTValue.release() was called) and can no longer " "be used." ) diff --git a/bindings/py/tests/test_phase1.py b/bindings/py/tests/test_phase1.py index 3cea9182..1a600a6c 100644 --- a/bindings/py/tests/test_phase1.py +++ b/bindings/py/tests/test_phase1.py @@ -55,6 +55,7 @@ _DynWinRTAsyncWithProgress, _dynwinrt_cache_projected, _dynwinrt_dispatch_progress, + _dynwinrt_link_cancellation, _dynwinrt_datetime_to_ticks, _dynwinrt_new_vector, _dynwinrt_projected_from_native, @@ -458,6 +459,34 @@ async def run_operation(): asyncio.run(run_operation()) +def test_nonagile_cancellation_bridge_calls_native_on_the_event_loop_thread(): + owner_thread = threading.get_ident() + cancellation_threads = [] + + class Native: + def is_released(self): + return False + def cancel(self): + cancellation_threads.append(threading.get_ident()) + + async def cancel_operation(): + loop = asyncio.get_running_loop() + raw_future = loop.create_future() + async def await_raw(): + return await raw_future + task = asyncio.create_task(await_raw()) + _dynwinrt_link_cancellation(task, raw_future, Native()) + await asyncio.sleep(0) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + await asyncio.sleep(0) + assert raw_future.cancelled() + + asyncio.run(cancel_operation()) + assert cancellation_threads == [owner_thread] + + def test_close_is_idempotent_and_prevents_future_execution(tmp_path): operation = _missing_storage_file_operation( str(tmp_path / "missing-closed-dynwinrt-file") diff --git a/tests/e2e/e2e_specs.json b/tests/e2e/e2e_specs.json index fed846f2..af554c09 100644 --- a/tests/e2e/e2e_specs.json +++ b/tests/e2e/e2e_specs.json @@ -18,7 +18,8 @@ { "kind": "property_equals", "member": "query", "expected": "?q=1" }, { "kind": "property_equals", "member": "fragment", "expected": "#frag" }, { "kind": "property_equals", "member": "port", "expected": 443 }, - { "kind": "method_result_contains", "member": "combine_uri", "args": ["sub/page"], "contains": "sub/page" } + { "kind": "method_result_contains", "member": "combine_uri", "args": ["sub/page"], "contains": "sub/page" }, + { "kind": "unscoped_uri_apartment_exit", "member": "__init__", "langs": ["py"] } ] }, { @@ -396,7 +397,8 @@ "langs": ["py"], "instantiate": { "kind": "none" }, "checks": [ - { "kind": "work_item_callback_passthrough", "member": "run_async" } + { "kind": "work_item_callback_passthrough", "member": "run_async" }, + { "kind": "threadpool_async_apartment_owner", "member": "run_async" } ] }, { @@ -623,6 +625,11 @@ "kind": "device_information_async_collection", "member": "find_all_async" }, + { + "kind": "nonagile_async_apartment_exit", + "member": "find_all_async", + "langs": ["py"] + }, { "kind": "object_value_device_properties", "member": "find_all_async", diff --git a/tests/e2e/e2e_specs.schema.json b/tests/e2e/e2e_specs.schema.json index 6e683d9b..8397e702 100644 --- a/tests/e2e/e2e_specs.schema.json +++ b/tests/e2e/e2e_specs.schema.json @@ -78,6 +78,9 @@ "cross_class_chain", "async_memory_roundtrip", "async_cancellation", + "unscoped_uri_apartment_exit", + "threadpool_async_apartment_owner", + "nonagile_async_apartment_exit", "property_set_equals", "vector_view_access", "vector_index_of", diff --git a/tests/e2e/runners/implementation_py.py b/tests/e2e/runners/implementation_py.py index eb2dc2e0..7769d596 100644 --- a/tests/e2e/runners/implementation_py.py +++ b/tests/e2e/runners/implementation_py.py @@ -1112,7 +1112,8 @@ def parse_double(self, text): RELEASED = ( - r"has been released \(its projected_lifetime_scope\(\) exited, or " + r"has been released \(its projected_lifetime_scope\(\) or managed COM " + r"apartment exited, or " r"release_projected\(\) / DynWinRTValue\.release\(\) was called\) and can no " r"longer be used\." ) diff --git a/tests/e2e/runners/py_runner.py b/tests/e2e/runners/py_runner.py index 7b807cbc..cdeb87f4 100644 --- a/tests/e2e/runners/py_runner.py +++ b/tests/e2e/runners/py_runner.py @@ -17,6 +17,7 @@ import inspect import json import re +import subprocess import sys import os import threading @@ -24,7 +25,8 @@ _WINRT_UINT_SUFFIXES = {'int8', 'int16', 'int32', 'int64'} _RELEASED_REASON = ( - 'has been released (its projected_lifetime_scope() exited, or ' + 'has been released (its projected_lifetime_scope() or managed COM ' + 'apartment exited, or ' 'release_projected() / DynWinRTValue.release() was called) and ' 'can no longer be used.' ) @@ -123,6 +125,149 @@ def wrap_arg(val): return val +def run_isolated_apartment_check(generated_dir, package, script, marker, *args): + env = os.environ.copy() + parent = os.path.dirname(os.path.abspath(generated_dir)) + env['PYTHONPATH'] = os.pathsep.join(filter(None, (parent, env.get('PYTHONPATH')))) + env['DYNWINRT_E2E_PACKAGE_PARENT'] = parent + child = subprocess.run( + [ + sys.executable, '-B', '-c', + 'import os, sys\n' + 'sys.path.insert(0, os.environ["DYNWINRT_E2E_PACKAGE_PARENT"])\n' + + script, + package, *args, + ], + capture_output=True, + text=True, + timeout=50, + env=env, + check=False, + ) + if child.returncode: + return ( + f'{marker}: isolated Python exited {hex(child.returncode & 0xFFFFFFFF)}: ' + f'{child.stdout} {child.stderr}' + ) + if marker not in child.stdout: + return f'{marker}: isolated Python omitted its success marker: {child.stdout}' + return None + + +_UNSCOPED_URI_APARTMENT = r""" +import importlib +import sys +from dynwinrt import RoApartment + +Uri = importlib.import_module(f'{sys.argv[1]}.windows.foundation').Uri +mode = sys.argv[2] +with RoApartment(1): + live = Uri('https://example.com/c') + assert live.host == 'example.com' +assert live._obj.is_released() +try: + live.host +except RuntimeError as error: + assert 'released' in str(error) +else: + raise AssertionError('unscoped Uri remained callable after apartment exit') +if mode == 'del': + del live + print('unscoped-uri-del', flush=True) +else: + print('unscoped-uri-shutdown', flush=True) +""" + +_NONAGILE_ASYNC_APARTMENT = r""" +import asyncio +import importlib +import sys +from dynwinrt import RoApartment + +DeviceInformation = importlib.import_module( + f'{sys.argv[1]}.windows.devices.enumeration' +).DeviceInformation + +async def query(): + with RoApartment(1): + operation = DeviceInformation.find_all_async() + devices = await operation + assert isinstance(devices.size, int) + assert devices._obj.is_released() + try: + devices.size + except RuntimeError as error: + assert 'released' in str(error) + else: + raise AssertionError('non-agile async result outlived its apartment') + +asyncio.run(query()) +print('nonagile-async-owner-thread', flush=True) +""" + +_THREADPOOL_ASYNC_APARTMENT = r""" +import asyncio +import importlib +import sys +import threading +from dynwinrt import RoApartment + +ThreadPool = importlib.import_module(f'{sys.argv[1]}.windows.system.threading').ThreadPool +mode = sys.argv[2] +started = threading.Event() +release = threading.Event() +finished = threading.Event() + +def work(_action): + started.set() + try: + assert release.wait(8), 'work item was not unblocked' + finally: + finished.set() + +async def pending(): + with RoApartment(1) as apartment: + operation = ThreadPool.run_async(work) + task = asyncio.create_task(operation) + assert await asyncio.to_thread(started.wait, 5) + await asyncio.sleep(0) + assert not task.done() + try: + apartment.close() + except RuntimeError as error: + assert 'future is pending' in str(error) + else: + raise AssertionError('a pending async operation closed its apartment') + release.set() + await task + assert not task.cancelled() and finished.is_set() + apartment.close() + try: + operation.wait() + except RuntimeError as error: + assert 'released' in str(error) + else: + raise AssertionError('async owner outlived its apartment') + +if mode == 'pending': + asyncio.run(pending()) + print('pending-async-retry', flush=True) +else: + with RoApartment(1): + operation = ThreadPool.run_async(work) + assert started.wait(5) + release.set() + assert finished.wait(5), 'agile work was cancelled on apartment exit' + try: + operation.wait() + except RuntimeError as error: + assert 'released' in str(error) + else: + raise AssertionError('async owner outlived its apartment') + print('agile-pending-work', flush=True) +""" + + async def run_spec(spec: dict, generated_dir: str, pkg_name: str) -> dict: """Run a single test spec. Returns a result dict.""" ns = spec['namespace'] @@ -2996,6 +3141,42 @@ def block_on_sta(): else: cr['pass'] = True + elif kind == 'unscoped_uri_apartment_exit': + for mode in ('del', 'shutdown'): + marker = f'unscoped-uri-{mode}' + error = run_isolated_apartment_check( + generated_dir, pkg_name, _UNSCOPED_URI_APARTMENT, marker, mode + ) + if error is not None: + cr['error'] = error + return cr + cr['pass'] = True + + elif kind == 'threadpool_async_apartment_owner': + for mode, marker in ( + ('pending', 'pending-async-retry'), + ('agile', 'agile-pending-work'), + ): + error = run_isolated_apartment_check( + generated_dir, pkg_name, _THREADPOOL_ASYNC_APARTMENT, marker, mode + ) + if error is not None: + cr['error'] = error + return cr + cr['pass'] = True + + elif kind == 'nonagile_async_apartment_exit': + error = run_isolated_apartment_check( + generated_dir, + pkg_name, + _NONAGILE_ASYNC_APARTMENT, + 'nonagile-async-owner-thread', + ) + if error is None: + cr['pass'] = True + else: + cr['error'] = error + elif kind == 'device_information_async_collection': devices = await getattr(cls, member)() if devices is None or not isinstance(devices.size, int): diff --git a/tools/dynwinrt-codegen/tests/python_raw_lifetime_test.rs b/tools/dynwinrt-codegen/tests/python_raw_lifetime_test.rs index c1ce0c75..e3c4044a 100644 --- a/tools/dynwinrt-codegen/tests/python_raw_lifetime_test.rs +++ b/tools/dynwinrt-codegen/tests/python_raw_lifetime_test.rs @@ -15,6 +15,10 @@ struct Generated { impl Generated { fn new() -> Option { + Self::for_class("Windows.Foundation.PropertyValue", "raw_lifetime") + } + + fn for_class(class_name: &str, prefix: &str) -> Option { if !Path::new(WINDOWS_WINMD).is_file() { eprintln!("Skipping raw lifetime regression: Windows.winmd not found"); return None; @@ -24,7 +28,7 @@ impl Generated { .unwrap() .parent() .unwrap(); - let package = format!("raw_lifetime_{}", std::process::id()); + let package = format!("{prefix}_{}", std::process::id()); let root = repo.join("target").join(&package); let output = Command::new(env!("CARGO_BIN_EXE_dynwinrt-codegen")) .args([ @@ -32,7 +36,7 @@ impl Generated { "--winmd", WINDOWS_WINMD, "--class-name", - "Windows.Foundation.PropertyValue", + class_name, "--lang", "py", "--output", @@ -79,6 +83,16 @@ impl Generated { ); assert!(stdout.contains(scenario), "{scenario}:\n{stdout}\n{stderr}"); } + + fn binding_available(&self) -> bool { + Command::new(self.python()) + .args([ + "-c", + "from dynwinrt import DynWinRTImplementationHandle, RoApartment", + ]) + .output() + .is_ok_and(|output| output.status.success()) + } } impl Drop for Generated { @@ -92,13 +106,7 @@ fn generated_raw_outputs_release_before_apartment_exit_even_when_they_escape() { let Some(generated) = Generated::new() else { return; }; - let available = Command::new(generated.python()) - .args([ - "-c", - "from dynwinrt import DynWinRTImplementationHandle, RoApartment", - ]) - .output() - .is_ok_and(|output| output.status.success()); + let available = generated.binding_available(); assert!( available || std::env::var("DYNWINRT_REQUIRE_IMPLEMENTATION_RUNTIME").as_deref() != Ok("1"), "raw lifetime regression requires the matching Python binding" @@ -276,3 +284,263 @@ print('foreign-scope-rejected', flush=True) "#, ); } + +#[test] +fn unscoped_generated_uri_cannot_outlive_its_managed_apartment() { + let Some(generated) = Generated::for_class("Windows.Foundation.Uri", "uri_lifetime") else { + return; + }; + let available = generated.binding_available(); + assert!( + available || std::env::var("DYNWINRT_REQUIRE_IMPLEMENTATION_RUNTIME").as_deref() != Ok("1"), + "generated Uri lifetime regression requires the matching Python binding" + ); + if !available { + eprintln!("Skipping generated Uri lifetime regression: Python binding not installed"); + return; + } + + generated.run( + "unscoped-uri-del", + r#" +from dynwinrt import RoApartment +from PY_PACKAGE.windows.foundation import Uri +with RoApartment(1): + live = Uri('https://example.com/c') + assert live.host == 'example.com' +assert live._obj.is_released() +try: + live.host +except RuntimeError as error: + assert 'released' in str(error) +else: + raise AssertionError('unscoped Uri remained callable after apartment exit') +del live +print('unscoped-uri-del', flush=True) +"#, + ); + generated.run( + "unscoped-uri-shutdown", + r#" +from dynwinrt import RoApartment +from PY_PACKAGE.windows.foundation import Uri +with RoApartment(1): + live = Uri('https://example.com/c') + assert live.host == 'example.com' +assert live._obj.is_released() +print('unscoped-uri-shutdown', flush=True) +# Keep live through interpreter shutdown, without an explicit lifetime scope. +"#, + ); + generated.run( + "sequential-apartment-statics", + r#" +import threading +from dynwinrt import RoApartment +from PY_PACKAGE.windows.foundation import Uri + +errors = [] +def use_uri(index): + try: + with RoApartment(1): + factory = Uri._get_s_IUriEscapeStatics() + assert Uri.escape_component('hello world') == 'hello%20world' + uri = Uri(f'https://example.com/{index}') + assert uri.host == 'example.com' + assert factory.is_released() and uri._obj.is_released() + except BaseException as error: + errors.append(error) + +for index in range(3): + worker = threading.Thread(target=use_uri, args=(index,)) + worker.start() + worker.join(10) + assert not worker.is_alive() +if errors: + raise errors[0] +print('sequential-apartment-statics', flush=True) +"#, + ); +} + +#[test] +fn nonagile_generated_async_completes_before_unscoped_apartment_teardown() { + let Some(generated) = Generated::for_class( + "Windows.Devices.Enumeration.DeviceInformation", + "device_lifetime", + ) else { + return; + }; + if !generated.binding_available() { + assert_ne!( + std::env::var("DYNWINRT_REQUIRE_IMPLEMENTATION_RUNTIME").as_deref(), + Ok("1"), + "non-agile async lifetime regression requires the matching Python binding" + ); + return; + } + generated.run( + "nonagile-async-owner-thread", + r#" +import asyncio +from dynwinrt import RoApartment +from PY_PACKAGE.windows.devices.enumeration import DeviceInformation + +async def query(): + with RoApartment(1): + operation = DeviceInformation.find_all_async() + devices = await operation + assert isinstance(devices.size, int) + assert devices._obj.is_released() + try: + devices.size + except RuntimeError as error: + assert 'released' in str(error) + else: + raise AssertionError('non-agile result outlived its COM apartment') + +asyncio.run(query()) +print('nonagile-async-owner-thread', flush=True) +"#, + ); +} + +#[test] +fn generated_threadpool_async_close_retries_without_cancelling_work() { + let Some(generated) = + Generated::for_class("Windows.System.Threading.ThreadPool", "threadpool_lifetime") + else { + return; + }; + if !generated.binding_available() { + assert_ne!( + std::env::var("DYNWINRT_REQUIRE_IMPLEMENTATION_RUNTIME").as_deref(), + Ok("1"), + "ThreadPool apartment lifetime regression requires the matching Python binding" + ); + return; + } + generated.run( + "pending-async-retry", + r#" +import asyncio +import threading +from dynwinrt import RoApartment +from PY_PACKAGE.windows.system.threading import ThreadPool + +started = threading.Event() +release = threading.Event() +finished = threading.Event() + +def work(_action): + started.set() + try: + assert release.wait(8), 'work item was not unblocked' + finally: + finished.set() + +async def run(): + with RoApartment(1) as apartment: + operation = ThreadPool.run_async(work) + task = asyncio.create_task(operation) + assert await asyncio.to_thread(started.wait, 5) + await asyncio.sleep(0) + assert not task.done() + try: + apartment.close() + except RuntimeError as error: + assert 'future is pending' in str(error) + else: + raise AssertionError('pending async operation closed its apartment') + release.set() + await task + assert not task.cancelled() and finished.is_set() + apartment.close() + try: + operation.wait() + except RuntimeError as error: + assert 'released' in str(error) + else: + raise AssertionError('async operation outlived its apartment') + +asyncio.run(run()) +print('pending-async-retry', flush=True) +"#, + ); + generated.run( + "agile-pending-work", + r#" +import threading +from dynwinrt import RoApartment +from PY_PACKAGE.windows.system.threading import ThreadPool + +started = threading.Event() +release = threading.Event() +finished = threading.Event() + +def work(_action): + started.set() + try: + assert release.wait(8), 'work item was not unblocked' + finally: + finished.set() + +with RoApartment(1): + operation = ThreadPool.run_async(work) + assert started.wait(5) +release.set() +assert finished.wait(5), 'agile work was cancelled when its Python owner exited' +try: + operation.wait() +except RuntimeError as error: + assert 'released' in str(error) +else: + raise AssertionError('async owner outlived its apartment') +print('agile-pending-work', flush=True) +"#, + ); + generated.run( + "scoped-pending-async-retry", + r#" +import asyncio +import threading +from dynwinrt import RoApartment, projected_lifetime_scope +from PY_PACKAGE.windows.system.threading import ThreadPool + +started = threading.Event() +release = threading.Event() + +def work(_action): + started.set() + assert release.wait(8), 'work item was not unblocked' + +async def run(): + with RoApartment(1), projected_lifetime_scope() as scope: + operation = ThreadPool.run_async(work) + task = asyncio.create_task(operation) + assert await asyncio.to_thread(started.wait, 5) + await asyncio.sleep(0) + assert not task.done() + try: + scope.close() + except RuntimeError as error: + assert 'future is pending' in str(error) + else: + raise AssertionError('scope disposed a pending async owner') + assert not task.cancelled() + release.set() + await task + scope.close() + assert scope.disposed and not task.cancelled() + try: + operation.wait() + except RuntimeError as error: + assert 'released' in str(error) + else: + raise AssertionError('async owner remained live after its scope') + +asyncio.run(run()) +print('scoped-pending-async-retry', flush=True) +"#, + ); +} diff --git a/tools/dynwinrt-codegen/tests/python_released_implementation_test.rs b/tools/dynwinrt-codegen/tests/python_released_implementation_test.rs index 3152bd66..f6e2700b 100644 --- a/tools/dynwinrt-codegen/tests/python_released_implementation_test.rs +++ b/tools/dynwinrt-codegen/tests/python_released_implementation_test.rs @@ -164,14 +164,16 @@ fn generated_implementation_results_reject_released_references() { format!( r#" import importlib +import os import sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import dynwinrt as dw ISource = importlib.import_module("pyviews.{source_module}").ISource Holder = importlib.import_module("pyviews.{holder_module}").Holder PYTHON_EXCEPTION = -1594998779 RELEASED = ( - "has been released (its projected_lifetime_scope() exited, or release_projected() / " + "has been released (its projected_lifetime_scope() or managed COM apartment exited, or release_projected() / " "DynWinRTValue.release() was called) and can no longer be used." ) state = {{"item": None, "pair": (None, None)}} From 0af18a31d89a34b3ed5e12c74f9127e3bc1dd73d Mon Sep 17 00:00:00 2001 From: Leilei Zhang Date: Wed, 30 Sep 2026 17:26:46 +0800 Subject: [PATCH 14/15] Make Python finalization fixtures self-contained Initialize the real binding module in both isolated Rust finalization children. Coverage can now exercise native callback owner shutdown without an installed Python wheel, using the same package bootstrap as the embedded-host gate test. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- bindings/py/src/runtime.rs | 77 ++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 36 deletions(-) diff --git a/bindings/py/src/runtime.rs b/bindings/py/src/runtime.rs index 21ebf6ff..b7a55374 100644 --- a/bindings/py/src/runtime.rs +++ b/bindings/py/src/runtime.rs @@ -3456,6 +3456,45 @@ mod tests { use std::ffi::c_void; use std::sync::atomic::{AtomicU32, Ordering}; + fn initialize_embedded_binding(py: Python<'_>) { + let package_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("python") + .join("dynwinrt"); + let package = PyModule::new(py, "dynwinrt").unwrap(); + package + .setattr("__path__", vec![package_path.to_string_lossy().to_string()]) + .unwrap(); + package.setattr("__package__", "dynwinrt").unwrap(); + let native = PyModule::new(py, "dynwinrt.dynwinrt").unwrap(); + native.setattr("__package__", "dynwinrt").unwrap(); + let spec = py + .import("importlib.machinery") + .unwrap() + .getattr("ModuleSpec") + .unwrap() + .call1(("dynwinrt.dynwinrt", py.None())) + .unwrap(); + spec.setattr( + "origin", + package_path + .join("dynwinrt.pyd") + .to_string_lossy() + .to_string(), + ) + .unwrap(); + native.setattr("__spec__", spec).unwrap(); + let modules = py.import("sys").unwrap().getattr("modules").unwrap(); + modules.set_item("dynwinrt", &package).unwrap(); + modules.set_item("dynwinrt.dynwinrt", &native).unwrap(); + package.setattr("dynwinrt", &native).unwrap(); + crate::dynwinrt::init(&native).unwrap(); + + let source = std::fs::read_to_string(package_path.join("__init__.py")).unwrap(); + let source = std::ffi::CString::new(source).unwrap(); + py.run(source.as_c_str(), Some(&package.dict()), None) + .unwrap(); + } + #[test] fn guarded_python_native_containers_are_send_and_sync() { fn assert_send_sync() {} @@ -4042,6 +4081,7 @@ mod tests { } Python::initialize(); + Python::attach(initialize_embedded_binding); let mut apartment = RoApartment::new(Some(1)); apartment.initialize().unwrap(); let (source, counts) = QueryProbe::new(); @@ -4259,42 +4299,7 @@ mod tests { let owner_thread = thread::current().id(); unsafe { RoInitialize(RO_INIT_MULTITHREADED) }.unwrap(); let (aliases, attempts_after_gate) = Python::attach(|py| { - let package_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) - .join("python") - .join("dynwinrt"); - let package = PyModule::new(py, "dynwinrt").unwrap(); - package - .setattr("__path__", vec![package_path.to_string_lossy().to_string()]) - .unwrap(); - package.setattr("__package__", "dynwinrt").unwrap(); - let native = PyModule::new(py, "dynwinrt.dynwinrt").unwrap(); - native.setattr("__package__", "dynwinrt").unwrap(); - let spec = py - .import("importlib.machinery") - .unwrap() - .getattr("ModuleSpec") - .unwrap() - .call1(("dynwinrt.dynwinrt", py.None())) - .unwrap(); - spec.setattr( - "origin", - package_path - .join("dynwinrt.pyd") - .to_string_lossy() - .to_string(), - ) - .unwrap(); - native.setattr("__spec__", spec).unwrap(); - let modules = py.import("sys").unwrap().getattr("modules").unwrap(); - modules.set_item("dynwinrt", &package).unwrap(); - modules.set_item("dynwinrt.dynwinrt", &native).unwrap(); - package.setattr("dynwinrt", &native).unwrap(); - crate::dynwinrt::init(&native).unwrap(); - - let source = std::fs::read_to_string(package_path.join("__init__.py")).unwrap(); - let source = std::ffi::CString::new(source).unwrap(); - py.run(source.as_c_str(), Some(&package.dict()), None) - .unwrap(); + initialize_embedded_binding(py); let locals = PyDict::new(py); let script = std::ffi::CString::new( From 41f67c56578f75dd02c85c1bb32a81c5530aa69e Mon Sep 17 00:00:00 2001 From: Leilei Zhang Date: Wed, 30 Sep 2026 18:05:33 +0800 Subject: [PATCH 15/15] Keep Python struct agility correct after failed writes Reclassify the actual object and nested-struct fields after native setters return, including errors, without masking the original failure. Cover failed agile writes, partial writes, owner-thread balance and successful non-agile transitions on both architectures; document the separate pre-existing reentrant apartment-close limitation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- bindings/py/README.md | 7 + bindings/py/src/runtime.rs | 127 ++++++++++- .../tests/test_native_container_lifetime.py | 208 ++++++++++++++++++ 3 files changed, 330 insertions(+), 12 deletions(-) diff --git a/bindings/py/README.md b/bindings/py/README.md index 95128ad6..2113b1a7 100644 --- a/bindings/py/README.md +++ b/bindings/py/README.md @@ -843,6 +843,13 @@ apartment state. Only an implicit wrong-thread finalizer or interpreter shutdown without a usable GIL can force a diagnostic and retain unsafe native references until process exit; normal close never treats a leak as success. +Reentrant apartment teardown is a separate limitation: do not call +`RoApartment.close()` inside a native event callback that is still dispatching +on that apartment. Closing inside `PropertySet.MapChanged` has crashed with +`0xC0000005` on both the original base and this branch. Let the callback +return before closing its apartment on the owning thread. The owner-after-exit +protection above does not make teardown during an active callback safe. + ### Embedded host callback shutdown If an embedded host retains native aliases to Python-backed delegates, diff --git a/bindings/py/src/runtime.rs b/bindings/py/src/runtime.rs index b7a55374..163b0027 100644 --- a/bindings/py/src/runtime.rs +++ b/bindings/py/src/runtime.rs @@ -2833,6 +2833,27 @@ impl DynWinRTStruct { .as_mut() .ok_or_else(|| released_native_container_error("DynWinRTStruct")) } + + fn complete_field_mutation(&mut self, py: Python<'_>, mutation: PyResult<()>) -> PyResult<()> { + let agility = self + .0 + .as_ref() + .ok_or_else(|| released_native_container_error("DynWinRTStruct")) + .and_then(native_struct_is_agile); + // Derive foreign-thread eligibility from the payload even if a setter failed. + self.2 = match &agility { + Ok(agile) => *agile, + Err(_) => false, + }; + match (mutation, agility) { + (Ok(()), Ok(_)) => Ok(()), + (Err(error), Ok(_)) | (Ok(()), Err(error)) => Err(error), + (Err(error), Err(agility_error)) => { + error.set_cause(py, Some(agility_error)); + Err(error) + } + } + } } #[pymethods] @@ -3045,7 +3066,7 @@ impl DynWinRTStruct { tracked_native_struct(py, data) } - fn set_struct(&mut self, index: i64, value: &DynWinRTStruct) -> PyResult<()> { + fn set_struct(&mut self, py: Python<'_>, index: i64, value: &DynWinRTStruct) -> PyResult<()> { let index = checked_index(index)?; self.data()?; let nested = value.data()?; @@ -3056,12 +3077,11 @@ impl DynWinRTStruct { "cannot store non-agile COM fields in a struct from another apartment thread", )); } - self.2 = false; - self.data_mut()? + let mutation = self + .data_mut()? .set_field_struct_checked(index, nested) - .map_err(map_dynwinrt_error)?; - self.2 = native_struct_is_agile(self.data()?)?; - Ok(()) + .map_err(map_dynwinrt_error); + self.complete_field_mutation(py, mutation) } fn get_object(&self, py: Python<'_>, index: i64) -> PyResult> { @@ -3077,7 +3097,7 @@ impl DynWinRTStruct { tracked_native_value(py, value) } - fn set_object(&mut self, index: i64, value: &DynWinRTValue) -> PyResult<()> { + fn set_object(&mut self, py: Python<'_>, index: i64, value: &DynWinRTValue) -> PyResult<()> { let index = checked_index(index)?; self.data()?; value.check_input("DynWinRTStruct.set_object()", InputSlot::Field(index))?; @@ -3098,12 +3118,11 @@ impl DynWinRTStruct { "cannot store a non-agile COM field from another apartment thread", )); } - self.2 = false; - self.data_mut()? + let mutation = self + .data_mut()? .set_field_object(index, object) - .map_err(map_dynwinrt_error)?; - self.2 = native_struct_is_agile(self.data()?)?; - Ok(()) + .map_err(map_dynwinrt_error); + self.complete_field_mutation(py, mutation) } /// Wrap as DynWinRTValue::Struct for passing to call(). @@ -3692,6 +3711,90 @@ mod tests { assert_eq!(array.to_i32_list().unwrap(), vec![0, 0x80004005u32 as i32]); } + #[test] + fn failed_field_mutation_reclassifies_partially_written_com_fields() { + Python::initialize(); + for nested in [false, true] { + let (source, counts) = QueryProbe::new(); + let field_type = TABLE.interface(QueryProbe::SUPPORTED); + let mut record = if nested { + let inner_type = + TABLE.struct_type("Tests.PartialAgilityInner", &[field_type.clone()]); + let outer_type = + TABLE.struct_type("Tests.PartialAgilityOuter", &[inner_type.clone()]); + let mut inner = inner_type.default_value(); + inner.set_field_object(0, Some(&source)).unwrap(); + let mut outer = DynWinRTStruct( + Some(outer_type.default_value()), + Some(thread::current().id()), + true, + ); + outer + .0 + .as_mut() + .unwrap() + .set_field_struct_checked(0, &inner) + .unwrap(); + outer + } else { + let struct_type = TABLE.struct_type("Tests.PartialAgilityObject", &[field_type]); + let mut direct = DynWinRTStruct( + Some(struct_type.default_value()), + Some(thread::current().id()), + true, + ); + direct + .0 + .as_mut() + .unwrap() + .set_field_object(0, Some(&source)) + .unwrap(); + direct + }; + + Python::attach(|py| { + let error = record + .complete_field_mutation( + py, + Err(PyIndexError::new_err("native setter failed after writing")), + ) + .unwrap_err(); + assert!(error.is_instance_of::(py)); + assert!(error.to_string().contains("failed after writing")); + }); + assert!(!record.2, "the partially written COM field is non-agile"); + let mut record = thread::spawn(move || { + assert!(record.release().is_err()); + assert!(!record.is_released()); + record + }) + .join() + .unwrap(); + assert_eq!(counts.wrong_thread_addrefs.load(Ordering::SeqCst), 0); + assert_eq!(counts.wrong_thread_releases.load(Ordering::SeqCst), 0); + let actual = if nested { + record + .data() + .unwrap() + .get_field_struct_checked(0) + .unwrap() + .get_field_object(0) + .unwrap() + .unwrap() + } else { + record.data().unwrap().get_field_object(0).unwrap().unwrap() + }; + assert_eq!(actual.as_raw(), source.as_raw()); + drop(actual); + record.release().unwrap(); + drop(source); + assert_eq!( + counts.releases.load(Ordering::SeqCst), + counts.addrefs.load(Ordering::SeqCst) + 1 + ); + } + } + #[test] fn private_query_guard_releases_successful_qi_and_preserves_other_failures() { Python::initialize(); diff --git a/bindings/py/tests/test_native_container_lifetime.py b/bindings/py/tests/test_native_container_lifetime.py index 5400d052..506b2ac3 100644 --- a/bindings/py/tests/test_native_container_lifetime.py +++ b/bindings/py/tests/test_native_container_lifetime.py @@ -1075,6 +1075,178 @@ def worker(): print('foreign-nonagile-struct-mutation-rejected', flush=True) """ +_FAILED_AGILE_STRUCT_SETTER = r""" +import gc +import sys +import threading +import weakref +from dynwinrt import ( + DynWinRTStruct, DynWinRTType, DynWinRtElementFactory, RoApartment, WinGUID, +) + +mode = sys.argv[1] +disposed = [] +errors = [] + +class Handler: + def get(self, _args): + return None + def recycle(self, _args): + return None + def __del__(self): + disposed.append(threading.get_ident()) + +with RoApartment(1): + handler = Handler() + retained = weakref.ref(handler) + owner = DynWinRtElementFactory.create( + WinGUID.parse('96369f54-8eb6-48f0-abce-c1b211e627c3'), + handler.get, handler.recycle, + ) + source = owner.to_value() + inner_type = DynWinRTType.struct_type( + 'Tests.FailedAgileInner', [DynWinRTType.object()] + ) + if mode == 'object': + record = DynWinRTStruct.create(inner_type) + record.set_object(0, source) + def identity(): + return record.get_object(0).identity_raw() + def invalid_setter(): + record.set_object(100, source) + else: + inner = DynWinRTStruct.create(inner_type) + inner.set_object(0, source) + outer_type = DynWinRTType.struct_type( + 'Tests.FailedAgileOuter', [inner_type] + ) + record = DynWinRTStruct.create(outer_type) + record.set_struct(0, inner) + def identity(): + return record.get_struct(0).get_object(0).identity_raw() + def invalid_setter(): + record.set_struct(100, inner) + + original = identity() + try: + invalid_setter() + except IndexError: + pass + else: + raise AssertionError('invalid field index was accepted') + assert identity() == original == source.identity_raw() + if mode == 'struct': + inner.release() + owner._release_apartment_owner() + source.release() + del owner, source, handler + assert retained() is not None + + def release_foreign(): + try: + with RoApartment(1): + assert identity() == original + record.release() + assert record.is_released() + gc.collect() + assert retained() is None, 'agile struct leaked its own COM reference' + except BaseException as error: + errors.append(error) + + worker = threading.Thread(target=release_foreign) + worker.start() + worker.join(10) + assert not worker.is_alive() and not errors, errors +assert len(disposed) == 1 +print('failed-agile-struct-setter-balanced', mode, flush=True) +""" + +_VALID_NONAGILE_STRUCT_SETTER = r""" +import gc +import sys +import threading +import weakref +from dynwinrt import ( + DynWinRTImplementation, DynWinRTImplementationMethod, DynWinRTInterfacePlan, + DynWinRTMethodSig, DynWinRTStruct, DynWinRTType, DynWinRTValue, + RoApartment, WinGUID, +) + +mode = sys.argv[1] +disposed = [] +errors = [] + +class Handler: + def dispatch(self, *_args): + return [DynWinRTValue.from_hstring('non-agile')] + def __del__(self): + disposed.append(threading.get_ident()) + +with RoApartment(1): + handler = Handler() + retained = weakref.ref(handler) + iid = WinGUID.parse('96369f54-8eb6-48f0-abce-c1b211e627c3') + signature = DynWinRTMethodSig().add_out(DynWinRTType.hstring()) + interface = DynWinRTType.register_interface( + 'Tests.IValidNonAgileSetter', iid + ).add_method('ToString', signature) + plan = DynWinRTInterfacePlan.create( + 'Tests.IValidNonAgileSetter', interface, + [DynWinRTImplementationMethod('ToString', 6, signature)], + ) + owner = DynWinRTImplementation.create([plan], handler.dispatch) + source = owner.to_value() + inner_type = DynWinRTType.struct_type( + 'Tests.ValidNonAgileInner', [DynWinRTType.object()] + ) + if mode == 'object': + record = DynWinRTStruct.create(inner_type) + record.set_object(0, source) + def identity(): + return record.get_object(0).identity_raw() + else: + inner = DynWinRTStruct.create(inner_type) + inner.set_object(0, source) + outer_type = DynWinRTType.struct_type( + 'Tests.ValidNonAgileOuter', [inner_type] + ) + record = DynWinRTStruct.create(outer_type) + record.set_struct(0, inner) + def identity(): + return record.get_struct(0).get_object(0).identity_raw() + + original = identity() + assert original == source.identity_raw() + def release_foreign(): + try: + with RoApartment(1): + try: + record.release() + except RuntimeError as error: + assert 'owning COM apartment thread' in str(error), error + else: + raise AssertionError('non-agile struct released on foreign thread') + assert not record.is_released() + except BaseException as error: + errors.append(error) + + worker = threading.Thread(target=release_foreign) + worker.start() + worker.join(10) + assert not worker.is_alive() and not errors, errors + assert identity() == original + record.release() + if mode == 'struct': + inner.release() + source.release() + owner.release() + del handler, source, owner + gc.collect() + assert retained() is None, 'non-agile owner was not released on its apartment' +assert len(disposed) == 1 +print('valid-nonagile-struct-setter-guarded', mode, flush=True) +""" + _UNSCOPED_ASYNC_OWNER = r""" from pathlib import Path from tempfile import TemporaryDirectory @@ -1682,6 +1854,42 @@ def test_nonagile_struct_fields_reject_cross_apartment_mutation_before_owning(): assert "foreign-nonagile-struct-mutation-rejected" in result.stdout +@pytest.mark.parametrize("mode", ["object", "struct"]) +def test_failed_agile_struct_setters_preserve_foreign_release_and_refs(mode): + result = subprocess.run( + [sys.executable, "-B", "-c", _FAILED_AGILE_STRUCT_SETTER, mode], + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, ( + mode, + hex(result.returncode & 0xFFFFFFFF), + result.stdout, + result.stderr, + ) + assert f"failed-agile-struct-setter-balanced {mode}" in result.stdout + + +@pytest.mark.parametrize("mode", ["object", "struct"]) +def test_valid_nonagile_struct_setters_reject_foreign_release(mode): + result = subprocess.run( + [sys.executable, "-B", "-c", _VALID_NONAGILE_STRUCT_SETTER, mode], + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert result.returncode == 0, ( + mode, + hex(result.returncode & 0xFFFFFFFF), + result.stdout, + result.stderr, + ) + assert f"valid-nonagile-struct-setter-guarded {mode}" in result.stdout + + def test_completed_async_owner_drops_its_reference_without_implicit_cancel(): result = subprocess.run( [sys.executable, "-B", "-c", _UNSCOPED_ASYNC_OWNER],