Skip to content

Commit d2a3e7f

Browse files
fix(serialization): support set and frozenset generic containers in JsonDataConverter (#310)
* fix(serialization): support set and frozenset generic containers in JsonDataConverter * fix(serialization): deterministic set ordering, opt-in set reconstruction, and pyright typing - Preserve replay compatibility by gating generic set reconstruction behind enable_generic_sets=False default on JsonDataConverter - Ensure deterministic set serialization with fallback sort key on recursive encoded values, qualname, and repr - Retain primitive set element annotations in type discovery when set reconstruction is enabled - Resolve strict pyright errors and update tests * fix(serialization): stabilize nested set ordering * fix(serialization): simplify set serialization by emitting sorted canonical JSON strings
1 parent 6a56be2 commit d2a3e7f

4 files changed

Lines changed: 427 additions & 21 deletions

File tree

‎durabletask/serialization.py‎

Lines changed: 100 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,8 @@
4242
import types
4343
import typing
4444
from abc import ABC, abstractmethod
45-
from collections.abc import Mapping, Sequence
45+
from collections.abc import Iterable, Mapping, Sequence
46+
from collections.abc import Set as AbstractSet
4647
from types import SimpleNamespace
4748
from typing import Any, cast
4849

@@ -68,7 +69,6 @@ class DataConverter(ABC):
6869
@abstractmethod
6970
def serialize(self, value: Any) -> str | None:
7071
"""Serialize ``value`` to a string, or ``None`` when ``value`` is ``None``."""
71-
pass
7272

7373
@abstractmethod
7474
def deserialize(self, data: str | None, target_type: type | None = None) -> Any:
@@ -85,7 +85,6 @@ def deserialize(self, data: str | None, target_type: type | None = None) -> Any:
8585
:class:`JsonDataConverter` is best-effort and falls back; a validating
8686
converter may instead raise.
8787
"""
88-
pass
8988

9089
@abstractmethod
9190
def coerce(self, value: Any, target_type: type | None = None) -> Any:
@@ -99,7 +98,6 @@ def coerce(self, value: Any, target_type: type | None = None) -> Any:
9998
(strict vs. best-effort) that an implementation applies in
10099
:meth:`deserialize` should apply here.
101100
"""
102-
pass
103101

104102
def can_reconstruct(self, target_type: Any) -> bool:
105103
"""Return True if this converter can rebuild ``target_type`` from a payload.
@@ -136,7 +134,9 @@ class JsonDataConverter(DataConverter):
136134
``SimpleNamespace`` handling, and nested values are encoded recursively, so
137135
a dataclass (or any object) with a ``to_json`` hook -- including one nested
138136
inside another dataclass, ``list``, or ``dict`` -- round-trips through its
139-
own hooks.
137+
own hooks. Sets and frozensets serialize to JSON arrays ordered
138+
deterministically by their canonical JSON representation (using lexical
139+
string order, so numeric sets like ``{2, 10}`` serialize as ``[10, 2]``).
140140
141141
Deserialization (and value-level :meth:`coerce`) is **best-effort**: when a
142142
``target_type`` is supplied and the value cannot be coerced to it, the raw
@@ -167,6 +167,18 @@ class JsonDataConverter(DataConverter):
167167
> payload; the destination type is always supplied by the caller.
168168
"""
169169

170+
def __init__(
171+
self,
172+
*,
173+
enable_generic_sets: bool = False,
174+
enable_set_coercion: bool = False,
175+
) -> None:
176+
self.enable_generic_sets: bool = enable_generic_sets or enable_set_coercion
177+
178+
@property
179+
def enable_set_coercion(self) -> bool:
180+
return self.enable_generic_sets
181+
170182
def serialize(self, value: Any) -> str | None:
171183
if value is None:
172184
return None
@@ -221,15 +233,40 @@ def _log_coercion_fallback(target_type: type, error: Exception) -> None:
221233
# ---------------------------------------------------------------------------
222234

223235

236+
def _is_supported_set_element(converter: DataConverter, elem_type: Any) -> bool:
237+
if elem_type is None or elem_type is typing.Any:
238+
return True
239+
elem_origin = typing.get_origin(elem_type)
240+
if elem_origin is not None:
241+
if elem_origin in (list, dict, Sequence, Mapping, set, AbstractSet):
242+
# Mutable collections are unhashable
243+
return False
244+
if elem_origin is typing.Union or elem_origin is types.UnionType:
245+
non_none = [a for a in typing.get_args(elem_type) if a is not type(None)]
246+
return bool(non_none) and all(_is_supported_set_element(converter, a) for a in non_none)
247+
if elem_origin is frozenset:
248+
args = typing.get_args(elem_type)
249+
return not args or _is_supported_set_element(converter, args[0])
250+
return False
251+
if not isinstance(elem_type, type):
252+
return False
253+
if issubclass(elem_type, (list, dict, set)):
254+
return False
255+
return True
256+
257+
224258
def _can_reconstruct(converter: DataConverter, target_type: Any) -> bool:
225259
""":class:`JsonDataConverter`'s reconstruction policy.
226260
227261
Recognizes dataclasses and ``from_json()``-capable types, plus ``Optional``
228262
/ ``list`` / ``Sequence`` hints wrapping them; builtins and unknown
229-
annotations are excluded. Recurses through ``converter.can_reconstruct``
230-
(not itself) so a :class:`JsonDataConverter` subclass that overrides
231-
``can_reconstruct`` still participates in the element-type checks of
232-
``Optional`` / ``list`` hints.
263+
annotations are excluded. When set support is enabled on the converter
264+
(via ``enable_generic_sets=True``), ``set`` / ``frozenset`` / ``Set``
265+
containers with supported element types are also recognized. Recurses
266+
through ``converter.can_reconstruct`` (not itself) so a
267+
:class:`JsonDataConverter` subclass that overrides ``can_reconstruct``
268+
still participates in the element-type checks of ``Optional`` / ``list``
269+
hints.
233270
"""
234271
origin = typing.get_origin(target_type)
235272
if origin is not None:
@@ -240,11 +277,19 @@ def _can_reconstruct(converter: DataConverter, target_type: Any) -> bool:
240277
)
241278
if origin in (list, Sequence):
242279
return any(converter.can_reconstruct(a) for a in args)
280+
if origin in (set, frozenset, AbstractSet, typing.AbstractSet):
281+
if not getattr(converter, "enable_generic_sets", False):
282+
return False
283+
if not args:
284+
return True
285+
return _is_supported_set_element(converter, args[0])
243286
return False
244287
if not isinstance(target_type, type):
245288
return False
246289
if dataclasses.is_dataclass(target_type):
247290
return True
291+
if target_type in (set, frozenset, AbstractSet):
292+
return getattr(converter, "enable_generic_sets", False)
248293
return callable(getattr(cast(Any, target_type), "from_json", None))
249294

250295

@@ -253,9 +298,12 @@ def _to_json(obj: Any) -> str:
253298
254299
Builtins serialize to plain JSON. Objects exposing a ``to_json()`` method,
255300
dataclasses, and ``SimpleNamespace`` instances are serialized to plain JSON
256-
as well (without any type marker), recursively. Custom objects can be
257-
reconstructed on the receiving side by passing ``expected_type`` to
258-
:func:`_from_json`.
301+
as well (without any type marker), recursively. Sets and frozensets are
302+
serialized as JSON arrays with elements deterministically ordered by their
303+
canonical JSON string representation (note that numeric set elements follow
304+
lexical JSON ordering, e.g. ``{2, 10}`` serializes as ``[10, 2]``). Custom
305+
objects can be reconstructed on the receiving side by passing ``expected_type``
306+
to :func:`_from_json`.
259307
"""
260308
try:
261309
return json.dumps(obj, default=_encode_custom_object)
@@ -319,6 +367,31 @@ def _encode_custom_object(o: Any) -> Any:
319367
to_json_hook = getattr(cast(Any, type(o)), "to_json", None)
320368
if callable(to_json_hook):
321369
return to_json_hook(o)
370+
if isinstance(o, (set, frozenset, AbstractSet)):
371+
items: AbstractSet[Any] = cast("AbstractSet[Any]", o)
372+
# Sets and frozensets are unordered collections. To ensure deterministic
373+
# serialization across processes, worker replays, and PYTHONHASHSEED
374+
# variations, elements are individually encoded to canonical JSON
375+
# (with sort_keys=True), sorted lexically by their encoded JSON string,
376+
# and emitted as normalized parsed values.
377+
#
378+
# Emitting the normalized parsed values rather than original objects
379+
# guarantees identical order and eliminates leaks from dict insertion
380+
# order variations, NaN/non-finite comparison anomalies, or partial
381+
# ordering in frozensets. A consequence of sorting canonical JSON strings
382+
# is lexical ordering for numeric sets (e.g., {2, 10} serializes as
383+
# [10, 2]).
384+
try:
385+
encoded_items = sorted(
386+
json.dumps(item, default=_encode_custom_object, sort_keys=True)
387+
for item in items
388+
)
389+
return [json.loads(encoded) for encoded in encoded_items]
390+
except Exception as e:
391+
raise TypeError(
392+
f"Cannot deterministically serialize set of type '{type(items).__name__}' "
393+
f"because its elements cannot be compared or deterministically encoded: {e}"
394+
) from e
322395
if isinstance(o, enum.Enum):
323396
# Emit the member's underlying value (a primitive). Reconstruction is
324397
# type-directed: passing the enum type to ``deserialize`` rebuilds the
@@ -477,6 +550,19 @@ def _coerce_generic(value: Any, expected_type: Any, origin: Any,
477550
if origin in (list, Sequence) and isinstance(value, list):
478551
elem_type = args[0] if args else None
479552
return [_coerce_to_type(item, elem_type, converter) for item in cast(list[Any], value)]
553+
if origin in (set, frozenset, AbstractSet, typing.AbstractSet) and isinstance(
554+
value, (list, set, frozenset, AbstractSet)
555+
):
556+
if not getattr(converter, "enable_generic_sets", False):
557+
return cast(Any, value)
558+
elem_type = args[0] if args else None
559+
coerced: list[Any] = [
560+
_coerce_to_type(item, elem_type, converter)
561+
for item in cast("Iterable[Any]", value)
562+
]
563+
if origin is frozenset:
564+
return frozenset(coerced)
565+
return set(coerced)
480566
if origin in (dict, Mapping) and isinstance(value, dict):
481567
# JSON object keys are always strings, so only the values can carry a
482568
# reconstructable type. Keys are passed through unchanged.
@@ -595,12 +681,12 @@ def _resolve_forward_refs(tp: Any, globalns: dict[str, Any]) -> Any:
595681
"""
596682
if isinstance(tp, str):
597683
try:
598-
tp = eval(tp, globalns) # noqa: S307 - resolves caller-authored annotations
684+
tp = eval(tp, globalns)
599685
except Exception:
600686
return tp
601687
elif isinstance(tp, typing.ForwardRef):
602688
try:
603-
tp = eval(tp.__forward_arg__, globalns) # noqa: S307
689+
tp = eval(tp.__forward_arg__, globalns)
604690
except Exception:
605691
return tp
606692

‎tests/durabletask/test_data_converter.py‎

Lines changed: 102 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,14 @@
1212
from unittest.mock import patch
1313

1414
import pytest
15-
1615
from durabletask.internal.entity_state_shim import StateShim
1716
from durabletask.serialization import (
1817
DEFAULT_DATA_CONVERTER,
1918
DataConverter,
2019
JsonDataConverter,
2120
)
21+
from durabletask.internal import type_discovery
22+
from durabletask.worker import _ActivityExecutor, _Registry
2223

2324

2425
@dataclass
@@ -219,7 +220,7 @@ def __init__(self, /, **kwargs):
219220
self.self = kwargs.get("self", 0)
220221

221222
converter = JsonDataConverter()
222-
encoded = converter.serialize(PositionalReceiver(**{"self": 42}))
223+
encoded = converter.serialize(PositionalReceiver(self=42))
223224
restored = converter.deserialize(encoded, PositionalReceiver)
224225
assert isinstance(restored, PositionalReceiver)
225226
assert restored.self == 42
@@ -375,3 +376,102 @@ def coerce(self, value: Any, target_type: type | None = None) -> Any:
375376
assert conv.serialize("hello") == '"HELLO"'
376377
assert conv.deserialize('"HELLO"') == "HELLO"
377378
assert conv.coerce("HELLO") == "HELLO"
379+
380+
381+
# ----- set and frozenset container support -----
382+
383+
384+
@dataclass(frozen=True)
385+
class TaggedOrder:
386+
item: str
387+
tags: set[str]
388+
flags: frozenset[int]
389+
390+
391+
def test_roundtrip_dataclass_with_sets():
392+
converter = JsonDataConverter(enable_generic_sets=True)
393+
original = TaggedOrder(item="laptop", tags={"electronics", "sale"}, flags=frozenset({1, 2}))
394+
serialized = converter.serialize(original)
395+
assert serialized is not None
396+
restored = converter.deserialize(serialized, TaggedOrder)
397+
assert isinstance(restored, TaggedOrder)
398+
assert restored.item == "laptop"
399+
assert isinstance(restored.tags, set)
400+
assert restored.tags == {"electronics", "sale"}
401+
assert isinstance(restored.flags, frozenset)
402+
assert restored.flags == frozenset({1, 2})
403+
404+
405+
def test_can_reconstruct_set_and_frozenset():
406+
# By default (enable_generic_sets=False), generic sets are not reconstructed to
407+
# preserve backward compatibility and existing-instance replay behavior.
408+
default_converter = JsonDataConverter()
409+
assert default_converter.can_reconstruct(set[PricedOrder]) is False
410+
assert default_converter.can_reconstruct(frozenset[PricedOrder]) is False
411+
assert default_converter.can_reconstruct(set[int]) is False
412+
assert default_converter.can_reconstruct(set[str]) is False
413+
414+
# When opted in, supported sets (including primitive elements) return True,
415+
# while unsupported unhashable element types return False.
416+
opt_in_converter = JsonDataConverter(enable_generic_sets=True)
417+
assert opt_in_converter.can_reconstruct(set[PricedOrder]) is True
418+
assert opt_in_converter.can_reconstruct(frozenset[PricedOrder]) is True
419+
assert opt_in_converter.can_reconstruct(set[int]) is True
420+
assert opt_in_converter.can_reconstruct(set[str]) is True
421+
assert opt_in_converter.can_reconstruct(frozenset[int]) is True
422+
assert opt_in_converter.can_reconstruct(set[list[str]]) is False
423+
assert opt_in_converter.can_reconstruct(set[dict[str, int]]) is False
424+
425+
426+
def test_existing_instance_replay_compatibility_preserves_list():
427+
"""Preserve existing-instance replay behavior for recorded arrays under set annotations.
428+
429+
When generic set coercion is not explicitly enabled, historical JSON arrays
430+
containing duplicates and order (e.g. {"tags": ["b", "a", "b"]}) remain raw
431+
lists, avoiding NonDeterminismError during orchestration replay.
432+
"""
433+
default_converter = JsonDataConverter()
434+
recorded_input = '{"item": "laptop", "tags": ["b", "a", "b"], "flags": [1, 2, 1]}'
435+
restored = default_converter.deserialize(recorded_input, TaggedOrder)
436+
assert isinstance(restored, TaggedOrder)
437+
assert isinstance(restored.tags, list)
438+
assert restored.tags == ["b", "a", "b"]
439+
assert len(restored.tags) == 3
440+
assert isinstance(restored.flags, list)
441+
assert restored.flags == [1, 2, 1]
442+
assert len(restored.flags) == 3
443+
444+
# With opt-in enabled, set coercion deduplicates and converts to sets
445+
opt_in_converter = JsonDataConverter(enable_generic_sets=True)
446+
opt_in_restored = opt_in_converter.deserialize(recorded_input, TaggedOrder)
447+
assert isinstance(opt_in_restored, TaggedOrder)
448+
assert isinstance(opt_in_restored.tags, set)
449+
assert opt_in_restored.tags == {"a", "b"}
450+
assert len(opt_in_restored.tags) == 2
451+
assert isinstance(opt_in_restored.flags, frozenset)
452+
assert opt_in_restored.flags == frozenset({1, 2})
453+
assert len(opt_in_restored.flags) == 2
454+
455+
456+
def test_type_discovery_and_activity_execution_with_sets():
457+
"""Verify that type discovery recognizes set annotations when opted in,
458+
enabling activities to receive sets rather than raw lists."""
459+
def tag_activity(ctx: Any, tags: set[str]) -> int:
460+
assert isinstance(tags, set)
461+
tags.add("processed")
462+
return len(tags)
463+
464+
reg = _Registry()
465+
reg.add_activity(tag_activity)
466+
logger = logging.getLogger("test_type_discovery_sets")
467+
468+
# With opt-in, activity input is discovered as set[str] and executed
469+
conv = JsonDataConverter(enable_generic_sets=True)
470+
assert type_discovery.activity_input_type(tag_activity, conv) == set[str]
471+
executor = _ActivityExecutor(reg, logger, conv)
472+
output = executor.execute("orch-1", "tag_activity", 1, '["item1", "item2"]')
473+
assert output == "3"
474+
475+
# Default converter preserves input as None (raw JSON), passing list
476+
default_conv = JsonDataConverter()
477+
assert type_discovery.activity_input_type(tag_activity, default_conv) is None

‎tests/durabletask/test_data_converter_roundtrip.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -545,8 +545,6 @@ class HasUnion:
545545
@pytest.mark.parametrize("value", [
546546
datetime(2020, 1, 1, 12, 0, 0),
547547
Decimal("1.5"),
548-
{1, 2, 3},
549-
frozenset({1, 2}),
550548
b"bytes",
551549
])
552550
def test_unsupported_types_raise_clear_typeerror(conv, value):

0 commit comments

Comments
 (0)