feat!: decrypt wire JSON by field schema (decrypt_task_payload, decrypt_task_summary, decrypt_submission, decrypt_event); remove looks_like_ciphertext

Author: tymm

src/simplepush/__init__.py

@@ -38,21 +38,26 @@
 _CRYPTO_NAMES = (
     "Decryptor", "OrgDecryptor", "Keyring", "DerivedKey",
     "derive_key", "encrypt", "decrypt", "key_fingerprint",
-    "try_decrypt_event_data", "looks_like_ciphertext",
+    "try_decrypt_event_data", "DecryptedWire",
+    "decrypt_task_payload", "decrypt_task_summary", "decrypt_submission", "decrypt_event",
 )
 
 try:
     from .crypto import (
         Decryptor, OrgDecryptor, Keyring, DerivedKey,
         derive_key, encrypt, decrypt, key_fingerprint,
     )
-    from .decrypt import try_decrypt_event_data, looks_like_ciphertext
+    from .decrypt import (
+        try_decrypt_event_data, DecryptedWire,
+        decrypt_task_payload, decrypt_task_summary, decrypt_submission, decrypt_event,
+    )
     # Literal list (not `list(_CRYPTO_NAMES)`) so static checkers can follow
     # the export list; keep it in sync with _CRYPTO_NAMES above.
     __all__ += [
         "Decryptor", "OrgDecryptor", "Keyring", "DerivedKey",
         "derive_key", "encrypt", "decrypt", "key_fingerprint",
-        "try_decrypt_event_data", "looks_like_ciphertext",
+        "try_decrypt_event_data", "DecryptedWire",
+        "decrypt_task_payload", "decrypt_task_summary", "decrypt_submission", "decrypt_event",
     ]
 except MissingCryptoExtra:
     # pynacl is absent: keep the crypto names out of the namespace (and out of

src/simplepush/decrypt.py

@@ -1,54 +1,261 @@
-"""Best-effort decryption of an event's ``data`` payload via a `Keyring`.
+"""Schema-driven decryption of the wire JSON the read surface returns: task
+payloads, chains, summaries, submissions, raw events.
 
-Mirrors the TS SDK's ``tryDecryptEventData``: for an encrypted event, resolve the
-key from its marker and recursively decrypt every base64 string in ``data`` that
-looks like spush ciphertext. Cheap heuristic — spush ciphertext is base64 of
->=28 bytes, so the encoded form is >=40 base64 chars. Use it to decrypt the raw
-``Client.events()`` feed across many candidate passwords:
+The encrypted fields are exactly the ones the send side seals (`api.py`) and
+the watch views decrypt (`client.py`); this module mirrors that map field by
+field.
+
+Marker rules:
+  - a payload/summary/entry marker covers its sender-authored fields;
+  - an answer record's own ``encryption`` wins over the envelope's (answers
+    sealed after an org key rotation carry newer keys than the task);
+  - replies, decline notes and cancel notes carry strictly their own marker
+    (authored after the send; absent marker = plaintext note);
+  - a location's coords ride as one ``encrypted`` JSON blob, expanded in
+    place, exactly like a slider input's sealed scale.
+
+Every known-encrypted field under a marker either decrypts or increments
+``undecryptable``; nothing else is ever attempted. A new encrypted wire field
+must be added here (and to the watch views).
 
     ring = client.keyring()
-    async for event in client.events():
-        data = try_decrypt_event_data(event, ring)   # decrypted dict, or None
+    page = decrypt_task_payload(read, ring)   # page.value, page.undecryptable
 """
 
 import copy
-import re
+import json
+from typing import Any, NamedTuple
 
 from .crypto import decrypt
 
-_CIPHERTEXT_RE = re.compile(r"^[A-Za-z0-9+/=]+$")
 
+class DecryptedWire(NamedTuple):
+    value: Any
+    undecryptable: int
 
-def looks_like_ciphertext(s: str) -> bool:
-    return 40 <= len(s) <= 8192 and bool(_CIPHERTEXT_RE.match(s))
 
+class _State:
+    __slots__ = ("undecryptable",)
 
-def try_decrypt_event_data(event, keyring) -> dict | None:
-    """Return a decrypted copy of ``event.data``, or None when the event isn't
-    encrypted or no held key matches its marker. Ciphertext that fails to decrypt
-    is left in place."""
-    marker = event.raw.get("encryption")
-    if not marker:
-        return None
+    def __init__(self) -> None:
+        self.undecryptable = 0
+
+
+def _marker_of(v: Any) -> dict | None:
+    return v if isinstance(v, dict) and isinstance(v.get("type"), str) else None
+
+
+def _dec_field(o: dict, field: str, marker: dict | None, keyring, st: _State) -> None:
+    """Decrypt ``o[field]`` in place when it is a string and a marker applies.
+    No marker = plaintext field, left alone; a marker with no matching key or a
+    failed authentication counts as undecryptable and leaves the ciphertext."""
+    v = o.get(field)
+    if not isinstance(v, str) or marker is None:
+        return
     key = keyring.key_for_marker(marker)
     if key is None:
+        st.undecryptable += 1
+        return
+    try:
+        o[field] = decrypt(v, key)
+    except Exception:
+        st.undecryptable += 1
+
+
+def _dec_list(o: dict, field: str, marker: dict | None, keyring, st: _State) -> None:
+    items = o.get(field)
+    if not isinstance(items, list):
+        return
+    for i, v in enumerate(items):
+        wrap = {"v": v}
+        _dec_field(wrap, "v", marker, keyring, st)
+        items[i] = wrap["v"]
+
+
+def _dec_blob(o: dict, marker: dict | None, keyring, st: _State) -> None:
+    """Decrypt-and-JSON-expand an ``{"encrypted": ...}`` blob (slider scale,
+    location coords) onto the record itself, dropping the blob on success."""
+    v = o.get("encrypted")
+    if not isinstance(v, str) or marker is None:
+        return
+    key = keyring.key_for_marker(marker)
+    if key is None:
+        st.undecryptable += 1
+        return
+    try:
+        plain = json.loads(decrypt(v, key))
+    except Exception:
+        st.undecryptable += 1
+        return
+    if isinstance(plain, dict):
+        del o["encrypted"]
+        o.update(plain)
+    else:
+        st.undecryptable += 1
+
+
+def _dec_input(inp: Any, marker: dict | None, keyring, st: _State) -> None:
+    if not isinstance(inp, dict):
+        return
+    _dec_field(inp, "description", marker, keyring, st)
+    kind = inp.get("type")
+    if kind == "text":
+        _dec_field(inp, "defaultValue", marker, keyring, st)
+    elif kind == "choice":
+        _dec_list(inp, "options", marker, keyring, st)
+    elif kind == "actions":
+        for a in inp.get("actions") or []:
+            if isinstance(a, dict):
+                _dec_field(a, "key", marker, keyring, st)
+                _dec_field(a, "label", marker, keyring, st)
+    elif kind == "slider":
+        _dec_blob(inp, marker, keyring, st)
+
+
+def _dec_upload(u: Any, envelope: dict | None, keyring, st: _State) -> None:
+    """An answer record (``textUploaded``, ``choiceSelected``, ...), as it appears
+    both in event data and in a payload's ``uploads``. Own marker wins over the
+    envelope's. File-kind records carry only plaintext metadata."""
+    if not isinstance(u, dict):
+        return
+    marker = _marker_of(u.get("encryption")) or envelope
+    u.pop("encryption", None)
+    kind = u.get("type")
+    if kind in ("textUploaded", "sliderUploaded"):
+        _dec_field(u, "value", marker, keyring, st)
+    elif kind == "choiceSelected":
+        _dec_field(u, "selectedValue", marker, keyring, st)
+    elif kind == "multiChoiceSelected":
+        _dec_list(u, "selectedValues", marker, keyring, st)
+    elif kind == "actionSelected":
+        _dec_field(u, "selectedKey", marker, keyring, st)
+    elif kind == "locationUploaded":
+        # The coords are flattened onto the record itself, no nested key.
+        _dec_blob(u, marker, keyring, st)
+
+
+def _dec_message_body(container: dict, marker: dict | None, keyring, st: _State) -> None:
+    """A reply/submission body plus inline location, under the given marker."""
+    body = container.get("body")
+    if isinstance(body, dict) and body.get("type") == "text":
+        _dec_field(body, "value", marker, keyring, st)
+    loc = container.get("location")
+    if isinstance(loc, dict):
+        _dec_blob(loc, marker, keyring, st)
+
+
+def _dec_reply_record(r: Any, keyring, st: _State) -> None:
+    """A reply off a payload's ``replies``: its own marker only."""
+    if not isinstance(r, dict):
+        return
+    marker = _marker_of(r.pop("encryption", None))
+    _dec_message_body(r, marker, keyring, st)
+
+
+def _dec_note_record(r: Any, keyring, st: _State) -> None:
+    """A decline record / the cancellation block: the note's own marker only."""
+    if not isinstance(r, dict):
+        return
+    marker = _marker_of(r.pop("encryption", None))
+    _dec_field(r, "note", marker, keyring, st)
+
+
+def _dec_payload_in_place(p: dict, keyring, st: _State) -> None:
+    marker = _marker_of(p.pop("encryption", None))
+    for f in ("tag", "title", "content"):
+        _dec_field(p, f, marker, keyring, st)
+    for a in p.get("attachments") or []:
+        if isinstance(a, dict) and a.get("type") == "link":
+            _dec_field(a, "url", marker, keyring, st)
+    for i in p.get("inputs") or []:
+        _dec_input(i, marker, keyring, st)
+    for u in p.get("uploads") or []:
+        _dec_upload(u, marker, keyring, st)
+    for r in p.get("replies") or []:
+        _dec_reply_record(r, keyring, st)
+    for d in p.get("declines") or []:
+        _dec_note_record(d, keyring, st)
+    _dec_note_record(p.get("cancellation"), keyring, st)
+
+
+def decrypt_task_payload(value: Any, keyring) -> DecryptedWire:
+    """A task or subtask payload (the shapes chain reads return)."""
+    st = _State()
+    out = copy.deepcopy(value)
+    if isinstance(out, dict):
+        _dec_payload_in_place(out, keyring, st)
+    return DecryptedWire(out, st.undecryptable)
+
+
+def decrypt_task_summary(value: Any, keyring) -> DecryptedWire:
+    """A task index / group roster row: ``title`` is its only sealed field."""
+    st = _State()
+    out = copy.deepcopy(value)
+    if isinstance(out, dict):
+        marker = _marker_of(out.pop("encryption", None))
+        _dec_field(out, "title", marker, keyring, st)
+    return DecryptedWire(out, st.undecryptable)
+
+
+def decrypt_submission(value: Any, keyring, marker: dict | None) -> DecryptedWire:
+    """A submission (body + inline location) under its feed entry's marker;
+    the submission carries no marker of its own."""
+    st = _State()
+    out = copy.deepcopy(value)
+    if isinstance(out, dict):
+        _dec_message_body(out, marker, keyring, st)
+    return DecryptedWire(out, st.undecryptable)
+
+
+def decrypt_event(raw: Any, keyring) -> DecryptedWire:
+    """One wire event's ``data``, decrypted in place on a copy of the raw event
+    dict by event-data type. Unrecognized types pass through untouched."""
+    st = _State()
+    out = copy.deepcopy(raw)
+    data = out.get("data") if isinstance(out, dict) else None
+    if not isinstance(data, dict):
+        return DecryptedWire(out, 0)
+    marker = _marker_of(out.get("encryption"))
+    kind = data.get("type")
+    if kind in ("taskInputUploaded", "taskInputCompleted", "subtaskInputUploaded", "subtaskInputCompleted"):
+        _dec_upload(data.get("inputUploaded"), marker, keyring, st)
+    elif kind in ("taskCompleted", "subtaskCompleted"):
+        for u in data.get("inputsUploaded") or []:
+            _dec_upload(u, marker, keyring, st)
+    elif kind == "replyAppended":
+        reply = data.get("reply")
+        if isinstance(reply, dict):
+            _dec_message_body(reply, marker, keyring, st)
+    elif kind == "submissionCreated":
+        submission = data.get("submission")
+        if isinstance(submission, dict):
+            _dec_message_body(submission, marker, keyring, st)
+    elif kind == "notificationCompleted":
+        reply = data.get("reply")
+        if isinstance(reply, dict):
+            rt = reply.get("type")
+            if rt == "text":
+                _dec_field(reply, "value", marker, keyring, st)
+            elif rt == "choice":
+                _dec_field(reply, "selectedValue", marker, keyring, st)
+            elif rt == "actions":
+                _dec_field(reply, "selectedKey", marker, keyring, st)
+    elif kind in (
+        "taskCanceled", "subtaskCanceled",
+        "taskDeclinedByRecipient", "subtaskDeclinedByRecipient",
+        "taskDeclined", "subtaskDeclined",
+    ):
+        # The envelope marker on these is the note's own; reason/supersededBy
+        # stay plaintext.
+        _dec_field(data, "note", marker, keyring, st)
+    return DecryptedWire(out, st.undecryptable)
+
+
+def try_decrypt_event_data(event, keyring) -> dict | None:
+    """A decrypted copy of ``event.data``, or None when the event isn't
+    encrypted or no held key matches its marker. A field that fails to decrypt
+    (e.g. sealed under a newer org key) is left as ciphertext."""
+    marker = _marker_of(event.raw.get("encryption"))
+    if marker is None or keyring.key_for_marker(marker) is None:
         return None
-    # event.data is always a dict, so inline its branch of _decrypt_in_place
-    # (which would just run this same comprehension) to keep the return a dict.
-    data = copy.deepcopy(event.data)
-    return {k: _decrypt_in_place(v, key) for k, v in data.items()}
-
-
-def _decrypt_in_place(value, key: bytes):
-    if isinstance(value, str):
-        if looks_like_ciphertext(value):
-            try:
-                return decrypt(value, key)
-            except Exception:
-                return value  # leave ciphertext in place
-        return value
-    if isinstance(value, list):
-        return [_decrypt_in_place(v, key) for v in value]
-    if isinstance(value, dict):
-        return {k: _decrypt_in_place(v, key) for k, v in value.items()}
-    return value
+    return decrypt_event(event.raw, keyring).value.get("data")

tests/e2e/test_cross_sdk.py

@@ -12,7 +12,7 @@
 
 pytest.importorskip("nacl", reason="crypto extra required: pip install 'simplepush[crypto]'")
 
-from simplepush import Client, derive_key, looks_like_ciphertext  # noqa: E402
+from simplepush import Client, derive_key  # noqa: E402
 from simplepush.crypto import decrypt  # noqa: E402  (bare `simplepush.decrypt` is shadowed by the submodule)
 
 
@@ -50,7 +50,7 @@ def test_python_encrypted_send_decrypts_in_js(conn, js_helper):
     assert out["decrypted"], "JS keyring cannot read a python send"
 
 
-def test_wire_shape_parity_for_encrypted_tagged_sends(conn, sender_read, js_helper):
+def test_wire_shape_parity_for_encrypted_tagged_sends(conn, sender_read, password_salt, js_helper):
     """Send the same logical task from both SDKs; the at-rest payloads must
     have the same shape AND classify identically as ciphertext/plaintext per
     field. Catches one-sided encryption or naming drift mechanically."""
@@ -62,18 +62,23 @@ def test_wire_shape_parity_for_encrypted_tagged_sends(conn, sender_read, js_help
     )
     out = js_helper("send-parity", PW, s)
 
-    shape_py = _shape(sender_read(py_task.task_id))
-    shape_js = _shape(sender_read(out["taskId"]))
+    key = derive_key(PW, password_salt).symmetric_key
+    shape_py = _shape(sender_read(py_task.task_id), key)
+    shape_js = _shape(sender_read(out["taskId"]), key)
     assert shape_py == shape_js, f"\npython: {shape_py}\njs:     {shape_js}"
 
 
 PARITY_FIELDS = ("title", "content", "tag", "contentFormat", "autoCommit", "inputs")
 
 
-def _shape(read: dict):
+def _shape(read: dict, key: bytes):
     def classify(v):
         if isinstance(v, str):
-            return "enc" if looks_like_ciphertext(v) else "plain"
+            try:
+                decrypt(v, key)
+                return "enc"
+            except Exception:
+                return "plain"
         if isinstance(v, dict):
             return {k: classify(x) for k, x in sorted(v.items()) if x is not None}
         if isinstance(v, list):