Support special characters in job names

.env.example

@@ -20,6 +20,31 @@
 # each process mints its own token, which only works single-process-per-host.
 # APOLLO_INTERNAL_TOKEN=
 
+# Which characters a workflow step name may contain. Apollo sanitises step names
+# on the way out and Lightning validates them on the way in, so the two rules
+# have to agree.
+#
+#   false (the default)  ASCII only: letters, digits, spaces, hyphens and
+#                        underscores. Accents are folded (Café -> Cafe) and
+#                        anything else is dropped. Matches the rule Lightning
+#                        enforces today.
+#   true                 Anything except control characters: letters and marks
+#                        from any script, all punctuation and symbols, emoji,
+#                        / : > & and quotes. Vérifier l'état and 患者確認
+#                        survive exactly as typed.
+#
+# Leave this off until Lightning ships its Unicode step names (Lightning#4577),
+# then turn it on. Turning it on first means Apollo emits names Lightning
+# rejects; leaving it off afterwards means Apollo renames steps people typed
+# deliberately.
+#
+# Both modes reject the same control set and nothing else: C0 (U+0000-U+001F,
+# NUL included), DEL (U+007F), C1 (U+0080-U+009F), U+FFFE / U+FFFF, the
+# surrogates U+D800-U+DFFF, and the separators U+2028 / U+2029. Names
+# are NFC-normalised and capped at 100 graphemes in both modes.
+# See services/name_rules.py.
+APOLLO_UNICODE_STEP_NAMES=false
+
 ANTHROPIC_API_KEY=sk-YOUR-API-KEY-HERE
 
 OPENAI_API_KEY=sk-YOUR-API-KEY-HERE

.github/workflows/unit-tests.yaml

@@ -28,6 +28,43 @@ jobs:
       - name: Run unit tests
         run: poetry run pytest services/*/tests/unit
 
+  unicode-parity:
+    name: Unicode parity with Elixir
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    # `services/name_rules.py` carries tables generated from the Elixir that
+    # Lightning runs: grapheme break classes, Extended_Pictographic, combining
+    # classes, the trim set and OTP's NFC. If Elixir's or Python's Unicode
+    # version moves and nobody re-runs the harness, the tables silently stop
+    # matching and Apollo starts emitting step names Lightning rejects. This
+    # job is the only thing that would notice.
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Set up Elixir
+        uses: erlef/setup-beam@v1
+        with:
+          elixir-version: "1.18.3"
+          otp-version: "27"
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v7
+        with:
+          python-version: "3.11"
+
+      - name: Generate range edges from the tables
+        working-directory: tools/unicode_parity
+        run: python3 edges.py
+
+      - name: Probe Elixir
+        working-directory: tools/unicode_parity
+        run: elixir probe.exs
+
+      - name: Compare against name_rules
+        working-directory: tools/unicode_parity
+        run: python3 check.py
+
   bun:
     name: Bun unit tests
     runs-on: ubuntu-latest

README.md

@@ -147,6 +147,73 @@ full list of keys and env vars Apollo reads.
 Also note that `tmp` dirs are untracked, so if you do want to store credentials
 in your json, keep it inside a tmp dir and it'll remain safe and secret.
 
+### `APOLLO_UNICODE_STEP_NAMES`
+
+Controls which characters a workflow step name may contain. Apollo sanitises
+step names on the way out and Lightning validates them on the way in, so the two
+rules have to agree. The default is `false`.
+
+| Value | Rule |
+| --- | --- |
+| `false` (default) | ASCII only. Letters, digits, spaces, hyphens, underscores. Accents are folded (`Café` becomes `Cafe`) and anything else is dropped. This is the rule Lightning enforces today. |
+| `true` | Anything except control characters. Letters and marks from any script, all punctuation and symbols, emoji, `/`, `:`, `>`, `&`, quotes and apostrophes. `Vérifier l'état` and `患者確認` survive exactly as typed. |
+
+Leave it off until Lightning ships Unicode step names (Lightning#4577), then
+turn it on. Turning it on first means Apollo emits names Lightning rejects.
+Leaving it off afterwards means Apollo renames steps people typed deliberately,
+across the whole workflow, on every turn that returns YAML.
+
+The permissive rule is deliberately maximal. Apollo being stricter than
+Lightning is the worse of the two failures: Lightning rejecting a name is loud
+and recoverable, whereas Apollo quietly renaming a valid name is the silent
+vandalism this flag exists to prevent.
+
+The rejected control set is the same in both modes: C0 (`U+0000`-`U+001F`,
+NUL included), DEL (`U+007F`), C1 (`U+0080`-`U+009F`), the noncharacters
+`U+FFFE` and `U+FFFF`, the surrogates `U+D800`-`U+DFFF`, and the line and
+paragraph separators `U+2028` and `U+2029`. A NUL byte in a name crashes the Postgres insert on
+Lightning's side. Names are NFC-normalised in both modes so that Apollo and
+Lightning agree on how to spell an accent, which is what step lookup matches
+on, and capped at 100 graphemes because that is what Ecto's `validate_length`
+counts.
+
+The rule lives in `services/name_rules.py`, and everything that states or
+enforces it is derived from there: the sanitiser, the workflow-generation
+prompt (`describe_rule_for_prompt`), the acceptance-test judges
+(`describe_rule_for_judge`, substituted into the rubric markdown by
+`judges.load_judge`), and the `assert_no_special_chars` test assertion. Change
+the rule in that one file and all four follow.
+
+The 100-character cap is counted in graphemes, because that is what Ecto's
+`validate_length` counts on Lightning's side. The clustering is hand-written in
+`name_rules`, with no third-party dependency, and it targets Elixir's
+`String.length/1` rather than UAX #29 — Elixir deviates from the spec in two
+places (it does not implement the Unicode 15.1 Indic conjunct rule, and it ends
+an emoji ZWJ run at the joiner unless a pictograph follows) and the whole point
+is to agree with Elixir, not with the spec.
+
+`tools/unicode_parity` is the harness that checks it. Run `python3 edges.py`,
+then `elixir probe.exs`, then `python3 check.py` with the Elixir version Lightning runs; `--tables`
+prints the literals to paste back into `name_rules`. It checks five things:
+every codepoint's break class, the `Extended_Pictographic` set, the trim set,
+what a GB11 emoji run may be separated from its joiner by, and cluster
+boundaries over a generated corpus. Normalisation is not among them:
+`normalize_nfc` is the standard library's, so there is no table of ours to
+check against Elixir.
+
+`Extended_Pictographic` needs its own check because it is *not* a break class,
+so a per-codepoint sweep cannot see it — an over-broad set there silently
+changes clustering either side of a ZWJ and nothing else notices. That is
+exactly how a hand-written table with 531 wrong codepoints survived two rounds
+of review.
+
+Re-run the harness whenever Python's or Elixir's Unicode version moves.
+`name_rules.PARITY_SOURCE` records what the committed tables were generated
+from, and a unit test pins it, so a silent regeneration fails loudly. Python
+moving ahead only makes Apollo overcount, which truncates early; Elixir moving
+ahead is the direction that reintroduces undercounting, and an undercount ships
+a name Lightning rejects.
+
 ## Debugging
 
 The server defaults to port 3000. You can test any service directly with curl to

pyproject.toml

@@ -45,6 +45,7 @@ pythonpath = ["services"]
 
 # Discovery roots. pytest walks these for test_*.py files.
 testpaths = [
+    "services/echo/tests",
     "services/global_chat/tests",
     "services/workflow_chat/tests",
     "services/job_chat/tests",
@@ -59,6 +60,9 @@ python_functions = ["test_*"]
 
 markers = [
     "unit: fast, isolated, no I/O. Runs on every PR push.",
+    # Declared but currently unused: nothing carries it, so `-m \"unit or service\"`
+    # is really `-m unit`. Kept because the tiers are referenced in the testing
+    # README; apply it when the first mocked-client suite lands.
     "service: mocks HTTP/LLM clients; exercises service handlers. Runs on merge.",
     "integration: hits real external services (LLM, Pinecone, Postgres). Manual/nightly.",
     "acceptance: end-to-end acceptance criteria. Manual/nightly.",

services/global_chat/PAYLOAD_SPEC.md

@@ -172,7 +172,9 @@ The `page` field is a simplified path/breadcrumb representing where the user is
 workflows/<workflow-name>/<step-name>
 ```
 
-The step name should match a job key in the workflow YAML (exact match or normalized — lowercase, non-alphanumeric chars replaced with hyphens). The backend parses the URL by splitting on `/` and reading the 3rd segment as the step name.
+The step name should match a job key in the workflow YAML, either exactly or after normalization — NFC-normalized, lowercased, with every character that is not a letter, mark or digit replaced by a hyphen. Normalization is Unicode-aware, so `患者確認` normalizes to itself rather than to the empty string; a name that normalizes to nothing is never fuzzy-matched.
+
+The backend parses the URL by splitting on `/` and taking everything after the workflow segment as the step name, so a step name containing a `/` survives. A workflow name containing a `/` still makes the split ambiguous, so the parsed step name is validated against the workflow YAML rather than trusted.
 
 | Page URL | Router signal | What happens |
 |---|---|---|

services/global_chat/tests/test_workflow_chat_pass_fail.py

@@ -222,9 +222,9 @@ def test_rename_two_jobs_commcare():
 
 def test_special_characters():
     print("==================TEST==================")
-    print("Description: Ask for a workflow that uses platforms with special characters in their names. "
-          "Verify that diacritics and punctuation removed/normalised correctly (e.g. é->e) in job names "
-          "in the generated YAML.")
+    print("Description: Ask for a workflow that uses platforms with accents and punctuation in their "
+          "names. Verify the job names in the generated YAML obey whichever step-name rule is active "
+          "(see name_rules): folded to ASCII by default, kept as typed with APOLLO_UNICODE_STEP_NAMES on.")
     existing_yaml = """"""
     history = [
         {"role": "user", "content": "Create a workflow that retrieves data from mwater, google sheets, netsuite, ferntech.io and processed it and sends it to frappé"},

services/global_chat/tests/unit/test_error_logging.py

@@ -369,18 +369,42 @@ def _leak_patterns(names: set[str]) -> list:
     # Job and edge names before and after sanitising, the adaptor a job
     # declares, and the `__ID_JOB_x__` placeholders this service invented
     # itself. Names and ids, never a body.
+    # The naming work replaced per-key logging with one line naming the whole
+    # renamed set, so the individual key expressions the leak branch vets are
+    # gone from this module here.
     "workflow_chat/workflow_chat.py": frozenset({
         "adaptor",
         "job_key",
-        "edge_key",
-        "sanitized_edge_key",
-        "original_name",
-        "sanitized_name",
-        "original_source",
-        "original_target",
-        "edge_data['source_job']",
-        "edge_data['target_job']",
         "current_id",
+        # Job names and edge endpoints, resolved or unresolved. `unclaimed`
+        # reads as bodies but holds the `__CODE_BLOCK_<key>__` tokens, so it is
+        # keys too. Same category as the names above, and the reason a name is
+        # loggable where a body is not: the user typed it into a form as a
+        # label, and a log line is unreadable without it.
+        "', '.join(sorted(matches))",
+        "', '.join(duplicated)",
+        "', '.join(unclaimed)",
+        "', '.join(renamed)",
+        "by_name",
+        "owner",
+        "', '.join(sorted(dangling))",
+        # Literals chosen at the call site, a parameter the callers pass a
+        # literal to, and a count. `msg` is built but only from `len()`.
+        "how",
+        "label",
+        "msg",
+        # More names: the reference as written, and what it sanitises to.
+        "reference",
+        "str(reference)",
+        "resolved",
+    }),
+
+    # Job names again, on the shared walkers.
+    "yaml_utils.py": frozenset({
+        "', '.join(sorted((str(match) for match in matches)))",
+        "job_key",
+        "how",
+        "step_name",
     }),
 }
 
@@ -978,7 +1002,7 @@ def test_a_vetted_expression_is_scoped_to_its_module() -> None:
 #: Every expression cleared by hand in `VETTED_INTERPOLATIONS`. Pinned for the
 #: same reason as `EXPECTED_MARKERS`: an opt-out nobody counts is an opt-out
 #: that spreads.
-EXPECTED_VETTED_INTERPOLATIONS = 51
+EXPECTED_VETTED_INTERPOLATIONS = 60
 
 
 def test_the_vetted_interpolations_are_inventoried() -> None: