Merge pull request #588 from dkackman/develop

Release 0.8.0

Author: dkackman

.github/codeql/dw-security/DwPathSanitizers.qll

@@ -44,7 +44,8 @@ private predicate pathValidatorName(string name) {
   name =
     [
       "validate_path", "validate_workflow_path", "validate_output_path",
-      "validate_prompt_path", "safe_join_path", "validate_media_path"
+      "validate_prompt_path", "validate_lora_path", "safe_join_path",
+      "validate_media_path"
     ]
 }
 
@@ -56,7 +57,8 @@ private predicate pathValidatorName(string name) {
 private predicate nameValidatorName(string name) {
   name =
     [
-      "validate_workspace_name", "validate_prompt_reference", "validate_asset_reference",
+      "validate_workspace_name", "validate_prompt_reference", "validate_lora_name",
+      "validate_asset_reference",
       "validate_output_reference", "validate_variable_name", "validate_commit_hash"
     ]
 }

.github/codeql/extensions/dw-models/models/dw-security.model.yml

@@ -10,6 +10,7 @@ extensions:
       - ["dw.security", "Member[validate_workflow_path].ReturnValue", "path-injection"]
       - ["dw.security", "Member[validate_output_path].ReturnValue", "path-injection"]
       - ["dw.security", "Member[validate_prompt_path].ReturnValue", "path-injection"]
+      - ["dw.security", "Member[validate_lora_path].ReturnValue", "path-injection"]
 
       # The name validators are regex whitelists that raise InvalidInputError:
       # a workspace name is one path segment (^[\w][\w.-]*\Z - no separator,
@@ -20,4 +21,5 @@ extensions:
       - ["dw.security", "Member[validate_asset_reference].ReturnValue", "path-injection"]
       - ["dw.security", "Member[validate_output_reference].ReturnValue", "path-injection"]
       - ["dw.security", "Member[validate_prompt_reference].ReturnValue", "path-injection"]
+      - ["dw.security", "Member[validate_lora_name].ReturnValue", "path-injection"]
       - ["dw.security", "Member[validate_variable_name].ReturnValue", "path-injection"]

.github/workflows/ci.yml

@@ -42,7 +42,7 @@ jobs:
         # goes on afterwards so the resolver doesn't replace it with a
         # release
         run: |
-          pip install -r requirements.txt -r requirements-test.txt
+          pip install -c constraints-openapi.txt -r requirements.txt -r requirements-test.txt
           pip install git+https://github.com/huggingface/diffusers
       - name: Format check
         run: ruff format --check dw dw_mcp tests scripts
@@ -68,10 +68,19 @@ jobs:
           cache: npm
           cache-dependency-path: ui/package-lock.json
       - run: npm ci
+      # The UI's response types are generated from the server's OpenAPI
+      # document (scripts/dump_openapi.py writes it; the backend job's
+      # tests/test_api_contract.py fails when it is stale). A stale
+      # api-schema.ts fails here; a field the UI reads that the server no
+      # longer sends fails the type check below
+      - name: Response contract
+        run: npm run gen:api && git diff --exit-code src/lib/generated
       - name: Format check
-        run: npx prettier --check src e2e *.ts *.js
+        run: npx prettier --check src e2e scripts *.ts *.js
       - name: Lint
         run: npm run lint
+      - name: Architecture ratchet
+        run: npm run metrics -- --check ../docs/stabilization/ui/baseline.json
       - name: Type check
         run: npm run check
       - name: Unit tests
@@ -80,10 +89,17 @@ jobs:
         run: npm run build
 
   e2e:
-    # Playwright against a real dw.serve, on PRs into master only - the
-    # release PR is where a stale spec has to be caught, and the job carries
-    # the backend install as well as the browser
-    if: github.event_name == 'pull_request' && github.base_ref == 'master'
+    # Playwright against a real dw.serve before develop moves - on PRs into
+    # develop or master, and on pushes to develop, since the agent loop
+    # pushes develop directly with no PR. The release PR (this repo's
+    # develop into master) is skipped: its every commit is a develop push,
+    # already run. The job carries the backend install as well as the browser
+    if: >-
+      (github.event_name == 'pull_request' &&
+       (github.base_ref == 'develop' || github.base_ref == 'master') &&
+       !(github.head_ref == 'develop' &&
+         github.event.pull_request.head.repo.full_name == github.repository)) ||
+      (github.event_name == 'push' && github.ref == 'refs/heads/develop')
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7
@@ -100,7 +116,7 @@ jobs:
         run: pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
       - name: Install dependencies
         run: |
-          pip install -r requirements.txt -r requirements-test.txt
+          pip install -c constraints-openapi.txt -r requirements.txt -r requirements-test.txt
           pip install git+https://github.com/huggingface/diffusers
       - uses: actions/setup-node@v7
         with:

README.md

@@ -83,7 +83,7 @@ once; without it the run fails partway through with a 401/403 from the Hub.
 
 ## Drive it from an agent
 
-Then just ask. The agent has 59 tools covering the whole surface — the
+Then just ask. The agent has 62 tools covering the whole surface — the
 workflow catalog, the real diffusers pipeline signatures, the job queue, the
 gallery, the model cache:
 

constraints-openapi.txt

@@ -0,0 +1,8 @@
+# The FastAPI and Pydantic that ui/src/lib/generated/openapi.json is
+# generated under. Either one's release can change the document with no
+# change here, so CI installs these exactly (pip -c) and
+# scripts/dump_openapi.py refuses to write under anything else. To move
+# them: bump both lines, install them, run the dump and `npm run gen:api`,
+# and commit the four files together.
+fastapi==0.142.2
+pydantic==2.13.5

docs/ARCHITECTURE.md

@@ -49,6 +49,8 @@ one to open.
 | Reference name shape | `dw/reference_names.py`: `reference_name_errors` | The shape of every `asset:`, `prompt:` and `output:` name is checked before the queue, and existence is checked later. | `tests/test_reference_names.py::TestTheValidationPass::test_a_malformed_reference_is_refused_before_the_queue` |
 | Library reads | `dw/library.py`: `LibraryPath` | Reads go through the roots front to back, so an earlier name shadows a later one. | `tests/test_library_path.py::TestResolution::test_the_front_of_the_path_shadows_the_rest` |
 | Library writes | `dw/library.py`: `LibraryPath` | Writes go only to the front root, so saving something opened from a read-only root writes a copy. | `tests/test_library_path.py::TestConstruction::test_the_writable_root_comes_first`, `tests/test_library_sources.py::TestServer::test_saving_an_example_prompt_writes_a_copy` |
+| LoRA catalog matching | `dw/lora_catalog.py`: `matches`, `workflow_bases` | An entry fits a base only by exact repo id (and H3 partition); no alias or family match. | `tests/test_lora_catalog.py::TestMatching::test_the_base_must_match_exactly` |
+| Hub search | `dw/lora_hub.py`: `search_hub` | The Hub is searched only by `GET /api/loras/recommend`, never raises, and offers no pickle-only repo. | `tests/test_lora_hub.py::TestCandidates::test_a_pickle_only_repo_is_dropped`, `tests/test_lora_hub.py::TestFailure::test_a_hub_error_is_returned_not_raised` |
 | Packaged builtins | `dw/library.py`: `builtin_root`, `resolve_sub_workflow_reference` | `builtin:` names the packaged `dw/workflows/`, not the top-level `workflows/` examples. | `tests/test_sub_workflow_resolver.py::TestTheResolver::test_a_builtin_resolves_in_the_packaged_root` |
 | Sub-workflow paths | `dw/library.py`: `resolve_sub_workflow_reference`, `resolve_sub_workflow` | A sub-workflow path is confined to the root it resolves in. The search order is in the docstring of `resolve_sub_workflow`. | `tests/test_sub_workflow_resolver.py::TestTheResolver::test_a_path_escaping_its_root_is_refused` |
 | Workspace | `dw/workspace.py`: `resolve_workspace`, `set_workspace` | `--workspace` beats `DW_WORKSPACE`, which beats the setting, which beats a working directory that looks like a workspace, which beats `~/diffusers-workspace`. | `tests/test_workspace.py::TestResolution::test_a_flag_wins_over_everything`, `tests/test_workspace.py::TestResolution::test_a_bare_working_directory_falls_back_to_the_home_workspace` |
@@ -99,6 +101,7 @@ one to open.
 | Worker protocol | `dw/worker_protocol.py`: `parse_reply` | Every command and reply is a frozen dataclass that travels as a wire dict. | `tests/test_worker_messages.py::test_from_wire_inverts_to_wire`, `tests/test_worker_messages.py::test_an_unknown_reply_type_is_kept_whole_rather_than_raised` |
 | Persistent worker | `dw/worker.py`, `dw/worker_manager.py`, `dw/serve.py` | Jobs run in one spawned worker that keeps models loaded, so a change to engine code needs a server restart. | — |
 | Failed-run reporting | `dw/worker.py`, `dw/worker_protocol.py`: `Failed`, `Cancelled` | A failed or cancelled run's reply still carries the manifest of the steps that ran. | `tests/test_worker_execute.py::test_failure_carries_the_manifest_of_the_steps_that_ran`, `tests/test_worker_execute.py::test_cancellation_carries_the_manifest_too` |
+| Failure path redaction | `dw/path_redaction.py`: `redact_paths`, called by `dw/worker.py` | A failed run's message and traceback name a file under the job's asset roots or output directory by its `asset:`/`output:` reference, never its absolute server path. | `tests/test_worker_execute.py::test_a_failure_names_an_asset_by_reference_not_by_server_path`, `tests/test_path_redaction.py` |
 | Run-time warnings | `dw/events.py`: `emit_warning` | A warning found at run time is emitted as an event, so it reaches the caller and not just the log. | `tests/test_concat_videos.py::TestWarningsReachTheCaller::test_the_level_spread_warning_is_emitted_as_an_event` |
 
 ## Media and DSP
@@ -140,11 +143,30 @@ one to open.
 | `dw_mcp` stays torch-free | `dw_mcp/` | `dw_mcp` reaches `dw.serve` over HTTP and imports no `dw` module, because `dw/__init__.py` pulls in torch. | `tests/test_mcp_server.py::TestStartupWeight::test_the_server_starts_without_importing_the_engine` |
 | Tool surface | `dw_mcp/server.py`, `dw_mcp/tools_*.py` | Only these modules import the MCP SDK, and each tool body is a one-line call into a handler. | `tests/test_mcp_server.py::test_the_wiring_table_covers_every_registered_tool`, `tests/test_mcp_server.py::test_the_stated_tool_count_is_the_registered_one` |
 | Surface text budget | the tool docstrings in `dw_mcp/tools_*.py`, the instructions in `dw_mcp/server.py` | The instructions and each tool description stay at or under 2,048 characters, and the whole surface stays within its token budget. | `tests/test_mcp_server.py`: `SURFACE_BUDGET`, `CLIENT_TEXT_LIMIT`; `tests/test_mcp_server.py::test_no_text_the_agent_reads_is_cut_off_by_the_client`, `tests/test_mcp_server.py::test_the_tool_surface_fits_the_budget` |
-| Spending needs consent | `dw_mcp/diagnose.py` | `run_workflow` and `rerun_job` refuse until `acknowledged_cost` is set. | `tests/test_mcp_diagnose.py::test_run_refuses_without_an_acknowledged_cost`, `tests/test_mcp_diagnose.py::test_rerun_refuses_without_an_acknowledged_cost` |
+| Spending needs consent | each handler behind a tool that takes `acknowledged_cost` (`dw_mcp/diagnose.py`, `models.py`, `prompts.py`, `workspaces.py`) | Every tool that takes `acknowledged_cost` refuses until it is set; `delete_workspace`'s refusal is the server's 409. | `tests/test_mcp_twins.py::test_every_tool_that_takes_acknowledged_cost_refuses_without_it` |
+| Copies of engine rules | the constants and word lists in `dw_mcp/` and `dw/run.py` | `dw_mcp` cannot import its owners, so each copy equals its owner; a copy that needs no twin is deleted, not pinned. | `tests/test_mcp_twins.py` |
+| The CLI is a client of the server | `dw/run.py` | `dw.run` queues on `dw.serve` through `dw_mcp.client`, so `dw` imports `dw_mcp` and never the reverse. | `tests/test_mcp_server.py::TestStartupWeight::test_the_server_starts_without_importing_the_engine`, `tests/test_mcp_twins.py` |
+| Inline images | `dw/server/inline_media.py` | Fitting an image to a longest side and halving it under a base64 budget happens on the server, for `/image` and `/frames` alike; `dw_mcp` forwards the size and budget and imports no Pillow. | `tests/test_server_gallery_image.py`, `tests/test_mcp_server.py::TestStartupWeight::test_the_server_starts_without_importing_the_engine` |
+| Workflow patch | `dw/library.py`: `merge_patch`; `dw/server/routes/library.py`: `patch_workflow` | A merge patch is applied on the server under the lock `PUT` takes; no client reads, merges and writes back. | `tests/test_server_library_path.py::TestPatchWorkflow::test_patch_merges_onto_the_stored_definition` |
+| Deleting a job's run | `dw/server/jobs.py`: `JobManager.run_location`; `dw/server/outputs.py`: `delete_run_directory` | The server resolves a job's run directory against the root the job ran in, and refuses a job still queued or running. | `tests/test_server_jobs.py::TestDeleteAJobsRun::test_a_running_job_is_409` |
+| Level findings | `dw/server/assess.py`: `level_findings` | Gallery metadata reports level problems from `dw/audio_qc.py`'s thresholds; no client restates a threshold. | `tests/test_server_assess.py::TestMetadataLevelFindings::test_level_findings_read_audio_qc_thresholds` |
 | API errors | `dw_mcp/client.py` | An API failure becomes a message a person can act on, here and nowhere else. | `tests/test_mcp_client.py::test_a_400_surfaces_the_servers_detail_verbatim` |
 
 ## UI
 
 | Concept | Owner | Rule | Enforced by |
 | --- | --- | --- | --- |
 | The UI reads engine fields | `ui/src/lib/plan.ts`: `describePlan`, `ui/src/lib/results.ts`: `sectionBySubfolder` | The UI reads `plan`, `version` and `subfolder` as fields the server sends and derives nothing of its own, and it never sends `acknowledged_cost`. | `ui/src/lib/plan.test.ts`, `ui/src/lib/results.test.ts` |
+| Reference prefixes in the UI | `ui/src/lib/references.ts` | The only module in `ui/src` that spells a reference prefix; each equals `dw/references.py`'s. | `tests/test_ui_twins.py::test_the_ui_spells_every_reference_prefix_the_engine_does`; `prefix_literals` in `ui/scripts/arch-metrics.mjs` |
+| Output kinds on the job page | `dw/server/outputs.py`: `MEDIA_KINDS`, `output_kinds` | The page renders an output by the `output_kinds` the job detail carries, never by its extension. | `tests/test_server.py::test_a_job_reports_each_output_files_media_kind`, `ui/src/lib/pages/JobPage.test.ts` |
+| The editor's live reference check | `ui/src/lib/flow.ts`: `danglingReferenceDetails`, owned by `dw/previous_results.py` and `dw/for_each.py` | The editor warns as the author types, so it keeps a copy of the reference rules; one case file is run through both, the engine deciding each case. | `tests/test_ui_twins.py::test_the_engine_decides_each_shared_reference_case`, `ui/src/lib/flow.test.ts` |
+| Name segments in the UI | `ui/src/lib/names.ts`: `isNameSegment` | Workspaces, folders and stored names follow `dw/security.py`'s `WORKSPACE_NAME_PATTERN` and length cap, counted in code points. | `tests/test_ui_twins.py::test_the_engine_decides_each_shared_workspace_name_case`, `ui/src/lib/names.test.ts` |
+| UI copies of engine rules | the constants `tests/test_ui_twins.py` reads | A list the UI must hold equals its owner, read from the TS source; a rule tested on both sides reads one case file in `tests/fixtures/`. | `tests/test_ui_twins.py` |
+| UI overlays and suggestions | `ui/src/lib/ui/`: `ConfirmDialog`, `Modal`, `Popover`, `Suggest` | Bits UI is imported only here; pages and components use the wrappers, which own focus, Escape, outside presses and scroll lock. Overlay styles and the stacking scale (`--layer-*`) live in `ui/src/app.css`. | `ui/eslint.config.js` (`no-restricted-imports`), `ui/src/lib/ui/*.test.ts`, `ui/e2e/chrome.spec.ts` |
+| Escape on a page under an overlay | `ui/src/lib/ui/layers.svelte.ts`: `holdLayer`, `overlayOpen` | Every open overlay holds a layer; a page's own Escape handling acts only when none is open. | `ui/src/lib/pages/GalleryPage.test.ts` (an Escape that closes a confirm leaves the selection) |
+| The editors' document | `ui/src/lib/editorShell.svelte.ts`: `DocumentEditor` | The workflow and prompt editors keep one rule each for the view (remembered per editor), the JSON draft (a failed parse pins it), dirty against the last load or save, the save path, and the tab-close guard. | `ui/src/lib/editorShell.test.ts`, `ui/src/lib/pages/PromptEditorPage.test.ts`, `ui/src/lib/pages/EditorPage.test.ts` |
+| UI polling | `ui/src/lib/poll.ts`: `poll`, `sleep` | A page that refreshes on a timer starts it through `poll` and stops it with the function `poll` returns. | `ui/src/lib/poll.test.ts` |
+| Workspace state in the UI | `ui/src/lib/workspaceState.svelte.ts`: `workspace` | The current workspace is one `$state`; `api.ts` reads it from here, so the request layer imports no page state. | `import_cycles` in `ui/scripts/arch-metrics.mjs` |
+| The UI's response contract | `dw/server/api_models.py`: `ApiModel`, `send_rejected_responses`; `scripts/dump_openapi.py` | Every JSON route `api.ts` calls declares a response model, and `types.ts` re-exports the types generated from it (`ui/src/lib/generated/`). The payload does not change: an absent key stays absent (`response_model_exclude_unset`), a sometimes-sent key is `sometimes()`. Runtime is lenient - an undeclared key passes and a rejected response is logged and sent as built; tests, the e2e server and the dump run strict (`DW_STRICT_RESPONSES=1`). The document is generated under the FastAPI and Pydantic `constraints-openapi.txt` pins: CI installs them, and the dump refuses to write under others. | `tests/test_api_contract.py`, `tests/test_ui_twins.py::test_every_json_route_the_ui_calls_declares_its_response`, CI's "Response contract" step |
+| Where `--live` appears | `ui/src/app.css` (its header) | The state colour marks machine state only, in the files `ui/scripts/design-rules.test.ts` lists. | `ui/scripts/design-rules.test.ts` |
+| UI architecture ratchet | `ui/scripts/arch-metrics.mjs` | No metric rises above `docs/stabilization/ui/baseline.json`. | the `ui` job in `.github/workflows/ci.yml`; `npm run preflight` |