From 37bd3bc70f598e9731694830628fe4420926d2c8 Mon Sep 17 00:00:00 2001 From: TURFPTAx Date: Mon, 25 May 2026 10:50:51 -0500 Subject: [PATCH 01/47] data: rewrite legacy converter with packet matching + add combine_csvs The legacy capture .txt converter previously emitted one row per packet with (device_id, ticks, data) columns -- not directly trainable. Rewrite it to deque-match the 3 SensorBand banks against LASK5 label packets by recency, producing the standard 12-sensor + 4-label paired-CSV shape plus per-stream timestamp columns. Dedup back-to-back identical records. Also lift combine_csvs() into this module so cli `train` and the web `/api/train` route can share one row-wise concatenation helper. dataset.py: detect_columns() now excludes any column with "Timestamp" in its name so the new Sensor_Timestamp / Label_Timestamp columns don't get fed to the model as features. Co-Authored-By: Claude Opus 4.7 (1M context) --- pc/src/openmuscle/data/converter.py | 102 ++++++++++++++++++++++++---- pc/src/openmuscle/data/dataset.py | 8 ++- 2 files changed, 94 insertions(+), 16 deletions(-) diff --git a/pc/src/openmuscle/data/converter.py b/pc/src/openmuscle/data/converter.py index f159d2b..c31c39c 100644 --- a/pc/src/openmuscle/data/converter.py +++ b/pc/src/openmuscle/data/converter.py @@ -1,16 +1,28 @@ -"""Convert legacy capture formats to standard CSV.""" +"""Convert legacy capture formats to standard CSV with packet matching.""" import ast import csv import os +from collections import deque from pathlib import Path +# Legacy device IDs for SensorBand (3 banks × 4 sensors = 12 total) +SENSOR_IDS = ['OM-SB-V1-C.0', 'OM-SB-V1-C.1', 'OM-SB-V1-C.2'] +LABEL_ID = 'OM-LASK5' + + def convert_legacy_capture(input_path: str, output_path: str) -> int: - """Convert a legacy capture_*.txt file to standard CSV format. + """Convert a legacy capture_*.txt file to matched CSV format. + + Matches sensor packets from 3 SensorBand banks with LASK5 label packets + by temporal proximity, producing rows with 12 sensor values + 4 labels. Legacy format: one Python dict repr per line, e.g.: - {'id': 'OM-LASK5', 'ticks': 164587, 'time': (2000, 1, 1, ...), 'data': [-30, -35, -30, -37]} + {'id': 'OM-SB-V1-C.0', 'ticks': 164587, 'data': [2686.1, 2926.1, 2519.7, 2653.1], ...} + + Output CSV columns: + Sensor_0..Sensor_11, Sensor_Timestamp, Label_0..Label_3, Label_Timestamp Args: input_path: path to legacy .txt capture file @@ -20,11 +32,19 @@ def convert_legacy_capture(input_path: str, output_path: str) -> int: Number of rows written """ Path(output_path).parent.mkdir(parents=True, exist_ok=True) + + buffers = {sid: deque(maxlen=5) for sid in SENSOR_IDS} + buffers[LABEL_ID] = deque(maxlen=5) + + header = ([f"Sensor_{i}" for i in range(12)] + ['Sensor_Timestamp'] + + [f"Label_{i}" for i in range(4)] + ['Label_Timestamp']) + rows_written = 0 + last_record = None with open(input_path, "r") as f_in, open(output_path, "w", newline="") as f_out: writer = csv.writer(f_out) - header_written = False + writer.writerow(header) for line in f_in: line = line.strip() @@ -38,17 +58,71 @@ def convert_legacy_capture(input_path: str, output_path: str) -> int: if not isinstance(pkt, dict) or "data" not in pkt: continue - device_id = pkt.get("id", "unknown") - ticks = pkt.get("ticks", 0) - data = pkt["data"] + pkt_id = pkt.get("id") + if pkt_id not in buffers: + continue + + buffers[pkt_id].append(pkt) + + # Try to match: need at least one packet from each sensor bank + labels + if not all(buffers[sid] for sid in SENSOR_IDS): + continue + if not buffers[LABEL_ID]: + continue + + # Combine latest from each sensor bank + sensor_values = [] + sensor_times = [] + for sid in SENSOR_IDS: + latest = buffers[sid][-1] + sensor_values.extend(latest.get('data', [])) + sensor_times.append(latest.get('rec_time', 0)) + + label_pkt = buffers[LABEL_ID][-1] + labels = label_pkt.get('data', []) + label_time = label_pkt.get('rec_time', 0) + sensor_time = min(sensor_times) - if not header_written: - n = len(data) - header = ["device_id", "ticks"] + [f"value_{i}" for i in range(n)] - writer.writerow(header) - header_written = True + record = sensor_values + [sensor_time] + labels + [label_time] - writer.writerow([device_id, ticks] + data) - rows_written += 1 + # Skip duplicate records (same data as last write) + if record == last_record: + continue + + if len(record) == 18: + writer.writerow(record) + rows_written += 1 + last_record = record return rows_written + + +def combine_csvs(csv_paths: list[str], output_path: str) -> int: + """Concatenate multiple CSVs (same schema) into one file. + + Args: + csv_paths: list of CSV file paths to combine + output_path: path for the combined output CSV + + Returns: + Total number of data rows written + """ + Path(output_path).parent.mkdir(parents=True, exist_ok=True) + total = 0 + header_written = False + + with open(output_path, "w", newline="") as f_out: + writer = csv.writer(f_out) + + for csv_path in csv_paths: + with open(csv_path, "r") as f_in: + reader = csv.reader(f_in) + header = next(reader) + if not header_written: + writer.writerow(header) + header_written = True + for row in reader: + writer.writerow(row) + total += 1 + + return total diff --git a/pc/src/openmuscle/data/dataset.py b/pc/src/openmuscle/data/dataset.py index 2b48fe9..b23d951 100644 --- a/pc/src/openmuscle/data/dataset.py +++ b/pc/src/openmuscle/data/dataset.py @@ -19,8 +19,12 @@ def detect_columns(df: pd.DataFrame) -> tuple[list[str], list[str]]: Returns: (sensor_columns, label_columns) """ - sensor_cols = [c for c in df.columns if c.startswith("R") or c.startswith("Sensor_")] - label_cols = [c for c in df.columns if c.startswith("label_") or c.startswith("Label_")] + sensor_cols = [c for c in df.columns + if (c.startswith("R") or c.startswith("Sensor_")) + and "Timestamp" not in c] + label_cols = [c for c in df.columns + if (c.startswith("label_") or c.startswith("Label_")) + and "Timestamp" not in c] if not sensor_cols: raise ValueError("No sensor columns found (expected R*C* or Sensor_* prefix)") From 49ec62321293e2e9f5f38d7e3283c5a72ebf2f3b Mon Sep 17 00:00:00 2001 From: TURFPTAx Date: Mon, 25 May 2026 10:51:00 -0500 Subject: [PATCH 02/47] web: hand forwarder -- thumb on ch1, piston-reverse for LASK5 parity Rewrite _forward_to_hand() to match the OpenHand firmware's anatomical channel layout (FINGER_CHANNELS = [1, 3, 5, 7, 9] -> thumb, index, middle, ring, pinky). The previous implementation appended joystick X as the 5th angle, which lined up with channel 9 (pinky) instead of channel 1 (thumb) -- the hand was getting "thumb=predicted_pinky" and "pinky=joystick_x", silently inverted. Also reverse the piston order (P4..P1) on the way out so the PC path matches the LASK5 ESP-NOW path, whose default 'L5' device config has reverse=True. Without this reversal the same prediction array drives different fingers depending on whether the packet came via the model or via direct ESP-NOW from the LASK5. Add rate-limited logs (first hit + every 500th) so the operator can verify forwarding is actually happening without strace. Co-Authored-By: Claude Opus 4.7 (1M context) --- pc/src/openmuscle/web/state.py | 84 +++++++++++++++++++++++----------- 1 file changed, 58 insertions(+), 26 deletions(-) diff --git a/pc/src/openmuscle/web/state.py b/pc/src/openmuscle/web/state.py index 4dde98a..83e7d10 100644 --- a/pc/src/openmuscle/web/state.py +++ b/pc/src/openmuscle/web/state.py @@ -400,24 +400,22 @@ def _write_jsonl(stream: Optional[IO], pkt: OpenMusclePacket): def _forward_to_hand(self, pred: list): """Send the prediction to the robot hand as a `PC,...` UDP datagram. - Builds 5 servo angles in 0..179 from the 4 piston predictions (assumed - normalized 0..1; clamped) plus the most recent LASK5 joystick X as the - 5th. The hand's `'PC'` device config uses linear 0..179 -> 0..179 - mapping, so values land directly on servo angles. + Builds 5 servo angles in 0..179. Channel order on the hand + (FINGER_CHANNELS = [1, 3, 5, 7, 9]) is anatomically: + channel 1 -> thumb + channel 3 -> index + channel 5 -> middle + channel 7 -> ring + channel 9 -> pinky + + The LASK5 has 4 pistons (the 4 closing fingers) and a joystick. + We map joystick X -> thumb, pistons 0..3 -> index..pinky. + The hand's 'PC' device config uses linear 0..179 -> 0..179, so values + land directly on servo angles. """ - # Pistons -> 0..179, assuming model output is normalized 0..1. - # Anything else gets clamped, which is the right failure mode -- - # bracelet finger goes to extreme rather than 4000-degree angle. - angles = [] - for v in pred[:4]: - try: - v = max(0.0, min(1.0, float(v))) - except Exception: - v = 0.0 - angles.append(int(v * 179)) - - # 5th finger = joystick X from the most recent LASK5 packet. Range - # 0..4095 -> 0..179. Default to center (90) if no LASK5 has been seen. + # Thumb (channel 1) = joystick X from the most recent LASK5 packet. + # Range 0..4095 -> 0..179. Default to center (90) if no LASK5 has been + # seen yet (so the thumb sits in a neutral pose instead of slamming open). joy_x = None for d in self.devices.values(): if d.device_type == "lask5" and d.last_joystick: @@ -425,23 +423,57 @@ def _forward_to_hand(self, pred: list): if isinstance(jx, (int, float)): joy_x = jx break - if joy_x is None: - angles.append(90) - else: - angles.append(max(0, min(179, int((joy_x / 4095.0) * 179)))) + thumb_angle = 90 if joy_x is None else max(0, min(179, int((joy_x / 4095.0) * 179))) + + # Index..pinky (channels 3, 5, 7, 9) from pistons 0..3. + # Model output is assumed normalized 0..1; anything else gets clamped, + # which is the right failure mode -- finger goes to extreme rather + # than commanding a 4000-degree servo angle. + finger_angles = [] + for v in pred[:4]: + try: + v = max(0.0, min(1.0, float(v))) + except Exception: + v = 0.0 + finger_angles.append(int(v * 179)) + + # The hand's 'PC' device config has reverse=False, but the LASK5's + # native ESPNow path uses the 'default' / 'L5' config (reverse=True) + # which flips the piston order before mapping to FINGER_CHANNELS. + # To match that mapping from our PC path, we reverse the pistons + # ourselves: P1 -> index, P2 -> middle, P3 -> ring, P4 -> pinky. + # (Documented in DEVICES of the hand firmware, archived 2026-05-14.) + angles = [thumb_angle] + finger_angles[::-1] # [thumb, P4, P3, P2, P1] # Build the CSV the hand expects: 'PC,a1,a2,a3,a4,a5' payload = ("PC," + ",".join(str(a) for a in angles)).encode("utf-8") + # Rate-limited log so we can SEE whether forwarding is working. + # First time + every 500th packet: log success/failure to the buffer + # so the operator can debug without strace. + self._hand_forward_count = getattr(self, "_hand_forward_count", 0) + 1 + log_now = (self._hand_forward_count == 1 + or self._hand_forward_count % 500 == 0) try: if self._hand_sock is None: self._hand_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self._hand_sock.setblocking(False) - self._hand_sock.sendto(payload, self.hand_target) - except Exception: - # Non-fatal: hand might be offline / on a different subnet. - # We don't spam logs since this fires per FlexGrid packet. - pass + n = self._hand_sock.sendto(payload, self.hand_target) + if log_now: + self.log_buffer.info("inference", + "hand forward #{}: sent {} bytes to {}:{} -> {!r}".format( + self._hand_forward_count, n, + self.hand_target[0], self.hand_target[1], + payload.decode("utf-8", errors="replace"))) + except Exception as e: + # Always log the first failure so the operator sees it; rate-limit + # subsequent ones (every 500) so we don't spam. + if log_now or not getattr(self, "_hand_forward_error_logged", False): + self.log_buffer.warn("inference", + "hand forward #{} FAILED: {} ({!r}) -> target={}".format( + self._hand_forward_count, type(e).__name__, str(e), + self.hand_target)) + self._hand_forward_error_logged = True async def _broadcast_latest_frames(self): """Push the latest frame for each device to all WS clients.""" From cd2ec75f23e2088fc752d3584b5c321389318c99 Mon Sep 17 00:00:00 2001 From: TURFPTAx Date: Mon, 25 May 2026 10:51:09 -0500 Subject: [PATCH 03/47] web: file-manager reveal + retroactive session<->capture linking Two additive endpoints for the Captures and Sessions panels: POST /api/reveal {name?} Opens captures_dir in the OS file manager. With a capture name, highlights that specific .csv inside its folder (Explorer /select on Windows, open -R on macOS, xdg-open on Linux). Whitelist-guarded via state.capture_path(). POST/DELETE /api/sessions/{id}/captures{/name} The "I forgot to start a session before recording" recovery path. Bulk-add or remove existing captures from a session after the fact. Updates both the session JSON's `captures` list AND the capture's .meta.json (tag `session:` + auto.session_id) so the captures filter and session expansion stay consistent. Co-Authored-By: Claude Opus 4.7 (1M context) --- pc/src/openmuscle/web/app.py | 146 +++++++++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) diff --git a/pc/src/openmuscle/web/app.py b/pc/src/openmuscle/web/app.py index 68f1c7f..ba22a3d 100644 --- a/pc/src/openmuscle/web/app.py +++ b/pc/src/openmuscle/web/app.py @@ -9,6 +9,9 @@ # string and falls back to treating the param as a query string field). import asyncio +import shutil +import subprocess +import sys from contextlib import asynccontextmanager from pathlib import Path from typing import Optional @@ -24,6 +27,48 @@ STATIC_DIR = Path(__file__).parent / "static" +def _reveal_path_in_file_manager(path: Path, select_file: bool) -> None: + """Open `path` in the OS file manager. If `select_file=True` and the + platform supports it, highlight the file inside its parent folder + rather than just opening the folder. Raises RuntimeError on failure. + + Whitelist-guarded by the caller: this function does NOT verify that + `path` is inside captures_dir. That check happens in the route. + """ + if not path.exists(): + raise RuntimeError(f"Path does not exist: {path}") + + # Explorer / Finder / xdg-open all need *absolute* paths -- they don't + # inherit our CWD predictably and a relative path like + # "data/raw/merged/foo.csv" silently fails with "location not found". + path = path.resolve() + + try: + if sys.platform.startswith("win"): + if select_file and path.is_file(): + # explorer /select,"C:\full\path\file.csv" -- highlights file + subprocess.Popen(["explorer", f"/select,{path}"]) + else: + # Open the folder itself + folder = path if path.is_dir() else path.parent + subprocess.Popen(["explorer", str(folder)]) + elif sys.platform == "darwin": + if select_file and path.is_file(): + subprocess.Popen(["open", "-R", str(path)]) + else: + folder = path if path.is_dir() else path.parent + subprocess.Popen(["open", str(folder)]) + else: + # Linux / other -- xdg-open only opens directories cleanly + opener = shutil.which("xdg-open") or shutil.which("gio") + if opener is None: + raise RuntimeError("No file-manager opener found (xdg-open / gio)") + folder = path if path.is_dir() else path.parent + subprocess.Popen([opener, str(folder)]) + except Exception as e: + raise RuntimeError(f"Failed to open file manager: {e}") + + def create_app(udp_port: int = 3141, captures_dir: Optional[str] = None, model_path: Optional[str] = None, hand_target: Optional[tuple] = None) -> FastAPI: @@ -168,6 +213,31 @@ async def download_capture(name: str): raise HTTPException(status_code=404, detail="Capture not found") return FileResponse(p, filename=p.name, media_type="text/csv") + class RevealBody(BaseModel): + # If empty/None -> just open captures_dir. Otherwise must be a + # capture name whitelisted by state.capture_path(). + name: Optional[str] = None + + @app.post("/api/reveal") + async def reveal_in_folder(body: RevealBody): + """Open the captures folder (and optionally highlight a specific + capture) in the OS file manager. Local-only convenience; the server + is intended for localhost use.""" + if body.name: + p = state.capture_path(body.name) + if p is None: + raise HTTPException(status_code=404, detail="Capture not found") + target = p + select = True + else: + target = state.captures_dir + select = False + try: + _reveal_path_in_file_manager(target, select_file=select) + except RuntimeError as e: + raise HTTPException(status_code=500, detail=str(e)) + return {"opened": str(target), "selected": select} + @app.delete("/api/captures/{name}") async def delete_capture(name: str): ok = state.delete_capture(name) @@ -358,6 +428,82 @@ async def delete_session_endpoint(session_id: str, unlink_captures: bool = True) raise HTTPException(status_code=404, detail="Session not found") return {"deleted": session_id} + # ----- REST: retroactive session<->capture linking ----- + # + # A capture made *outside* an active session can be added to one + # afterwards, and vice versa removed. This is the "I forgot to start a + # session before recording" recovery path. We update both: + # 1. the session JSON's `captures` list (authoritative) + # 2. the capture's meta sidecar (tag `session:` + auto.session_id) + # so the captures-panel filter, the past-sessions expansion, and any + # future export all agree on which session a capture belongs to. + + class LinkCapturesBody(BaseModel): + capture_names: list[str] # bulk add + + @app.post("/api/sessions/{session_id}/captures") + async def add_captures_to_session(session_id: str, body: LinkCapturesBody): + s = state.get_session(session_id) + if s is None: + raise HTTPException(status_code=404, detail="Session not found") + + tag = "session:" + session_id + added, skipped = [], [] + for name in body.capture_names: + if state.capture_path(name) is None: + skipped.append({"name": name, "reason": "capture not found"}) + continue + if name in s.get("captures", []): + skipped.append({"name": name, "reason": "already in session"}) + continue + try: + state.link_capture_to_session(session_id, name) + # Update the capture's meta so the tag-based filter + any + # future export sees this capture as part of the session. + meta = state.read_capture_meta(name) or {} + tags = list(meta.get("tags") or []) + if tag not in tags: + tags.append(tag) + state.write_capture_meta(name, { + "tags": tags, + "auto": {"session_id": session_id}, + }) + added.append(name) + except Exception as e: + skipped.append({"name": name, "reason": str(e)}) + + return { + "added": added, + "skipped": skipped, + "session": state.get_session(session_id), + } + + @app.delete("/api/sessions/{session_id}/captures/{capture_name}") + async def remove_capture_from_session(session_id: str, capture_name: str): + s = state.get_session(session_id) + if s is None: + raise HTTPException(status_code=404, detail="Session not found") + if capture_name not in s.get("captures", []): + raise HTTPException(status_code=404, detail="Capture not in session") + try: + state.unlink_capture_from_session(session_id, capture_name) + # Strip the session tag + clear auto.session_id, but ONLY for this + # session (leave any other 'session:xxx' tags alone -- though by + # the data model a capture should only ever belong to one session). + tag = "session:" + session_id + meta = state.read_capture_meta(capture_name) or {} + new_tags = [t for t in (meta.get("tags") or []) if t != tag] + state.write_capture_meta(capture_name, { + "tags": new_tags, + "auto": {"session_id": None}, + }) + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + return { + "removed": capture_name, + "session": state.get_session(session_id), + } + return app From ab8be83e55075f3cc02861b6c05cb6e6198d21b2 Mon Sep 17 00:00:00 2001 From: TURFPTAx Date: Mon, 25 May 2026 10:51:19 -0500 Subject: [PATCH 04/47] web: Studio UI redesign with pipeline strip + stage-based layout Rebrand the web UI from "OpenMuscle Live" to "OpenMuscle Studio" and restructure the page around the data-pipeline narrative: SENSOR -> LABEL -> CAPTURE -> MODEL -> HAND A topbar pipeline-strip shows live status for each stage with click- to-scroll anchors. Body splits into numbered stages: (1) Live -- heatmap + GT-vs-Predicted comparator hero, (2) Capture, (3) Models, (4) Output. The Devices list collapses into a thin left rail. styles.css gets the matching grid-template + new pipe-pill / stage-* classes. app.js adds renderPipelinePills() (called every WS tick) and moves the per-panel renderers into the new stage containers. No behavior changes -- same WS contract, same REST surface. Pure restructure + visual rework. Co-Authored-By: Claude Opus 4.7 (1M context) --- pc/src/openmuscle/web/static/app.js | 387 +++++++++++++++++++- pc/src/openmuscle/web/static/index.html | 420 +++++++++++++-------- pc/src/openmuscle/web/static/styles.css | 463 ++++++++++++++++++++++-- 3 files changed, 1091 insertions(+), 179 deletions(-) diff --git a/pc/src/openmuscle/web/static/app.js b/pc/src/openmuscle/web/static/app.js index 474637c..7e86cc9 100644 --- a/pc/src/openmuscle/web/static/app.js +++ b/pc/src/openmuscle/web/static/app.js @@ -18,10 +18,31 @@ const selStatus = document.getElementById('captures-sel-status'); const checkAll = document.getElementById('captures-check-all'); const modelsBody = document.getElementById('models-body'); const modelsCount = document.getElementById('models-count'); +const openFolderBtn = document.getElementById('captures-open-folder'); + +// Ask the server to open the captures folder in the OS file manager. +// If `name` is given, highlight that capture file inside the folder. +async function revealCaptureFolder(name) { + try { + const r = await fetch('/api/reveal', { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({name: name || null}), + }); + if (!r.ok) throw new Error(await readError(r)); + } catch (e) { + alert('Could not open folder: ' + e.message); + } +} + +if (openFolderBtn) { + openFolderBtn.onclick = () => revealCaptureFolder(null); +} // Per-user pick preferences that survive a refresh const STORE_SENSOR = 'om.sensor_device_id'; const STORE_LABEL = 'om.label_device_id'; +const STORE_HAND = 'om.hand_target'; // last successfully-applied "host:port" — auto-restored on next launch // Set of capture filenames currently checked in the table const selectedCaptures = new Set(); @@ -41,6 +62,11 @@ function connectWS() { ws.onopen = () => { wsStatus.textContent = 'connected'; wsStatus.className = 'badge online'; + // Re-arm the hand-target auto-restore: every fresh WS connect (which + // includes server restarts) gets a chance to re-apply the saved hand + // target. Otherwise the operator has to remember to click Apply + // after every `openmuscle web` restart. + handTargetRestoreAttempted = false; }; ws.onclose = () => { wsStatus.textContent = 'disconnected'; @@ -84,6 +110,11 @@ function handleTick(msg) { const lask = lastDevices.find(d => d.device_type === 'lask5'); renderLask(lask); renderInference(msg.inference); + // Comparator + top-bar pipeline strip are Studio-shell additions. + // They derive everything from the per-tick snapshot, so they update + // in lockstep with the underlying bars and the WS message. + renderResiduals(lask, msg.inference); + renderPipelinePills(msg, lask); } // ---------- device list ---------- @@ -370,6 +401,7 @@ function renderActiveSession() { ${armBit}${subj} · ${s.capture_count || 0} captures · ${formatUptime(dur)}${gestures}
+
@@ -377,6 +409,8 @@ function renderActiveSession() { ${s.notes ? `
${escapeHtml(s.notes)}
` : ''} `; document.getElementById('session-end-btn').onclick = endSession; + const addBtn = document.getElementById('active-session-add-btn'); + if (addBtn) addBtn.onclick = () => openLinkModal(activeSession); sessionStartBtn.disabled = true; sessionStartBtn.title = 'End the current session before starting a new one'; capturesFilterLabel.textContent = `· filtered to ${s.name || s.id}`; @@ -400,6 +434,137 @@ async function refreshPastSessions() { } catch (e) { /* best-effort */ } } +// Sessions whose capture list is currently expanded in the UI. Persisted +// across re-renders (refreshPastSessions can fire on its own) so a poll +// doesn't collapse what the user just opened. +const expandedSessions = new Set(); + +// ---------- Add-captures-to-session picker modal ---------- +// +// Lets the operator retroactively assign past recordings (made without an +// active session) to a session. The picker shows every capture NOT +// currently linked to the target session, with checkboxes for bulk add. +// +// Wires up: +// - "+ Add captures" button in each past-session card +// - "×" remove button on each capture in the expanded view + +const linkModal = document.getElementById('link-modal'); +const linkSessionName = document.getElementById('link-session-name'); +const linkCaptureList = document.getElementById('link-capture-list'); +const linkAddBtn = document.getElementById('link-add-btn'); +let linkSessionId = null; // current session being edited +const linkSelected = new Set(); // capture names currently checked + +function openLinkModal(session) { + linkSessionId = session.id; + linkSelected.clear(); + linkSessionName.textContent = session.name || session.id; + linkAddBtn.disabled = true; + linkAddBtn.textContent = 'Add 0 captures'; + linkCaptureList.innerHTML = '
Loading captures…
'; + linkModal.classList.add('open'); + linkModal.setAttribute('aria-hidden', 'false'); + + // Fetch the full capture list, filter out ones already in this session. + fetch('/api/captures') + .then(r => r.ok ? r.json() : Promise.reject('fetch failed')) + .then(list => { + const alreadyLinked = new Set(session.captures || []); + const candidates = list.filter(c => !alreadyLinked.has(c.name)); + if (!candidates.length) { + linkCaptureList.innerHTML = '
All captures are already in this session.
'; + return; + } + // Render rows with checkbox + name + meta summary + (if linked + // to a different session) an annotation so the operator doesn't + // accidentally yank a capture out of another session. + linkCaptureList.innerHTML = candidates.map(c => { + const meta = c.meta || {}; + const otherSession = (meta.tags || []).find(t => t.startsWith('session:')); + const otherNote = otherSession + ? `⚠ ${escapeHtml(otherSession)}` + : ''; + const kb = (c.size_bytes / 1024).toFixed(1); + return ``; + }).join(''); + linkCaptureList.querySelectorAll('input[type=checkbox]').forEach(cb => { + cb.onchange = () => { + if (cb.checked) linkSelected.add(cb.dataset.name); + else linkSelected.delete(cb.dataset.name); + const n = linkSelected.size; + linkAddBtn.disabled = (n === 0); + linkAddBtn.textContent = `Add ${n} capture${n === 1 ? '' : 's'}`; + }; + }); + }) + .catch(err => { + linkCaptureList.innerHTML = '
Could not load captures.
'; + console.warn('link picker fetch:', err); + }); +} + +function closeLinkModal() { + linkModal.classList.remove('open'); + linkModal.setAttribute('aria-hidden', 'true'); + linkSessionId = null; + linkSelected.clear(); +} + +linkModal.querySelectorAll('[data-close]').forEach(el => { + el.addEventListener('click', closeLinkModal); +}); +document.addEventListener('keydown', e => { + if (e.key === 'Escape' && linkModal.classList.contains('open')) closeLinkModal(); +}); + +linkAddBtn.onclick = async () => { + if (!linkSessionId || linkSelected.size === 0) return; + linkAddBtn.disabled = true; + linkAddBtn.textContent = 'Adding…'; + try { + const r = await fetch(`/api/sessions/${encodeURIComponent(linkSessionId)}/captures`, { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({capture_names: [...linkSelected]}), + }); + if (!r.ok) throw new Error(await readError(r)); + const result = await r.json(); + if ((result.skipped || []).length) { + // Surface skips inline -- e.g. "already in another session" + console.warn('some captures skipped:', result.skipped); + } + closeLinkModal(); + await refreshPastSessions(); + await refreshCaptures(); + } catch (e) { + alert('Add failed: ' + (e.message || e)); + linkAddBtn.disabled = false; + const n = linkSelected.size; + linkAddBtn.textContent = `Add ${n} capture${n === 1 ? '' : 's'}`; + } +}; + +async function removeCaptureFromSession(sessionId, captureName) { + if (!confirm(`Remove ${captureName} from this session?\n(The capture file itself stays — just the link is cleared.)`)) return; + try { + const r = await fetch( + `/api/sessions/${encodeURIComponent(sessionId)}/captures/${encodeURIComponent(captureName)}`, + {method: 'DELETE'} + ); + if (!r.ok) throw new Error(await readError(r)); + await refreshPastSessions(); + await refreshCaptures(); + } catch (e) { + alert('Remove failed: ' + (e.message || e)); + } +} + function renderPastSessions() { if (!pastSessions.length) { pastSessionsList.innerHTML = '
No past sessions yet.
'; @@ -408,21 +573,59 @@ function renderPastSessions() { pastSessionsList.innerHTML = pastSessions.map(s => { const dur = (s.ended_at && s.started_at) ? Math.floor(s.ended_at - s.started_at) : null; const armBit = s.arm ? escapeHtml(s.arm) + ' arm' : '—'; - const captures = s.capture_count || (s.captures || []).length; - return `
-
+ const captureList = Array.isArray(s.captures) ? s.captures : []; + const captureCount = s.capture_count != null ? s.capture_count : captureList.length; + const isOpen = expandedSessions.has(s.id); + const caret = captureList.length ? (isOpen ? '▾' : '▸') : '·'; + // The captures sub-list is a sibling div, toggled by .hidden. We + // render it eagerly (with .hidden if closed) so the open/close + // animation isn't required and so screen readers can find it. + const capturesInner = captureList.length + ? captureList.map(name => ` +
  • + ${escapeHtml(name)} + + + + download + + +
    • `).join('') + : '
  • No captures linked to this session.
    • '; + + return `
    +
    + ${caret} ${escapeHtml(s.name || s.id)} - ${armBit} · ${escapeHtml(s.subject || '—')} · ${captures} captures${dur != null ? ' · ' + formatUptime(dur) : ''} + ${armBit} · ${escapeHtml(s.subject || '—')} · ${captureCount} captures${dur != null ? ' · ' + formatUptime(dur) : ''}
    +
    ${s.notes ? `
    ${escapeHtml(s.notes)}
    ` : ''} +
      ${capturesInner}
    `; }).join(''); + + // Stop session-action buttons from triggering the row-toggle handler + pastSessionsList.querySelectorAll('.session-actions button').forEach(btn => { + btn.addEventListener('click', e => e.stopPropagation()); + }); + + // Toggle expand/collapse when the session header row is clicked + pastSessionsList.querySelectorAll('[data-toggle-session]').forEach(head => { + head.onclick = () => { + const sid = head.dataset.toggleSession; + if (expandedSessions.has(sid)) expandedSessions.delete(sid); + else expandedSessions.add(sid); + renderPastSessions(); + }; + }); + pastSessionsList.querySelectorAll('button[data-delete-session]').forEach(btn => { btn.onclick = async () => { const sid = btn.dataset.deleteSession; @@ -435,6 +638,34 @@ function renderPastSessions() { } catch (e) { alert('Delete failed: ' + e.message); } }; }); + + // Per-capture actions inside the expanded list + pastSessionsList.querySelectorAll('button[data-reveal-cap]').forEach(btn => { + btn.onclick = (e) => { + e.stopPropagation(); + revealCaptureFolder(btn.dataset.revealCap); + }; + }); + pastSessionsList.querySelectorAll('button[data-edit-cap]').forEach(btn => { + btn.onclick = (e) => { + e.stopPropagation(); + openMetaModal(btn.dataset.editCap); + }; + }); + pastSessionsList.querySelectorAll('button[data-unlink-cap]').forEach(btn => { + btn.onclick = (e) => { + e.stopPropagation(); + removeCaptureFromSession(btn.dataset.fromSession, btn.dataset.unlinkCap); + }; + }); + pastSessionsList.querySelectorAll('button[data-add-to-session]').forEach(btn => { + btn.onclick = (e) => { + e.stopPropagation(); + const sid = btn.dataset.addToSession; + const session = pastSessions.find(s => s.id === sid); + if (session) openLinkModal(session); + }; + }); } pastSessionsToggle.onclick = () => { @@ -643,6 +874,7 @@ function renderCaptures(list) { ${escapeHtml(date)} + download @@ -669,6 +901,9 @@ function renderCaptures(list) { capturesBody.querySelectorAll('button[data-edit]').forEach(btn => { btn.onclick = () => openMetaModal(btn.dataset.edit); }); + capturesBody.querySelectorAll('button[data-reveal]').forEach(btn => { + btn.onclick = () => revealCaptureFolder(btn.dataset.reveal); + }); updateSelectionStatus(); } @@ -1035,7 +1270,43 @@ function renderInference(inf) { }); } +// One-shot: if the server has no hand_target on first snapshot but we have +// one saved in localStorage, auto-apply it so launching `openmuscle web` +// doesn't lose the address every time. UDP-only (the only protocol we +// support); port defaults to 3145. +let handTargetRestoreAttempted = false; +function maybeRestoreHandTarget(inf) { + if (handTargetRestoreAttempted) return; + if (!inf) return; // wait for first inference snapshot + handTargetRestoreAttempted = true; // one-shot regardless of outcome + if (inf.hand_target) return; // server already has one (e.g. --hand on CLI) + const saved = localStorage.getItem(STORE_HAND); + if (!saved) return; + autoApplyHandTarget(saved); +} + +async function autoApplyHandTarget(raw) { + let host = raw, port = 3145; + if (raw.includes(':')) { + const idx = raw.lastIndexOf(':'); + host = raw.slice(0, idx); + const portN = parseInt(raw.slice(idx + 1), 10); + if (Number.isFinite(portN) && portN > 0 && portN < 65536) port = portN; + } + try { + await fetch('/api/inference/hand', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ host, port }), + }); + } catch (e) { + console.warn('hand target auto-restore failed', e); + } +} + function renderInferenceControls(inf) { + maybeRestoreHandTarget(inf); + const hasModel = !!(inf && inf.model); const enabled = !!(inf && inf.enabled); @@ -1111,6 +1382,10 @@ async function applyHandTarget() { body: JSON.stringify({ host, port }), }); if (!r.ok) throw new Error(await readError(r)); + // Persist so next launch auto-restores. Clear on explicit empty + // so the operator can "forget" the target deliberately. + if (host) localStorage.setItem(STORE_HAND, raw); + else localStorage.removeItem(STORE_HAND); // Force the snapshot side to refresh by clearing the cache so the // next tick syncs the (possibly normalized) value back into the input. lastSnapshotHand = undefined; @@ -1124,6 +1399,110 @@ inferHandInput.addEventListener('keydown', (e) => { if (e.key === 'Enter') applyHandTarget(); }); +// ---------- Studio shell: comparator residuals (Δ) ---------- + +// Compute per-piston residual (predicted - ground_truth) and write it into +// the .delta-row elements in the comparator. Color-codes by direction so +// the operator can see at a glance whether the model is over- or under- +// shooting each finger. +// +// CLOSE_THRESHOLD picked at 0.05 (5% of the 0..1 scale) — below that, the +// difference is below the noise floor of the LASK5 measurement itself. +const RESIDUAL_CLOSE_THRESHOLD = 0.05; + +function renderResiduals(laskDev, inf) { + const deltaRows = document.querySelectorAll('#comparator-deltas .delta-row'); + if (!deltaRows.length) return; + const gt = laskDev && Array.isArray(laskDev.values) ? laskDev.values : null; + const pred = inf && Array.isArray(inf.piston_values) ? inf.piston_values : null; + + deltaRows.forEach((row, i) => { + const valEl = row.querySelector('.delta-val'); + row.classList.remove('over', 'under', 'close'); + if (!gt || !pred || i >= gt.length || i >= pred.length) { + if (valEl) valEl.textContent = '--'; + return; + } + const g = pistonFraction(gt[i]); + const p = pistonFraction(pred[i]); + const d = p - g; + valEl.textContent = (d >= 0 ? '+' : '') + d.toFixed(2); + if (Math.abs(d) < RESIDUAL_CLOSE_THRESHOLD) row.classList.add('close'); + else if (d > 0) row.classList.add('over'); + else row.classList.add('under'); + }); +} + +// ---------- Studio shell: top-bar pipeline status strip ---------- + +// Set a pipe-pill's status + value text. State controls colour: +// 'live' -- blue accent (data flowing) +// 'ok' -- green (idle but healthy) +// 'warn' -- orange +// 'bad' -- red +// '' -- neutral grey +function setPipePill(id, state, valText) { + const el = document.getElementById(id); + if (!el) return; + el.classList.remove('ok', 'warn', 'bad', 'live'); + if (state) el.classList.add(state); + const valEl = el.querySelector('.pipe-val'); + if (valEl) valEl.textContent = valText; +} + +function renderPipelinePills(msg, laskDev) { + // SENSOR pill = the active flexgrid (the one driving the heatmap) + const dev = selectedDevice(); + if (dev && dev.device_type === 'flexgrid') { + const stale = dev.last_seen_age > 2.0; + setPipePill('pipe-sensor', stale ? 'warn' : 'live', `${dev.hz.toFixed(0)}Hz`); + } else { + setPipePill('pipe-sensor', '', '--'); + } + + // LABEL pill = LASK5 stream + if (laskDev) { + const stale = laskDev.last_seen_age > 2.0; + setPipePill('pipe-label', stale ? 'warn' : 'live', `${laskDev.hz.toFixed(0)}Hz`); + } else { + setPipePill('pipe-label', '', '--'); + } + + // CAPTURE pill + if (recordingState) { + const matchRate = recordingState.match_rate ?? 0; + const cls = matchRate < 0.5 ? 'bad' : (matchRate < 0.9 ? 'warn' : 'live'); + setPipePill('pipe-capture', cls, `REC ${recordingState.rows ?? 0}r`); + } else if (activeSession) { + setPipePill('pipe-capture', 'ok', `session: ${activeSession.name || activeSession.id}`); + } else { + setPipePill('pipe-capture', '', 'idle'); + } + + // MODEL pill + const inf = msg.inference; + if (inf && inf.model && inf.enabled) setPipePill('pipe-model', 'live', inf.model); + else if (inf && inf.model && !inf.enabled) setPipePill('pipe-model', 'ok', inf.model + ' (paused)'); + else setPipePill('pipe-model', '', 'none'); + + // HAND pill = UDP forwarding target + if (inf && inf.hand_target) setPipePill('pipe-hand', 'live', inf.hand_target); + else setPipePill('pipe-hand', '', 'off'); +} + +// ---------- Studio shell: diagnostics drawer ---------- + +const diagToggle = document.getElementById('diag-toggle'); +const diagBody = document.getElementById('diag-body'); +if (diagToggle && diagBody) { + diagToggle.onclick = () => { + const isHidden = diagBody.classList.toggle('hidden'); + diagToggle.setAttribute('aria-expanded', isHidden ? 'false' : 'true'); + diagToggle.textContent = (isHidden ? '▸' : '▾') + ' Diagnostics & logs'; + // Logs poll runs unconditionally; we just hide the DOM. Cheap. + }; +} + // ---------- utils ---------- function escapeHtml(s) { diff --git a/pc/src/openmuscle/web/static/index.html b/pc/src/openmuscle/web/static/index.html index b795c34..dc44e29 100644 --- a/pc/src/openmuscle/web/static/index.html +++ b/pc/src/openmuscle/web/static/index.html @@ -3,179 +3,292 @@ - OpenMuscle Live + OpenMuscle Studio -
    -

    OpenMuscle Live

    -
    +
    +
    +

    OpenMuscle Studio

    +
    + + + + +
    disconnected
    -
    -
    -

    Devices

    -
      -
    • Waiting for a device to send a packet…
      • -
    -
    +
    -
    -
    -

    Heatmap

    - -
    - -
    + +
    +
    +

    1 Live

    + what the band feels · what the model thinks +
    -
    -
    -

    LASK5 — Ground Truth

    - no device -
    -
    -
    P1
    --
    -
    P2
    --
    -
    P3
    --
    -
    P4
    --
    -
    -
    - joystick - - --, -- -
    -
    +
    + + -
    -
    -

    LASK Inference — Predicted

    - no model loaded -
    -
    -
    P1̂
    --
    -
    P2̂
    --
    -
    P3̂
    --
    -
    P4̂
    --
    -
    -
    - - - - -
    -
    + +
    +
    + +
    + +
    -
    -

    Record

    -
    - - - - + +
    +
    +

    Ground truth vs Predicted

    +
    + GT: no device + MODEL: no model loaded +
    +
    + +
    + +
    +
    P1
    --
    +
    P2
    --
    +
    P3
    --
    +
    P4
    --
    +
    + + +
    +
    Δ--
    +
    Δ--
    +
    Δ--
    +
    Δ--
    +
    + + +
    +
    P1̂
    --
    +
    P2̂
    --
    +
    P3̂
    --
    +
    P4̂
    --
    +
    +
    + + +
    + + --, -- +
    + + +
    + + predictions to a robot hand: see Stage 4 +
    +
    -
    -
    -
    -

    Sessions

    -
    - + +
    +
    +

    2 Capture

    + session + record · paired sensor/label rows go to disk +
    + +
    + +
    +
    + Session + +
    +
    +
    No active session — recordings won't be grouped. Click "New session" to start one.
    +
    +
    + + +
    +
    + Record +
    +
    + + + + +
    +
    -
    -
    No active session — recordings won't be grouped. Click "New session" to start one.
    -
    + +
    -
    -
    -

    Captures

    -
    - 0 selected - + +
    +
    +

    3 Data & Models

    + captures become models · audit honestly · iterate +
    + +
    + +
    +
    +

    Captures

    +
    + 0 selected + +
    +
    + + + + + + + + + + + + + + +
    NameMetaSizeModified
    No captures saved yet.
    +
    + + +
    + +
    +
    + + +
    +
    +

    Models

    + +
    + + + + + + + + + + + + + + +
    NameCreatedR²MSEFeatures × Labels
    No models trained yet.
    - - - - - - - - - - - - - - -
    NameMetaSizeModified
    No captures saved yet.
    -
    -
    -
    -

    Models

    - + +
    +
    +

    4 Output

    + where the predictions are sent +
    +
    + + + no hand target
    - - - - - - - - - - - - - - -
    NameCreatedR²MSEFeatures × Labels
    No models trained yet.
    -
    -
    -

    Logs

    -
    - - - -
    + +
    + +
    - Open-Muscle · FlexGrid · github + Open-Muscle · FlexGrid Studio · github
    - + - + + + +
    + +
    +
    +
    + ● real + ● predicted + drag to rotate +
    +
    + diff --git a/pc/src/openmuscle/web/static/styles.css b/pc/src/openmuscle/web/static/styles.css index 10d938e..d833aeb 100644 --- a/pc/src/openmuscle/web/static/styles.css +++ b/pc/src/openmuscle/web/static/styles.css @@ -1269,3 +1269,37 @@ footer { font-size: 12px; text-align: center; } + +/* ---- quest_hand 3D viewer (Studio Live stage) ---- + Shown in place of the LASK5 piston comparator when a quest_hand label + source is streaming. Toggled by app.js adding .hand-mode to .comparator; + the viewer's own visibility is also set imperatively by OMHandViewer + (inline display), which defers to these rules when shown. */ +.hand-viewer { + display: none; + flex-direction: column; + gap: 6px; +} +.comparator.hand-mode .comparator-row { display: none; } +.comparator.hand-mode .hand-viewer { display: flex; } + +.hand-viewer-canvas { + width: 100%; + height: 240px; + border-radius: 8px; + background: radial-gradient(ellipse at center, #11161f 0%, #0b0e14 100%); + border: 1px solid var(--border, #23262d); + touch-action: none; /* let pointer-drag rotate instead of scroll */ +} +.hand-viewer-canvas canvas { display: block; border-radius: 8px; } + +.hand-viewer-legend { + display: flex; + gap: 16px; + font-size: 12px; + color: var(--fg-faint, #8b96a8); + align-items: center; +} +.hand-viewer-legend .hv-real { color: #34d399; } +.hand-viewer-legend .hv-pred { color: #fbbf24; } +.hand-viewer-legend .hv-hint { margin-left: auto; opacity: 0.7; font-style: italic; } From 5c3a4959e2f12658bbccca6083103e9176590aea Mon Sep 17 00:00:00 2001 From: TURFPTAx Date: Tue, 9 Jun 2026 07:18:36 -0500 Subject: [PATCH 47/47] tests: lock the snapshot contract the desktop hand viewer depends on The desktop Studio 3D hand viewer reads each device's flat `values` from the /ws/live snapshot. Add a test asserting a quest_hand device surfaces in _snapshot() as device_type 'quest_hand' with its full 25*7 flat joint vector, so a future snapshot refactor can't silently break the viewer's only data source. Suite now 32 passing. Co-Authored-By: turfptax-claude O4.8 --- pc/tests/test_quest_ingest.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/pc/tests/test_quest_ingest.py b/pc/tests/test_quest_ingest.py index 059aa9e..86dcaab 100644 --- a/pc/tests/test_quest_ingest.py +++ b/pc/tests/test_quest_ingest.py @@ -100,3 +100,29 @@ def test_repeated_packets_increment_packet_count(self): s.ingest_quest_packet({"joints": [_full_joint("wrist", offset=i * 0.01)]}) d = next(iter(s.devices.values())) assert d.packets_total == 5 + + +class TestSnapshotExposesQuestHand: + """The desktop Studio 3D hand viewer reads each device's flat `values` + from the /ws/live snapshot. Lock that contract: a quest_hand device must + surface in _snapshot() as device_type 'quest_hand' with its full flat + joint vector, so a future snapshot refactor can't silently break the + viewer's only data source. + """ + + def test_snapshot_has_quest_device_with_flat_values(self): + # _snapshot() touches the full inference machinery, so use a real + # AppState (its __init__ sets engine_status etc.) rather than the bare + # __new__ helper. The UDP listener is never started, so no socket binds. + import tempfile + with tempfile.TemporaryDirectory() as d: + s = AppState(udp_port=53997, captures_dir=d) + joints = [_full_joint(f"j{i}", i * 0.01) for i in range(25)] + s.ingest_quest_packet({"device_id": "quest-right", "handedness": "right", + "joints": joints}) + snap = s._snapshot() + quest = [dev for dev in snap["devices"] if dev["device_type"] == "quest_hand"] + assert len(quest) == 1 + # 25 joints * 7 floats -> the viewer slices [i*7 .. i*7+6] per joint. + assert len(quest[0]["values"]) == 25 * 7 + assert quest[0]["device_id"] == "quest-right"