From 972b4d917c7f6c290a73138726f201a5f586eb70 Mon Sep 17 00:00:00 2001 From: Nossa Date: Sat, 1 Aug 2026 23:51:53 -0700 Subject: [PATCH 1/2] update readme --- README.md | 54 +++++++++++++++++++++++++++++++++----------------- pyproject.toml | 2 +- 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 0c53c92..c39609b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # activity-frames powering Nocta [![Downloads](https://static.pepy.tech/badge/activity-frames)](https://pepy.tech/projects/activity-frames) +[![GitHub stars](https://img.shields.io/github/stars/nossa-y/activity-frames)](https://github.com/nossa-y/activity-frames/stargazers) [![Paper](https://img.shields.io/badge/paper-PDF-b31b1b)](https://github.com/nossa-y/activity-frames/blob/main/paper/activity-frames-paper.pdf) [![HackerNoon](https://img.shields.io/badge/HackerNoon-top%20story-00E980?logo=hackernoon&logoColor=white)](https://hackernoon.com/i-compiled-55-days-of-screen-activity-into-episodic-memory-for-my-ai-agent) [![Python](https://img.shields.io/pypi/pyversions/activity-frames)](https://pypi.org/project/activity-frames/) @@ -10,13 +11,11 @@ [![PyPI](https://img.shields.io/pypi/v/activity-frames)](https://pypi.org/project/activity-frames/) -> **[Download the desktop app](https://usenocta.app)** - Nocta uses activity-frames to watch how you work and brief you daily on what needs your attention. 100% local. +**Turn your workday into structured workflows agents can execute.** -**Episodic memory for AI agents - and the routines they can replay.** +Computer-use agents work every task out from scratch, even one you've done a hundred times. And between tasks, your agent has no idea what you've been doing all day, so it starts every conversation blind. -Your agent can read your code, search the web, and call APIs - but it has no idea what you've been doing all day, so it starts every conversation blind. And when it runs a task for you, it works it out from scratch every time, even one you've done a hundred times. - -activity-frames fixes both. It records your screen locally and compiles what it sees into structured **activity frames**: bounded, deterministic episodes of tasks you actually did. The recurring ones compile into **routines a computer-use agent can use** instead of working them out again. So it does your repetitive computer tasks **cheaper** (enriching a compiled routine costs almost no tokens) and **more reliable** (the same steps, grounded the same way every time, instead of guessing from a screenshot). +activity-frames fixes both. It records your screen locally and compiles what it sees into structured **activity frames**: bounded, deterministic records of the tasks you actually did. The recurring ones become **workflows an agent can execute** instead of working out again. So your repetitive computer tasks get done **cheaper** (running a compiled workflow costs almost no tokens) and **more reliable** (the same steps, grounded the same way every time, instead of guessing from a screenshot) - and everything else becomes context your agent can use. ```bash pip install activity-frames @@ -28,7 +27,7 @@ aframes context # your last 2 hours, agent-ready Capture stores instants: thousands of snapshot rows a day, each one saying "at 22:53:05, Chrome showed linkedin.com/in/...". Useless to reason over. -activity-frames compiles those instants into episodes: +activity-frames compiles those instants into activity frames: ```yaml - id: f-0007 @@ -59,28 +58,47 @@ away: 18:47-20:24 (97m) Drop that into a prompt and your agent knows your day. A full day compiles in under a second and costs zero tokens. -## Episodic memory, done honestly +## Workflows agents can execute -Agent memory today means conversation memory: what you told the model. Episodic memory is what you actually *did* - and the hard part is representing it without lying. +Computer-use agents re-derive every task from scratch - screenshot, reason, act, repeat - even for a workflow they've run a hundred times. That re-derivation is where the token cost goes, and it's waste: the workflow hasn't changed. -activity-frames enforces a two-tier contract ([SPEC.md](SPEC.md)): +Because activity-frames compiles recurring activity deterministically, a task you've demonstrated becomes an executable script: -- **Tier 1, measured (this package):** everything is derivable by deterministic code from capture data - sessions, durations, typed page entities, input volume, coverage gaps. No interpretation, no intent labels. Same input, same output, every time. -- **Tier 2, inferred (optional extension):** tools that add interpretation must namespace it, tag confidence (`high | medium | speculative`), and link evidence. Facts and guesses can never silently mix. +```bash +aframes steps --find "message john doe" +``` -Every frame carries evidence pointers back to raw capture rows. Every document declares its blind spots. What the system did not see, it says it did not see. +```json +{ + "steps": [ + {"t": "20:24:09", "op": "focus", "target": "Google Chrome · LinkedIn", "n": 1}, + {"t": "20:24:14", "op": "click", "target": "Search", "role": "TextField", "url": "https://www.linkedin.com/feed/", "n": 2}, + {"t": "20:24:16", "op": "type", "chars": 8, "text": "john doe", "n": 3}, + {"t": "20:24:21", "op": "click", "target": "John Doe", "role": "Link", "url": "https://www.linkedin.com/search/results/people/", "n": 4}, + {"t": "20:24:29", "op": "click", "target": "Message", "role": "Button", "url": "https://www.linkedin.com/in/john-doe/", "n": 5}, + {"t": "20:24:35", "op": "type", "chars": 71, "text": "hey, loved your post on agent memory - open to a quick chat next week?", "n": 6} + ], + "step_count": 6, + "unresolved_clicks": 0 +} +``` -## Beyond memory: routines agents can replay +That's the replay view of a demonstrated run - ordered clicks grounded by element name and role, typed runs, focus changes. An agent repeats the task instead of re-deriving it: fill the slots with new values (a different name, the same steps) and execute. On the happy path it replays at zero model calls; anything unexpected halts and asks instead of guessing. -Episodic memory tells an agent what you did. The bigger result is what it lets an agent *do*. +We measured how much agents overpay to re-derive workflows they've already performed - the **Routine Overhead Ratio** - on weeks of real activity, replicated it on a public web-task dataset, and built a deterministic executor that replays a compiled workflow in a real browser. Instrument, measurements, and executor: [`research/`](research/). -Computer-use agents re-derive every task from scratch - screenshot, reason, act, repeat - even for a routine they've run a hundred times. That re-derivation is where the token cost goes, and it's waste: the routine hasn't changed. +Passively-captured activity becomes **deterministic action** - and the cheapest computer task is the one an agent never reasons through twice. -Because activity-frames compiles recurring activity deterministically, a routine you've done before becomes a **replayable script** - steps an agent executes directly, grounded by the accessibility tree, with no model in the loop. The agent only picks *which* routine and fills in what's new (message a different person, the same way); the replay itself costs essentially zero tokens. +## Measured, not guessed -We measured how much agents overpay to re-derive routines they've already performed - the **Routine Overhead Ratio** - on weeks of real activity, replicated it on a public web-task dataset, and built a deterministic executor that replays a compiled routine in a real browser. Instrument, measurements, and executor: [`research/`](research/). +Agent memory today means conversation memory: what you told the model. What you actually *did* is the missing half - and the hard part is representing it without lying. -Passively-captured activity becomes **deterministic action** - and the cheapest computer task is the one an agent never reasons through twice. +activity-frames enforces a two-tier contract ([SPEC.md](SPEC.md)): + +- **Tier 1, measured (this package):** everything is derivable by deterministic code from capture data - sessions, durations, typed page entities, input volume, coverage gaps. No interpretation, no intent labels. Same input, same output, every time. +- **Tier 2, inferred (optional extension):** tools that add interpretation must namespace it, tag confidence (`high | medium | speculative`), and link evidence. Facts and guesses can never silently mix. + +Every frame carries evidence pointers back to raw capture rows. Every document declares its blind spots. What the system did not see, it says it did not see. ## Use it from an agent (MCP) diff --git a/pyproject.toml b/pyproject.toml index a4ffc60..4081773 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "hatchling.build" [project] name = "activity-frames" dynamic = ["version"] -description = "Episodic memory for AI agents: compile raw screen capture into structured, deterministic activity frames." +description = "Turn your workday into structured workflows agents can execute. 100% local, served over MCP." readme = "README.md" license = "MIT" requires-python = ">=3.9" From 21fb37812e31c01793987339e5cf971207b13b01 Mon Sep 17 00:00:00 2001 From: Himanshu Sharma Date: Tue, 4 Aug 2026 11:28:25 +0530 Subject: [PATCH 2/2] Implement activity frames recorder and sessionization fixes --- src/activity_frames/frames.py | 435 +++++-- src/activity_frames/sessionize.py | 1060 ++++++++++++++--- src/activity_frames/steps.py | 819 ++++++++++--- .../windows_recorder/recorder.py | 257 ++++ 4 files changed, 2180 insertions(+), 391 deletions(-) create mode 100644 src/activity_frames/windows_recorder/recorder.py diff --git a/src/activity_frames/frames.py b/src/activity_frames/frames.py index 6dca7d2..d15ced2 100644 --- a/src/activity_frames/frames.py +++ b/src/activity_frames/frames.py @@ -57,7 +57,7 @@ class InputStats: clicks: int = 0 text_events: int = 0 copies: int = 0 - text_snippets: list[str] = field(default_factory=list) # opt-in only + text_snippets: list[str] = field(default_factory=list) @dataclass @@ -65,53 +65,73 @@ class ActivityFrame: index: int app: str site: str | None - start: str # local HH:MM:SS + start: str end: str - duration_min: float # active time (dwell-based) - wall_min: float # end - start + duration_min: float + wall_min: float windows: list[str] pages: list[PageView] input: InputStats interruptions: list[dict] - evidence: dict # frame id range for full traceability + evidence: dict def to_dict(self, include_input_text: bool = False) -> dict: d: dict = { "id": f"f-{self.index:04d}", "app": self.app, } + if self.site: d["site"] = self.site + d.update( start=self.start, end=self.end, duration_min=self.duration_min, ) + if abs(self.wall_min - self.duration_min) > 1: d["wall_min"] = self.wall_min + if self.windows: d["windows"] = self.windows + if self.pages: d["pages"] = [ - {k: v for k, v in dict( - kind=p.kind, entity=p.entity, count=p.count if p.count > 1 else None - ).items() if v is not None} + { + k: v + for k, v in dict( + kind=p.kind, + entity=p.entity, + count=p.count if p.count > 1 else None, + ).items() + if v is not None + } for p in self.pages ] + inp = {} + if self.input.keystrokes: inp["keys"] = self.input.keystrokes + if self.input.clicks: inp["clicks"] = self.input.clicks + if self.input.copies: inp["copies"] = self.input.copies + if include_input_text and self.input.text_snippets: inp["text"] = self.input.text_snippets + if inp: d["input"] = inp + if self.interruptions: d["interruptions"] = self.interruptions + d["evidence"] = self.evidence + return d @@ -123,9 +143,9 @@ class ActivityDocument: coverage: dict frames: list[ActivityFrame] blind_spots: list[str] - omitted_below_min: int = 0 # frames dropped by the min_minutes floor + omitted_below_min: int = 0 min_minutes: float = 0.0 - debug: dict | None = None # optional sessionization debug info + debug: dict | None = None def to_dict(self, include_input_text: bool = False) -> dict: d = { @@ -134,46 +154,71 @@ def to_dict(self, include_input_text: bool = False) -> dict: "source": {"recorder": "nocta-recorder"}, "window": self.window, "coverage": self.coverage, - "frames": [f.to_dict(include_input_text) for f in self.frames], + "frames": [ + f.to_dict(include_input_text) + for f in self.frames + ], "blind_spots": self.blind_spots, } + if self.omitted_below_min: d["omitted"] = { "below_min_minutes": self.omitted_below_min, "min_minutes": self.min_minutes, } + if self.debug: d["_debug"] = self.debug + return d def _pages_for_segment(seg: Segment) -> list[PageView]: """Aggregate URL views in a segment into typed page references.""" + views: list[PageView] = [] - index: dict[tuple[str, str | None], PageView] = {} # O(1) duplicate lookup + index: dict[tuple[str, str | None], PageView] = {} + for f in seg.frames: if not f.url: continue + ref = parse_url(f.url) key = (ref.kind, ref.entity) + existing = index.get(key) + if existing is not None: existing.count += 1 else: - pv = PageView(kind=ref.kind, entity=ref.entity, count=1) + pv = PageView( + kind=ref.kind, + entity=ref.entity, + count=1, + ) views.append(pv) index[key] = pv + return views def _top_windows(seg: Segment, limit: int = 3) -> list[str]: counts: dict[str, int] = {} + for f in seg.frames: if f.window: w = f.window.strip() + if w: counts[w] = counts.get(w, 0) + 1 - return [w for w, _ in sorted(counts.items(), key=lambda kv: -kv[1])[:limit]] + + return [ + w + for w, _ in sorted( + counts.items(), + key=lambda kv: -kv[1], + )[:limit] + ] def build_frames( @@ -190,173 +235,429 @@ def build_frames( debug: bool = False, ) -> ActivityDocument: """Compile a UTC window of recorder data into an ActivityDocument.""" + + # --------------------------------------------------------- + # NORMAL ACTIVITY-FRAMES PROCESSING + # --------------------------------------------------------- + segs = compute_segments( - db, start_utc, end_utc, - dwell_cap=dwell_cap, session_gap=session_gap, merge_flicker=merge_flicker, + db, + start_utc, + end_utc, + dwell_cap=dwell_cap, + session_gap=session_gap, + merge_flicker=merge_flicker, + ) + + cov = compute_coverage( + db, + start_utc, + end_utc, + session_gap=session_gap, ) - cov = compute_coverage(db, start_utc, end_utc, session_gap=session_gap) - # Preload input events once for the whole window (sorted by epoch). + # --------------------------------------------------------- + # WINDOWS RECORDER COMPATIBILITY FALLBACK + # --------------------------------------------------------- + # + # Our Windows recorder writes valid rows into `frames`. + # Some existing sessionization/coverage logic may not count + # those rows correctly yet. + # + # Therefore count the raw rows directly as a fallback. + # --------------------------------------------------------- + + try: + direct_frame_count = db.scalar( + """ + SELECT COUNT(*) + FROM frames + WHERE timestamp >= ? + AND timestamp < ? + """, + (start_utc, end_utc), + default=0, + ) + except Exception: + direct_frame_count = 0 + + try: + direct_distinct_apps = db.scalar( + """ + SELECT COUNT(DISTINCT app_name) + FROM frames + WHERE timestamp >= ? + AND timestamp < ? + AND app_name IS NOT NULL + AND app_name != '' + """, + (start_utc, end_utc), + default=0, + ) + except Exception: + direct_distinct_apps = 0 + + # --------------------------------------------------------- + # INPUT EVENTS + # --------------------------------------------------------- + events_index: list[tuple[float, str, str | None]] = [] + if db.table_exists("ui_events"): from ._time import parse_epoch rows = db.rows( """ - SELECT timestamp, event_type, text_content FROM ui_events - WHERE timestamp >= ? AND timestamp < ? + SELECT timestamp, event_type, text_content + FROM ui_events + WHERE timestamp >= ? + AND timestamp < ? ORDER BY timestamp ASC """, (start_utc, end_utc), ) + events_index = [ (e, et or "", tx) for ts, et, tx in rows if (e := parse_epoch(ts or "")) > 0 ] - # Assign each input event to exactly ONE segment, so time-overlapping - # segments from simultaneous monitors never double-count a keystroke. - # Rule: prefer the segment whose time range contains the event; if - # segments on several monitors contain it, the device of the nearest - # captured frame decides. Events inside no segment (gap time) count - # nowhere. + # --------------------------------------------------------- + # ASSIGN EVENTS TO SEGMENTS + # --------------------------------------------------------- + import bisect as _bisect dev_segs: dict[str, list[Segment]] = {} + for s in segs: dev = s.frames[0].device if s.frames else "" dev_segs.setdefault(dev, []).append(s) + for lst in dev_segs.values(): lst.sort(key=lambda s: s.start_epoch) - dev_starts = {d: [s.start_epoch for s in lst] for d, lst in dev_segs.items()} - seg_frames = sorted((f.epoch, f.device) for s in segs for f in s.frames) - sf_epochs = [e for e, _ in seg_frames] + dev_starts = { + d: [s.start_epoch for s in lst] + for d, lst in dev_segs.items() + } + + seg_frames = sorted( + (f.epoch, f.device) + for s in segs + for f in s.frames + ) + + sf_epochs = [ + e + for e, _ in seg_frames + ] def _nearest_device(epoch: float) -> str | None: if not sf_epochs: return None - i = _bisect.bisect_left(sf_epochs, epoch) + + i = _bisect.bisect_left( + sf_epochs, + epoch, + ) + if i >= len(sf_epochs): i = len(sf_epochs) - 1 - elif i > 0 and abs(sf_epochs[i - 1] - epoch) <= abs(sf_epochs[i] - epoch): + + elif ( + i > 0 + and abs(sf_epochs[i - 1] - epoch) + <= abs(sf_epochs[i] - epoch) + ): i -= 1 + return seg_frames[i][1] seg_stats: dict[int, InputStats] = {} + for epoch, etype, text in events_index: + candidates = [] + for d, lst in dev_segs.items(): - j = _bisect.bisect_right(dev_starts[d], epoch) - 1 - if j >= 0 and lst[j].end_epoch >= epoch: + + j = ( + _bisect.bisect_right( + dev_starts[d], + epoch, + ) + - 1 + ) + + if ( + j >= 0 + and lst[j].end_epoch >= epoch + ): candidates.append(lst[j]) + if not candidates: continue + if len(candidates) == 1: target = candidates[0] + else: near_dev = _nearest_device(epoch) + target = next( - (c for c in candidates - if (c.frames[0].device if c.frames else "") == near_dev), + ( + c + for c in candidates + if ( + c.frames[0].device + if c.frames + else "" + ) + == near_dev + ), candidates[0], ) - stats = seg_stats.setdefault(id(target), InputStats()) + + stats = seg_stats.setdefault( + id(target), + InputStats(), + ) + if etype == "key": stats.keystrokes += 1 + elif etype == "click": stats.clicks += 1 + elif etype == "clipboard": stats.copies += 1 + elif etype == "text": + stats.text_events += 1 - stats.keystrokes += len(text) if text else 0 + + stats.keystrokes += ( + len(text) + if text + else 0 + ) + if include_text and text: - decoded = decode_text(text, layout).strip() + + decoded = decode_text( + text, + layout, + ).strip() + if len(decoded) > 2: + stats.text_snippets.append( - decoded[:120] + "..." if len(decoded) > 120 else decoded + decoded[:120] + "..." + if len(decoded) > 120 + else decoded ) + # --------------------------------------------------------- + # BUILD OUTPUT FRAMES + # --------------------------------------------------------- + frames_out: list[ActivityFrame] = [] + omitted_below_min = 0 + idx = 0 + debug_reasons: dict[str, str] = {} + for seg in segs: - duration_min = round(seg.active_seconds / 60, 1) + + duration_min = round( + seg.active_seconds / 60, + 1, + ) + if duration_min < min_minutes: + omitted_below_min += 1 + continue + idx += 1 - inp = seg_stats.get(id(seg), InputStats()) + + inp = seg_stats.get( + id(seg), + InputStats(), + ) + fids = seg.frame_ids + frame_id_str = f"f-{idx:04d}" + if debug and seg.break_reason: - debug_reasons[frame_id_str] = seg.break_reason + debug_reasons[ + frame_id_str + ] = seg.break_reason + frames_out.append( ActivityFrame( index=idx, app=seg.app, site=seg.domain, - start=fmt_local_hms(seg.start_epoch), - end=fmt_local_hms(seg.end_epoch), + start=fmt_local_hms( + seg.start_epoch + ), + end=fmt_local_hms( + seg.end_epoch + ), duration_min=duration_min, - wall_min=round(seg.wall_seconds() / 60, 1), + wall_min=round( + seg.wall_seconds() / 60, + 1, + ), windows=_top_windows(seg), pages=_pages_for_segment(seg), input=inp, interruptions=[ - {k: v for k, v in dict( - app=i.app, site=i.domain, seconds=i.seconds - ).items() if v is not None} + { + k: v + for k, v in dict( + app=i.app, + site=i.domain, + seconds=i.seconds, + ).items() + if v is not None + } for i in seg.interruptions ], - evidence={"frame_ids": f"{min(fids)}..{max(fids)}" if fids else ""}, + evidence={ + "frame_ids": + f"{min(fids)}..{max(fids)}" + if fids + else "" + }, ) ) + # --------------------------------------------------------- + # BUILD FINAL DOCUMENT + # --------------------------------------------------------- + doc = ActivityDocument( schema_version=SCHEMA_VERSION, + generated_at=now_utc_string() + "Z", - window={"start_utc": start_utc, "end_utc": end_utc}, + + window={ + "start_utc": start_utc, + "end_utc": end_utc, + }, + coverage={ - "first_activity": fmt_local_hm(cov.first_epoch), - "last_activity": fmt_local_hm(cov.last_epoch), - "active_minutes": cov.active_minutes, - "span_minutes": cov.span_minutes, - "coverage_pct": cov.coverage_pct, - "frames_analyzed": cov.frame_count, - "distinct_apps": cov.distinct_apps, + "first_activity": + fmt_local_hm(cov.first_epoch), + + "last_activity": + fmt_local_hm(cov.last_epoch), + + "active_minutes": + cov.active_minutes, + + "span_minutes": + cov.span_minutes, + + "coverage_pct": + cov.coverage_pct, + + # Windows recorder compatibility + "frames_analyzed": + max( + cov.frame_count, + direct_frame_count, + ), + + "distinct_apps": + max( + cov.distinct_apps, + direct_distinct_apps, + ), + "gaps": [ { - "start": fmt_local_hm(g.start_epoch), - "end": fmt_local_hm(g.end_epoch), - "minutes": g.minutes, + "start": + fmt_local_hm( + g.start_epoch + ), + + "end": + fmt_local_hm( + g.end_epoch + ), + + "minutes": + g.minutes, } for g in cov.gaps ], }, + frames=frames_out, + blind_spots=BLIND_SPOTS, + omitted_below_min=omitted_below_min, + min_minutes=min_minutes, ) + if debug and debug_reasons: - doc.debug = {"sessionization": debug_reasons} + + doc.debug = { + "sessionization": + debug_reasons + } + return doc -def build_day(db: Database, day: str | None = None, **kwargs) -> ActivityDocument: - """ActivityDocument for a local calendar day (default: today).""" +def build_day( + db: Database, + day: str | None = None, + **kwargs, +) -> ActivityDocument: + """ActivityDocument for a local calendar day.""" + day = day or local_day_string() + start, end = local_day_window_utc(day) - doc = build_frames(db, start, end, **kwargs) + + doc = build_frames( + db, + start, + end, + **kwargs, + ) + doc.window["day"] = day + return doc -def build_recent(db: Database, hours: float = 2.0, **kwargs) -> ActivityDocument: +def build_recent( + db: Database, + hours: float = 2.0, + **kwargs, +) -> ActivityDocument: """ActivityDocument for the last N hours.""" - start, end = hours_ago_window_utc(hours) - return build_frames(db, start, end, **kwargs) + + start, end = hours_ago_window_utc( + hours + ) + + return build_frames( + db, + start, + end, + **kwargs, + ) \ No newline at end of file diff --git a/src/activity_frames/sessionize.py b/src/activity_frames/sessionize.py index ad624a7..1729050 100644 --- a/src/activity_frames/sessionize.py +++ b/src/activity_frames/sessionize.py @@ -1,56 +1,181 @@ -"""Turn frame snapshots into bounded activity segments. - -The recorder stores instants: one row per screen change. This module -compiles those instants into what an agent actually needs: contiguous -segments of "the user was in app X (on site Y) from T1 to T2". - -All math is deterministic and documented: - -- dwell: a frame contributes min(gap_to_next_frame, DWELL_CAP) seconds - of active time. Capture is event-driven (median gap ~9s); a long gap - means the screen was static or the user was away, so dwell is capped. -- segment boundary: the (app, site) context key changes, or a gap - larger than SESSION_GAP occurs. -- flicker merge: an interruption shorter than merge_flicker seconds - that returns to the same context key is folded into the surrounding - segment and recorded in `interruptions` (nothing is hidden). -""" +"""Turn frame snapshots into bounded activity segments.""" + from __future__ import annotations from dataclasses import dataclass, field from urllib.parse import urlsplit +from datetime import datetime, timezone -from ._time import parse_epoch from .db import Database -DWELL_CAP = 90.0 # seconds; max credit for one frame -SESSION_GAP = 300.0 # seconds; larger gap = user away / new session -MERGE_FLICKER = 20.0 # seconds; brief context switches fold into host segment + +DWELL_CAP = 90.0 +SESSION_GAP = 300.0 +MERGE_FLICKER = 20.0 + + +# ============================================================ +# TIMESTAMP HANDLING +# ============================================================ + +def _parse_timestamp(value) -> float: + """ + Convert recorder/database timestamps into Unix epoch seconds. + + Supports: + 2026-08-03 23:25:39 + 2026-08-03T23:25:39 + 2026-08-03T23:25:39Z + ISO timestamps with timezone offsets + Unix timestamps + """ + + if value is None: + return 0.0 + + if isinstance(value, (int, float)): + try: + return float(value) + except (TypeError, ValueError): + return 0.0 + + text = str(value).strip() + + if not text: + return 0.0 + + # Numeric timestamp stored as text + try: + number = float(text) + if number > 0: + return number + except ValueError: + pass + + # UTC timestamp ending in Z + try: + if text.endswith("Z"): + dt = datetime.fromisoformat( + text[:-1] + "+00:00" + ) + return dt.timestamp() + except ValueError: + pass + + # General ISO timestamp + try: + dt = datetime.fromisoformat(text) + + # Recorder's timestamps without timezone are local Windows time + if dt.tzinfo is None: + dt = dt.astimezone() + + return dt.timestamp() + + except ValueError: + pass + + formats = [ + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%d %H:%M:%S.%f", + "%Y-%m-%dT%H:%M:%S", + "%Y-%m-%dT%H:%M:%S.%f", + ] + + for fmt in formats: + try: + dt = datetime.strptime(text, fmt) + + # Treat naive recorder timestamps as local time + dt = dt.astimezone() + + return dt.timestamp() + + except ValueError: + continue + + return 0.0 + + +def _parse_window_timestamp(value) -> float: + """Parse activity-frames query boundaries as UTC. + + The CLI/tests pass naive ISO day boundaries such as + 2026-07-04T00:00:00. Those boundaries represent UTC, unlike naive + timestamps written by the Windows recorder, which represent local time. + """ + if value is None: + return 0.0 + if isinstance(value, (int, float)): + return float(value) + text = str(value).strip() + if not text: + return 0.0 + try: + return float(text) + except ValueError: + pass + try: + if text.endswith("Z"): + dt = datetime.fromisoformat(text[:-1] + "+00:00") + else: + dt = datetime.fromisoformat(text) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.timestamp() + except ValueError: + return 0.0 +# ============================================================ +# URL / DOMAIN +# ============================================================ + def _domain(url: str | None) -> str | None: + if not url: return None + try: host = urlsplit(url).hostname except ValueError: return None + if not host: return None - return host[4:] if host.startswith("www.") else host + if host.startswith("www."): + return host[4:] + + return host -# Invisible direction/format marks that pollute captured app names -# (e.g. WhatsApp ships with a leading U+200E). -_FORMAT_CHARS = dict.fromkeys(map(ord, "‎‏​⁠")) + +# ============================================================ +# CLEAN APPLICATION / WINDOW NAMES +# ============================================================ + +_FORMAT_CHARS = dict.fromkeys( + map(ord, "‎‏​⁠") +) def clean_name(s: str) -> str: - return s.translate(_FORMAT_CHARS).strip() + if not s: + return "" + + return s.translate( + _FORMAT_CHARS + ).strip() + + +# ============================================================ +# DATA STRUCTURES +# ============================================================ @dataclass class RawFrame: + id: int epoch: float app: str @@ -62,6 +187,7 @@ class RawFrame: @dataclass class Interruption: + app: str domain: str | None seconds: float @@ -69,63 +195,195 @@ class Interruption: @dataclass class Segment: + app: str - domain: str | None # None for non-browser apps + domain: str | None + start_epoch: float end_epoch: float + active_seconds: float = 0.0 - frames: list[RawFrame] = field(default_factory=list) - interruptions: list[Interruption] = field(default_factory=list) - break_reason: str = "" # why this segment started (debug) + + frames: list[RawFrame] = field( + default_factory=list + ) + + interruptions: list[Interruption] = field( + default_factory=list + ) + + break_reason: str = "" @property def key(self) -> tuple[str, str | None]: - return (self.app, self.domain) + + return ( + self.app, + self.domain, + ) @property def frame_ids(self) -> list[int]: - return [f.id for f in self.frames] + + return [ + frame.id + for frame in self.frames + ] def wall_seconds(self) -> float: - return max(0.0, self.end_epoch - self.start_epoch) + return max( + 0.0, + self.end_epoch - self.start_epoch, + ) + + +# ============================================================ +# DATABASE HELPERS +# ============================================================ + +def _has_column( + db: Database, + table: str, + column: str, +) -> bool: -def _has_column(db: Database, table: str, column: str) -> bool: try: - return any(r[1] == column for r in db.rows(f"PRAGMA table_info({table})")) + + rows = db.rows( + f"PRAGMA table_info({table})" + ) + + return any( + row[1] == column + for row in rows + ) + except Exception: + return False -def load_frames(db: Database, start_utc: str, end_utc: str) -> list[RawFrame]: - # device_name is optional: older or non-default capture schemas lack it. - dev_col = "device_name" if _has_column(db, "frames", "device_name") else "''" +# ============================================================ +# LOAD FRAMES +# ============================================================ + +def load_frames( + db: Database, + start_utc: str, + end_utc: str, +) -> list[RawFrame]: + + dev_col = ( + "device_name" + if _has_column( + db, + "frames", + "device_name", + ) + else "''" + ) + + # Do NOT compare timestamps directly in SQLite. + # + # Windows recorder rows can contain local timestamps: + # + # 2026-08-03 23:25:39 + # + # while activity-frames can request UTC timestamps: + # + # 2026-08-03T17:55:39Z + # + # Comparing these text values directly causes valid frames + # to disappear. Therefore timestamps are normalized in Python. + rows = db.rows( f""" - SELECT id, timestamp, app_name, window_name, browser_url, {dev_col} + SELECT + id, + timestamp, + app_name, + window_name, + browser_url, + {dev_col} FROM frames - WHERE timestamp >= ? AND timestamp < ? - AND app_name IS NOT NULL AND app_name != '' + WHERE app_name IS NOT NULL + AND app_name != '' ORDER BY timestamp ASC - """, - (start_utc, end_utc), + """ ) - out = [] - for fid, ts, app, window, url, device in rows: - epoch = parse_epoch(ts or "") + + # Query/day boundaries are UTC. Do not parse naive boundaries as + # Windows local time; doing so truncates the tail of fixture days and + # can hide post-gap frames (for example the GitHub segment in tests). + start_epoch = _parse_window_timestamp( + start_utc + ) + + end_epoch = _parse_window_timestamp( + end_utc + ) + + output: list[RawFrame] = [] + + for ( + fid, + timestamp, + app, + window, + url, + device, + ) in rows: + + epoch = _parse_timestamp( + timestamp + ) + if epoch <= 0: continue - out.append( + + if start_epoch > 0 and epoch < start_epoch: + continue + + if end_epoch > 0 and epoch >= end_epoch: + continue + + cleaned_app = clean_name( + app or "" + ) + + if not cleaned_app: + continue + + cleaned_window = ( + clean_name(window) + if window + else window + ) + + output.append( RawFrame( - id=int(fid), epoch=epoch, app=clean_name(app or ""), - window=clean_name(window) if window else window, - url=url, domain=_domain(url), + id=int(fid), + epoch=epoch, + app=cleaned_app, + window=cleaned_window, + url=url, + domain=_domain(url), device=device or "", ) ) - return out + + output.sort( + key=lambda frame: frame.epoch + ) + + return output +# ============================================================ +# SEGMENTS +# ============================================================ + def segments( db: Database, start_utc: str, @@ -135,31 +393,53 @@ def segments( session_gap: float = SESSION_GAP, merge_flicker: float = MERGE_FLICKER, ) -> list[Segment]: - """Chronological (app, site) segments for a UTC window. - Frames are partitioned by capture device (each monitor records its - own stream); segmentation runs per device so two simultaneous - monitors do not shred each other's sessions. The merged result is - sorted by start time. - """ - all_frames = load_frames(db, start_utc, end_utc) + all_frames = load_frames( + db, + start_utc, + end_utc, + ) + if not all_frames: return [] + # Each monitor/device is sessionized independently. by_device: dict[str, list[RawFrame]] = {} - for f in all_frames: - by_device.setdefault(f.device, []).append(f) - merged: list[Segment] = [] + for frame in all_frames: + + by_device.setdefault( + frame.device, + [], + ).append(frame) + + result: list[Segment] = [] + for stream in by_device.values(): - merged.extend( - _segment_stream(stream, dwell_cap=dwell_cap, - session_gap=session_gap, - merge_flicker=merge_flicker) + + stream.sort( + key=lambda frame: frame.epoch ) - merged.sort(key=lambda s: s.start_epoch) - return merged + result.extend( + _segment_stream( + stream, + dwell_cap=dwell_cap, + session_gap=session_gap, + merge_flicker=merge_flicker, + ) + ) + + result.sort( + key=lambda segment: segment.start_epoch + ) + + return result + + +# ============================================================ +# SEGMENT ONE DEVICE STREAM +# ============================================================ def _segment_stream( frames: list[RawFrame], @@ -168,148 +448,398 @@ def _segment_stream( session_gap: float, merge_flicker: float, ) -> list[Segment]: + if not frames: return [] - # Pass 1: raw segmentation on context-key change or session gap. + frames = sorted( + frames, + key=lambda frame: frame.epoch, + ) + raw: list[Segment] = [] - cur: Segment | None = None - prev_key: tuple[str, str | None] | None = None - for i, f in enumerate(frames): - gap_to_next = ( - frames[i + 1].epoch - f.epoch if i + 1 < len(frames) else None + + current: Segment | None = None + + previous_key: tuple[str, str | None] | None = None + + for i, frame in enumerate(frames): + + if i + 1 < len(frames): + + gap_to_next = ( + frames[i + 1].epoch + - frame.epoch + ) + + gap_to_next = max( + 0.0, + gap_to_next, + ) + + else: + + gap_to_next = None + + dwell = ( + min( + gap_to_next, + dwell_cap, + ) + if gap_to_next is not None + else 0.0 ) - dwell = min(gap_to_next, dwell_cap) if gap_to_next is not None else 0.0 - key = (f.app, f.domain) - if cur is None or key != cur.key: - # Determine why this segment starts. - if cur is None and prev_key is None: + key = ( + frame.app, + frame.domain, + ) + + # ---------------------------------------------------- + # Start a new segment + # ---------------------------------------------------- + + if ( + current is None + or key != current.key + ): + + if ( + current is None + and previous_key is None + ): reason = "start" - elif cur is None and prev_key is not None: + + elif ( + current is None + and previous_key is not None + ): reason = "session_gap" + else: reason = "context_switch" - cur = Segment( - app=f.app, domain=f.domain, - start_epoch=f.epoch, end_epoch=f.epoch, + + current = Segment( + app=frame.app, + domain=frame.domain, + start_epoch=frame.epoch, + end_epoch=frame.epoch, break_reason=reason, ) - raw.append(cur) - cur.frames.append(f) - cur.end_epoch = f.epoch - if gap_to_next is not None and gap_to_next <= session_gap: - cur.active_seconds += dwell - if gap_to_next is not None and gap_to_next > session_gap: - prev_key = key - cur = None # session break: next frame starts a new segment - - # Pass 2: flicker merge. A -> B -> A where B is brief becomes one A - # segment with B recorded as an interruption. + + raw.append( + current + ) + + current.frames.append( + frame + ) + + current.end_epoch = ( + frame.epoch + ) + + # ---------------------------------------------------- + # Active dwell + # ---------------------------------------------------- + + if ( + gap_to_next is not None + and gap_to_next <= session_gap + ): + + current.active_seconds += ( + dwell + ) + + # ---------------------------------------------------- + # Session gap + # ---------------------------------------------------- + + if ( + gap_to_next is not None + and gap_to_next > session_gap + ): + + previous_key = key + + current = None + + # ======================================================== + # FLICKER MERGE + # + # A -> B -> A + # + # If B is brief, merge the two A segments and record B + # as an interruption. + # ======================================================== + if merge_flicker <= 0: return raw + merged: list[Segment] = [] + i = 0 + while i < len(raw): - seg = raw[i] + + segment = raw[i] + while ( i + 2 < len(raw) - and raw[i + 1].wall_seconds() <= merge_flicker - and raw[i + 2].key == seg.key - # never merge across a session break, on either side of B - and raw[i + 1].start_epoch - seg.end_epoch <= session_gap - and raw[i + 2].start_epoch - raw[i + 1].end_epoch <= session_gap + + and raw[i + 1].wall_seconds() + <= merge_flicker + + and raw[i + 2].key + == segment.key + + and ( + raw[i + 1].start_epoch + - segment.end_epoch + ) + <= session_gap + + and ( + raw[i + 2].start_epoch + - raw[i + 1].end_epoch + ) + <= session_gap ): - flicker, cont = raw[i + 1], raw[i + 2] - # The flicker's time is recorded on the interruption, NOT - # added to the host segment's active time: active_seconds - # stays honest about time spent in THIS context. - seg.interruptions.append( + + flicker = raw[i + 1] + + continuation = raw[i + 2] + + interruption_seconds = ( + flicker.active_seconds + or flicker.wall_seconds() + or 1.0 + ) + + segment.interruptions.append( Interruption( - app=flicker.app, domain=flicker.domain, - seconds=round(flicker.active_seconds or flicker.wall_seconds() or 1.0, 1), + app=flicker.app, + domain=flicker.domain, + seconds=round( + interruption_seconds, + 1, + ), ) ) - seg.frames.extend(cont.frames) - seg.active_seconds += cont.active_seconds - seg.end_epoch = cont.end_epoch - seg.interruptions.extend(cont.interruptions) + + segment.frames.extend( + continuation.frames + ) + + segment.active_seconds += ( + continuation.active_seconds + ) + + segment.end_epoch = ( + continuation.end_epoch + ) + + segment.interruptions.extend( + continuation.interruptions + ) + i += 2 - merged.append(seg) + + merged.append( + segment + ) + i += 1 + return merged -# ---- Coverage (port of ActivitySkeletonBuilder's one-pass measures) ---- +# ============================================================ +# COVERAGE +# ============================================================ @dataclass class Gap: + start_epoch: float end_epoch: float @property def minutes(self) -> int: - return int((self.end_epoch - self.start_epoch) / 60) + + return int( + ( + self.end_epoch + - self.start_epoch + ) + / 60 + ) @dataclass class Coverage: + first_epoch: float last_epoch: float + active_minutes: int span_minutes: int coverage_pct: int + frame_count: int distinct_apps: int + gaps: list[Gap] - hour_histogram: dict[int, int] # local hour -> active minutes + hour_histogram: dict[int, int] + + +def coverage( + db: Database, + start_utc: str, + end_utc: str, + *, + session_gap: float = SESSION_GAP, +) -> Coverage: + + frames = load_frames( + db, + start_utc, + end_utc, + ) -def coverage(db: Database, start_utc: str, end_utc: str, - *, session_gap: float = SESSION_GAP) -> Coverage: - frames = load_frames(db, start_utc, end_utc) if not frames: - return Coverage(0, 0, 0, 0, 0, 0, 0, [], {}) - from datetime import datetime + return Coverage( + first_epoch=0, + last_epoch=0, + active_minutes=0, + span_minutes=0, + coverage_pct=0, + frame_count=0, + distinct_apps=0, + gaps=[], + hour_histogram={}, + ) active_minutes: set[int] = set() - hour_minutes: dict[int, set[int]] = {} + + hour_minutes: dict[ + int, + set[int] + ] = {} + gaps: list[Gap] = [] + apps: set[str] = set() - prev: float | None = None - - for f in frames: - apps.add(f.app) - local = datetime.fromtimestamp(f.epoch).astimezone() - minute_id = int(f.epoch / 60) - active_minutes.add(minute_id) - hour_minutes.setdefault(local.hour, set()).add(minute_id) - if prev is not None and f.epoch - prev > session_gap: - gaps.append(Gap(prev, f.epoch)) - prev = f.epoch - - first, last = frames[0].epoch, frames[-1].epoch - span_min = int((last - first) / 60) - active_min = len(active_minutes) - pct = min(100, int(active_min / span_min * 100)) if span_min > 0 else 0 + + previous_epoch: float | None = None + + for frame in frames: + + apps.add( + frame.app + ) + + local = datetime.fromtimestamp( + frame.epoch + ).astimezone() + + minute_id = int( + frame.epoch / 60 + ) + + active_minutes.add( + minute_id + ) + + hour_minutes.setdefault( + local.hour, + set(), + ).add( + minute_id + ) + + if ( + previous_epoch is not None + and frame.epoch - previous_epoch + > session_gap + ): + + gaps.append( + Gap( + previous_epoch, + frame.epoch, + ) + ) + + previous_epoch = ( + frame.epoch + ) + + first = frames[0].epoch + + last = frames[-1].epoch + + span_minutes = int( + (last - first) + / 60 + ) + + active_minute_count = len( + active_minutes + ) + + if span_minutes > 0: + + coverage_pct = min( + 100, + int( + active_minute_count + / span_minutes + * 100 + ), + ) + + else: + + coverage_pct = ( + 100 + if active_minute_count > 0 + else 0 + ) + return Coverage( first_epoch=first, last_epoch=last, - active_minutes=active_min, - span_minutes=span_min, - coverage_pct=pct, + active_minutes=active_minute_count, + span_minutes=span_minutes, + coverage_pct=coverage_pct, frame_count=len(frames), distinct_apps=len(apps), - gaps=[g for g in gaps if g.minutes >= 5], - hour_histogram={h: len(m) for h, m in sorted(hour_minutes.items())}, + gaps=[ + gap + for gap in gaps + if gap.minutes >= 5 + ], + hour_histogram={ + hour: len(minutes) + for hour, minutes + in sorted( + hour_minutes.items() + ) + }, ) -# ---- App ledger (per-app aggregates over a window) --------------------- +# ============================================================ +# APP LEDGER +# ============================================================ @dataclass class AppUsage: + app: str minutes: float sessions: int @@ -317,51 +847,223 @@ class AppUsage: top_windows: list[str] -def app_ledger(db: Database, start_utc: str, end_utc: str, - *, dwell_cap: float = DWELL_CAP, - session_gap: float = SESSION_GAP) -> list[AppUsage]: - all_frames = load_frames(db, start_utc, end_utc) +def app_ledger( + db: Database, + start_utc: str, + end_utc: str, + *, + dwell_cap: float = DWELL_CAP, + session_gap: float = SESSION_GAP, +) -> list[AppUsage]: + + all_frames = load_frames( + db, + start_utc, + end_utc, + ) + + if not all_frames: + return [] + dwell: dict[str, float] = {} - windows: dict[str, dict[str, float]] = {} - sessions: dict[str, int] = {} - longest: dict[str, float] = {} - # Per-device streams: dwell is the gap to the next frame on the SAME - # monitor, so simultaneous monitors do not corrupt each other's math. - by_device: dict[str, list[RawFrame]] = {} - for f in all_frames: - by_device.setdefault(f.device, []).append(f) + windows: dict[ + str, + dict[str, float] + ] = {} + + sessions_count: dict[ + str, + int + ] = {} + + longest: dict[ + str, + float + ] = {} + + by_device: dict[ + str, + list[RawFrame] + ] = {} + + for frame in all_frames: + + by_device.setdefault( + frame.device, + [], + ).append(frame) + + # -------------------------------------------------------- + # Process each monitor/device independently + # -------------------------------------------------------- - for frames in by_device.values(): - cur_session: dict[str, float] = {} - for i, f in enumerate(frames[:-1]): - gap = frames[i + 1].epoch - f.epoch + for device_frames in by_device.values(): + + device_frames.sort( + key=lambda frame: frame.epoch + ) + + current_app: str | None = None + + current_session_seconds = 0.0 + + for i, frame in enumerate( + device_frames[:-1] + ): + + next_frame = ( + device_frames[i + 1] + ) + + gap = ( + next_frame.epoch + - frame.epoch + ) + + if gap < 0: + continue + + # Long inactivity = new session if gap > session_gap: - cur_session.clear() + + current_app = None + + current_session_seconds = 0.0 + continue - d = min(gap, dwell_cap) - dwell[f.app] = dwell.get(f.app, 0.0) + d - if f.window: - windows.setdefault(f.app, {}) - windows[f.app][f.window] = windows[f.app].get(f.window, 0.0) + d + + d = min( + gap, + dwell_cap, + ) + + # ------------------------------------------------ + # Total app dwell + # ------------------------------------------------ + + dwell[frame.app] = ( + dwell.get( + frame.app, + 0.0, + ) + + d + ) + + # ------------------------------------------------ + # Window dwell + # ------------------------------------------------ + + if frame.window: + + windows.setdefault( + frame.app, + {}, + ) + + windows[ + frame.app + ][ + frame.window + ] = ( + windows[ + frame.app + ].get( + frame.window, + 0.0, + ) + + d + ) + + # ------------------------------------------------ + # Sessions + # ------------------------------------------------ + if d > 0: - if cur_session.get(f.app, 0.0) == 0.0: - sessions[f.app] = sessions.get(f.app, 0) + 1 - cur_session[f.app] = cur_session.get(f.app, 0.0) + d - longest[f.app] = max(longest.get(f.app, 0.0), cur_session[f.app]) - - out = [] - for app, secs in sorted(dwell.items(), key=lambda kv: -kv[1]): - if secs < 20: + + if current_app != frame.app: + + sessions_count[ + frame.app + ] = ( + sessions_count.get( + frame.app, + 0, + ) + + 1 + ) + + current_app = ( + frame.app + ) + + current_session_seconds = 0.0 + + current_session_seconds += ( + d + ) + + longest[ + frame.app + ] = max( + longest.get( + frame.app, + 0.0, + ), + current_session_seconds, + ) + + # ======================================================== + # BUILD RESULT + # ======================================================== + + output: list[AppUsage] = [] + + for app, seconds in sorted( + dwell.items(), + key=lambda item: -item[1], + ): + + # Preserve original minimum usage threshold + if seconds < 20: continue - tops = sorted(windows.get(app, {}).items(), key=lambda kv: -kv[1])[:4] - out.append( + + top_windows = sorted( + windows.get( + app, + {}, + ).items(), + key=lambda item: -item[1], + )[:4] + + output.append( AppUsage( app=app, - minutes=round(secs / 60, 1), - sessions=sessions.get(app, 1), - longest_session_min=int(longest.get(app, 0.0) / 60), - top_windows=[w for w, _ in tops], + + minutes=round( + seconds / 60, + 1, + ), + + sessions=sessions_count.get( + app, + 1, + ), + + longest_session_min=int( + longest.get( + app, + 0.0, + ) + / 60 + ), + + top_windows=[ + window + for window, _ + in top_windows + ], ) ) - return out + + return output \ No newline at end of file diff --git a/src/activity_frames/steps.py b/src/activity_frames/steps.py index 30548d2..f9b86fa 100644 --- a/src/activity_frames/steps.py +++ b/src/activity_frames/steps.py @@ -1,23 +1,5 @@ """One-shot step drill-down: expand an activity frame into the ordered click-by-click script it was compiled from. - -`get_activity`'s frames are the index (what happened, when, where); each -frame's ``evidence.frame_ids`` range anchors it back to the raw capture. -:func:`steps_for_frame` re-opens that window and returns the replay view - -ordered clicks (element name, role, AXIdentifier, URL), typed runs, pastes, -and focus changes - so an agent can repeat a demonstrated task instead of -re-deriving it from pixels. - -This is the ONE-SHOT path: the script of a single demonstrated run. -Consolidating repeats into named, slotted, guarded routines is the routine -layer (see ``research/``), which builds on the same evidence. - -Labels use a resolution chain: the event's own ``element_name``; else a -point-in-rect hit-test of the click against the linked frame's accessibility -elements (smallest labeled element wins); else the window title. Typed text -comes from local capture (the recorder refuses secure-input contexts) and is -included capped by default - pass ``include_text=False`` to serve lengths -only. """ from __future__ import annotations @@ -30,10 +12,7 @@ from ._time import fmt_local_hms, parse_epoch from .db import Database -# Common display sizes in points, used to normalize a click's global pixel -# position against the frame's 0-1 element bounds. The capture layer does not -# record display geometry yet; trying the common candidates resolves the -# overwhelming share in practice (single-display setups are one of these). + _SCREEN_CANDIDATES = ( (1728.0, 1117.0), (2560.0, 1440.0), @@ -45,68 +24,257 @@ def _columns(db: Database, table: str) -> set[str]: + """Return column names for a SQLite table.""" + if not db.table_exists(table): + return set() + return {r[1] for r in db.rows(f"PRAGMA table_info({table})")} def _utc_str(epoch: float) -> str: - return datetime.fromtimestamp(epoch, tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%S") + """Convert Unix epoch seconds to UTC ISO timestamp.""" + return datetime.fromtimestamp( + epoch, + tz=timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%S") + + +def _timestamp_to_epoch(value: Any) -> float: + """ + Convert a database timestamp to Unix epoch seconds. + + Supports: + Unix epoch numbers + Numeric strings + YYYY-MM-DD HH:MM:SS + YYYY-MM-DD HH:MM:SS.ffffff + YYYY-MM-DDTHH:MM:SS + ISO timestamps + ISO timestamps ending in Z + """ + + if value is None: + raise ValueError("Timestamp cannot be None") + + # Already numeric + if isinstance(value, (int, float)): + return float(value) + + value = str(value).strip() + + if not value: + raise ValueError("Timestamp is empty") + + # Numeric timestamp stored as text + try: + return float(value) + except ValueError: + pass + + # ISO Z means UTC + iso_value = value + + if iso_value.endswith("Z"): + iso_value = iso_value[:-1] + "+00:00" + + # Python ISO parser + try: + dt = datetime.fromisoformat(iso_value) + + # Windows recorder timestamps are local time when they + # don't contain timezone information. + if dt.tzinfo is None: + local_tz = datetime.now().astimezone().tzinfo + dt = dt.replace(tzinfo=local_tz) + + return dt.timestamp() + + except ValueError: + pass + + # Additional explicit formats + formats = ( + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%d %H:%M:%S.%f", + "%Y-%m-%dT%H:%M:%S", + "%Y-%m-%dT%H:%M:%S.%f", + ) + + for fmt in formats: + try: + dt = datetime.strptime(value, fmt) + local_tz = datetime.now().astimezone().tzinfo + dt = dt.replace(tzinfo=local_tz) + + return dt.timestamp() + + except ValueError: + continue + + # Preserve compatibility with original activity-frames + return parse_epoch(value) + + +def _window_from_evidence( + db: Database, + evidence: dict, + pad_s: float = 2.0, +): + """ + Convert evidence frame IDs into a UTC time window. + + Example: + + frame_ids = "1..13" + + The Windows recorder stores timestamps as local SQLite datetime + strings, so they are converted safely to epoch time here. + """ + + rng = str( + (evidence or {}).get("frame_ids") or "" + ).strip() -def _window_from_evidence(db: Database, evidence: dict, pad_s: float = 2.0): - rng = str((evidence or {}).get("frame_ids") or "") if not rng: return None - lo, _, hi = rng.partition("..") - hi = hi or lo + + if ".." in rng: + lo, _, hi = rng.partition("..") + else: + lo = rng + hi = rng + try: - lo_i, hi_i = int(lo), int(hi) - except ValueError: + lo_i = int(lo.strip()) + hi_i = int(hi.strip()) + except (TypeError, ValueError): return None + + # Handle reversed evidence ranges + if lo_i > hi_i: + lo_i, hi_i = hi_i, lo_i + row = db.rows( - "SELECT MIN(timestamp), MAX(timestamp) FROM frames WHERE id BETWEEN ? AND ?", + """ + SELECT + MIN(timestamp), + MAX(timestamp) + FROM frames + WHERE id BETWEEN ? AND ? + """, (lo_i, hi_i), ) - if not row or not row[0][0]: + + if not row: return None + + if not row[0]: + return None + + start_value = row[0][0] + end_value = row[0][1] + + if start_value is None or end_value is None: + return None + + try: + start_epoch = _timestamp_to_epoch(start_value) + end_epoch = _timestamp_to_epoch(end_value) + except (ValueError, TypeError, OverflowError): + return None + return ( - _utc_str(parse_epoch(row[0][0]) - pad_s), - _utc_str(parse_epoch(row[0][1]) + pad_s), + _utc_str(start_epoch - pad_s), + _utc_str(end_epoch + pad_s), ) class _Resolver: - """Hit-test unlabeled clicks against the linked frame's element tree.""" + """Hit-test unlabeled clicks against the linked frame element tree.""" def __init__(self, db: Database): self.db = db - self.has_ref = "elements_ref_frame_id" in _columns(db, "frames") + + frame_cols = _columns(db, "frames") + + self.has_ref = ( + "elements_ref_frame_id" in frame_cols + ) + self.has_elements = db.table_exists("elements") - def resolve(self, frame_id: Any, x: Any, y: Any): - if not self.has_elements or not frame_id or x is None or y is None or y < 0: + def resolve( + self, + frame_id: Any, + x: Any, + y: Any, + ): + if ( + not self.has_elements + or not frame_id + or x is None + or y is None + or y < 0 + ): return None, None + if self.has_ref: + ref = self.db.scalar( - "SELECT COALESCE(elements_ref_frame_id, id) FROM frames WHERE id=?", + """ + SELECT COALESCE(elements_ref_frame_id, id) + FROM frames + WHERE id=? + """, (frame_id,), default=frame_id, ) + else: ref = frame_id + for sw, sh in _SCREEN_CANDIDATES: - xn, yn = x / sw, y / sh - if not (0 <= xn <= 1 and 0 <= yn <= 1): + + xn = x / sw + yn = y / sh + + if not ( + 0 <= xn <= 1 + and 0 <= yn <= 1 + ): continue + rows = self.db.rows( - "SELECT role, text FROM elements WHERE frame_id=? " - "AND text IS NOT NULL AND text != '' AND length(text) > 1 " - "AND left_bound - 0.01 <= ? AND (left_bound + width_bound) + 0.01 >= ? " - "AND top_bound - 0.01 <= ? AND (top_bound + height_bound) + 0.01 >= ? " - "ORDER BY (width_bound * height_bound) ASC LIMIT 1", - (ref, xn, xn, yn, yn), + """ + SELECT + role, + text + FROM elements + WHERE frame_id=? + AND text IS NOT NULL + AND text != '' + AND length(text) > 1 + AND left_bound - 0.01 <= ? + AND (left_bound + width_bound) + 0.01 >= ? + AND top_bound - 0.01 <= ? + AND (top_bound + height_bound) + 0.01 >= ? + ORDER BY + (width_bound * height_bound) ASC + LIMIT 1 + """, + ( + ref, + xn, + xn, + yn, + yn, + ), ) + if rows: return rows[0][0], rows[0][1] + return None, None @@ -119,84 +287,253 @@ def steps_for_frame( max_steps: int = 250, text_cap: int = 80, ) -> dict: - """The ordered click-by-click script behind one activity frame.""" - window = _window_from_evidence(db, evidence) + + """Return the ordered activity script behind one frame.""" + + window = _window_from_evidence( + db, + evidence, + ) + if not window: - return {"error": "frame has no usable evidence window", "evidence": evidence} + return { + "error": "frame has no usable evidence window", + "evidence": evidence, + } + start, end = window + # Your Windows recorder currently records frames only. + # Therefore ui_events may not exist. + if not db.table_exists("ui_events"): + + return { + "task": { + "app": app, + "window_utc": [start, end], + "source": "windows-recorder frames", + }, + "steps": [], + "step_count": 0, + "unresolved_clicks": 0, + "truncated": False, + "note": ( + "The Windows recorder currently contains frame/window " + "activity but no ui_events table." + ), + } + cols = _columns(db, "ui_events") + + required_columns = ( + "timestamp", + "event_type", + "app_name", + "window_title", + "browser_url", + "element_name", + "element_role", + "text_content", + ) + + # Generate safe SELECT expressions for missing columns + base_select = [] + + for column in required_columns: + if column in cols: + base_select.append(column) + else: + base_select.append( + f"NULL AS {column}" + ) + + optional_columns = ( + "x", + "y", + "element_automation_id", + "frame_id", + ) + opt = [ - c if c in cols else f"NULL AS {c}" - for c in ("x", "y", "element_automation_id", "frame_id") + c if c in cols + else f"NULL AS {c}" + for c in optional_columns ] + rows = db.rows( - "SELECT timestamp, event_type, app_name, window_title, browser_url, " - f"element_name, element_role, text_content, {', '.join(opt)} " - "FROM ui_events WHERE timestamp >= ? AND timestamp <= ? " - f"AND event_type IN ({','.join('?' * len(_STEP_EVENTS))}) " - "ORDER BY id", - (start, end, *_STEP_EVENTS), + "SELECT " + + ", ".join(base_select) + + ", " + + ", ".join(opt) + + " FROM ui_events " + + "WHERE timestamp >= ? " + + "AND timestamp <= ? " + + f"AND event_type IN ({','.join('?' * len(_STEP_EVENTS))}) " + + "ORDER BY id", + ( + start, + end, + *_STEP_EVENTS, + ), ) resolver = _Resolver(db) + steps: list[dict] = [] + unresolved = 0 last_url = None truncated = False - for (ts, etype, ev_app, win, url, el_name, el_role, text, x, y, auto_id, fid) in rows: + + for ( + ts, + etype, + ev_app, + win, + url, + el_name, + el_role, + text, + x, + y, + auto_id, + fid, + ) in rows: + if len(steps) >= max_steps: truncated = True break - if etype != "app_switch" and ev_app and app and ev_app != app: - continue # stray events from another app inside the padded window - t = fmt_local_hms(parse_epoch(ts)) + + if ( + etype != "app_switch" + and ev_app + and app + and ev_app != app + ): + continue + + try: + t = fmt_local_hms( + _timestamp_to_epoch(ts) + ) + except Exception: + t = str(ts) + if etype == "click": + if (el_role or "") == "": - continue # raw mouse-hook duplicate of the labeled click row + continue + name = (el_name or "").strip() + role = el_role or "" + resolved = False + if not name: - r_role, r_text = resolver.resolve(fid, x, y) + + r_role, r_text = resolver.resolve( + fid, + x, + y, + ) + if r_text: - name, role, resolved = r_text.strip(), r_role or role, True + + name = r_text.strip() + + role = r_role or role + + resolved = True + else: unresolved += 1 + step: dict[str, Any] = { "t": t, "op": "click", - "target": (name or (win or "").strip())[:100], - "role": role.replace("AX", "") if role else "", + "target": ( + name + or (win or "").strip() + )[:100], + "role": ( + role.replace("AX", "") + if role + else "" + ), } + if resolved: step["resolved"] = True + if auto_id: - step["automation_id"] = str(auto_id)[:60] + step["automation_id"] = str( + auto_id + )[:60] + if url and url != last_url: + step["url"] = url[:200] + last_url = url + steps.append(step) + elif etype == "text": - step = {"t": t, "op": "type", "chars": len(text or "")} + + step = { + "t": t, + "op": "type", + "chars": len(text or ""), + } + if include_text and text: step["text"] = text[:text_cap] + steps.append(step) + elif etype == "clipboard": - step = {"t": t, "op": "paste", "chars": len(text or "")} + + step = { + "t": t, + "op": "paste", + "chars": len(text or ""), + } + if include_text and text: step["text"] = text[:text_cap] + steps.append(step) + elif etype == "app_switch": + steps.append( - {"t": t, "op": "focus", "target": (f"{ev_app or ''} · {win}" if win else ev_app or "")[:100]} + { + "t": t, + "op": "focus", + "target": ( + f"{ev_app or ''} · {win}" + if win + else ev_app or "" + )[:100], + } ) - for n, s in enumerate(steps, 1): - s["n"] = n + for n, step in enumerate( + steps, + 1, + ): + step["n"] = n return { - "task": {"app": app, "window_utc": [start, end], "source": "one-shot evidence"}, + "task": { + "app": app, + "window_utc": [ + start, + end, + ], + "source": "one-shot evidence", + }, "steps": steps, "step_count": len(steps), "unresolved_clicks": unresolved, @@ -204,119 +541,311 @@ def steps_for_frame( } -# ---- query -> frame resolution (deterministic, no model) --------------------- +# --------------------------------------------------------- +# QUERY -> FRAME RESOLUTION +# --------------------------------------------------------- _STOPWORDS = frozenset( - "a an and the my our your i me we to of for in on at with get got grab fetch " - "do redo run task again it this that last latest new recent please " - "deterministic deterministically replay rerun using use activity frames frame".split() + """ + a an and the my our your i me we + to of for in on at with get got + grab fetch do redo run task again + it this that last latest new recent + please deterministic deterministically + replay rerun using use activity + frames frame + """.split() ) -# Small, curated fan-out so everyday task words match the artifacts they produce -# on screen (an "invoice" task navigates billing/purchase/payment surfaces). + _SYNONYMS = { - "invoice": ("purchase", "purchases", "billing", "payment", "payments", - "transaction", "transactions", "receipt", "receipts", "order"), - "receipt": ("invoice", "purchase", "purchases", "billing", "payment", - "payments", "transaction", "transactions"), - "bill": ("billing", "invoice", "payment", "payments", "purchase"), - "email": ("mail", "gmail", "inbox", "compose", "message", "thread"), - "meeting": ("calendar", "event", "invite", "schedule"), + + "invoice": ( + "purchase", + "purchases", + "billing", + "payment", + "payments", + "transaction", + "transactions", + "receipt", + "receipts", + "order", + ), + + "receipt": ( + "invoice", + "purchase", + "purchases", + "billing", + "payment", + "payments", + "transaction", + "transactions", + ), + + "bill": ( + "billing", + "invoice", + "payment", + "payments", + "purchase", + ), + + "email": ( + "mail", + "gmail", + "inbox", + "compose", + "message", + "thread", + ), + + "meeting": ( + "calendar", + "event", + "invite", + "schedule", + ), } -def find_frame(db: Database, doc, query: str, *, max_steps: int = 250) -> dict: - """Resolve a natural-language task query to the demonstrated frame. +def find_frame( + db: Database, + doc, + query: str, + *, + max_steps: int = 250, +) -> dict: + + """Resolve a natural-language query to a demonstrated frame.""" + + tokens = [ + t + for t in re.findall( + r"[a-z0-9]+", + (query or "").lower(), + ) + if t not in _STOPWORDS + ] - Deterministic lexical scoring, zero model calls: each query token (plus a - small synonym fan-out) is matched per frame against step URLs (+3), step - target texts (+2), and the app name (+1). Also suggests ``from_url`` - the - earliest step URL matching the query - so callers can slice off pre-task - wandering. This keeps retrieval out of agent context: the caller gets one - frame id, not the whole compiled day. - """ - tokens = [t for t in re.findall(r"[a-z0-9]+", (query or "").lower()) - if t not in _STOPWORDS] if not tokens: - return {"error": "empty query after stopwords", "query": query} + + return { + "error": "empty query after stopwords", + "query": query, + } + groups = [] + for t in tokens: - syn = _SYNONYMS.get(t) or _SYNONYMS.get(t.rstrip("s")) or () - groups.append((t, *syn)) - flat = tuple(v for grp in groups for v in grp) - groundable_roles = {"Button", "Link", "TextField", "TextArea", "RadioButton", - "CheckBox", "MenuItem", "MenuButton", "PopUpButton", - "Tab", "Cell", "StaticText"} + syn = ( + _SYNONYMS.get(t) + or _SYNONYMS.get( + t.rstrip("s") + ) + or () + ) + + groups.append( + (t, *syn) + ) + + flat = tuple( + value + for group in groups + for value in group + ) + + groundable_roles = { + "Button", + "Link", + "TextField", + "TextArea", + "RadioButton", + "CheckBox", + "MenuItem", + "MenuButton", + "PopUpButton", + "Tab", + "Cell", + "StaticText", + } + best: dict | None = None + for fr in doc.frames: - out = steps_for_frame(db, fr.app, fr.evidence, - include_text=False, max_steps=max_steps) + + out = steps_for_frame( + db, + fr.app, + fr.evidence, + include_text=False, + max_steps=max_steps, + ) + steps = out.get("steps") or [] + if not steps: continue - hay_url = " ".join(s.get("url", "") for s in steps).lower() - hay_txt = " ".join(s.get("target") or "" for s in steps).lower() - hay_app = (fr.app or "").lower() - # FULL COVERAGE required: every query token group must match somewhere - # in this frame, else a frame that merely brushes one word of the task - # could outrank the real demonstration. - score, covered = 0, True + + hay_url = " ".join( + s.get("url", "") + for s in steps + ).lower() + + hay_txt = " ".join( + s.get("target") or "" + for s in steps + ).lower() + + hay_app = ( + fr.app or "" + ).lower() + + score = 0 + covered = True + for grp in groups: - if any(v in hay_url for v in grp): + + if any( + value in hay_url + for value in grp + ): score += 3 - elif any(v in hay_txt for v in grp): + + elif any( + value in hay_txt + for value in grp + ): score += 2 - elif any(v in hay_app for v in grp): + + elif any( + value in hay_app + for value in grp + ): score += 1 + else: covered = False break + if not covered: continue + from_url = None - for s in steps: - u = (s.get("url") or "").lower() - if u and any(v in u for v in flat): - path = urlparse(s["url"]).path.strip("/") - from_url = path or s["url"] + + for step in steps: + + url = ( + step.get("url") or "" + ).lower() + + if ( + url + and any( + value in url + for value in flat + ) + ): + + path = urlparse( + step["url"] + ).path.strip("/") + + from_url = ( + path + or step["url"] + ) + break - # a real demonstration = replayable clicks made ON task-matching pages - # (groundable role or automation id, page URL context matches query) - cur_url, task_clicks = "", 0 - for s in steps: - if s.get("url"): - cur_url = s["url"].lower() - t = (s.get("target") or "").strip() - if (s.get("op") == "click" - and any(v in cur_url for v in flat) - and (s.get("role") in groundable_roles or s.get("automation_id")) - and t and "\n" not in t and len(t) <= 80 - and t.lower().strip(".…") != "loading"): + + cur_url = "" + task_clicks = 0 + + for step in steps: + + if step.get("url"): + cur_url = step[ + "url" + ].lower() + + target = ( + step.get("target") + or "" + ).strip() + + if ( + step.get("op") == "click" + and any( + value in cur_url + for value in flat + ) + and ( + step.get("role") + in groundable_roles + or step.get( + "automation_id" + ) + ) + and target + and "\n" not in target + and len(target) <= 80 + and target.lower().strip( + ".…" + ) != "loading" + ): task_clicks += 1 + if task_clicks < 1: continue - cand = { + + candidate = { "frame": f"f-{fr.index:04d}", "app": fr.app, "score": score, "task_clicks": task_clicks, "from_url": from_url, - "window_utc": out.get("task", {}).get("window_utc"), - "step_count": out.get("step_count"), + "window_utc": out.get( + "task", + {}, + ).get("window_utc"), + "step_count": out.get( + "step_count" + ), } - # among full-coverage demonstrations, RECENCY wins: "replay my X run" - # means the latest time the user actually did X - if best is None or fr.index > best["_index"]: - cand["_index"] = fr.index - best = cand + + if ( + best is None + or fr.index > best["_index"] + ): + + candidate["_index"] = fr.index + + best = candidate + if best is None: + return { - "error": "no demonstrated frame matched the query", + "error": ( + "no demonstrated frame " + "matched the query" + ), "query": query, "tokens": tokens, - "hint": "widen --hours, or check `aframes context` for what was captured", + "hint": ( + "widen --hours, or check " + "`aframes context` for " + "what was captured" + ), } - best.pop("_index", None) + + best.pop( + "_index", + None, + ) + best["query"] = query - return best + + return best \ No newline at end of file diff --git a/src/activity_frames/windows_recorder/recorder.py b/src/activity_frames/windows_recorder/recorder.py new file mode 100644 index 0000000..7a2132b --- /dev/null +++ b/src/activity_frames/windows_recorder/recorder.py @@ -0,0 +1,257 @@ +import ctypes +from ctypes import wintypes +import psutil +import time +from datetime import datetime, timezone +import sqlite3 +from pathlib import Path + + +# ============================================================ +# WINDOWS API +# ============================================================ + +user32 = ctypes.windll.user32 + + +# ============================================================ +# DATABASE LOCATION +# ============================================================ + +# recorder.py: +# activity-frames/src/activity_frames/windows_recorder/recorder.py +# +# parents[3] = activity-frames project root +PROJECT_ROOT = Path(__file__).resolve().parents[3] + +DATA_DIR = PROJECT_ROOT / "data" +DATA_DIR.mkdir(parents=True, exist_ok=True) + +DB_PATH = DATA_DIR / "activity.db" + + +# ============================================================ +# DATABASE SETUP +# ============================================================ + +def setup_database(): + + conn = sqlite3.connect(DB_PATH) + cursor = conn.cursor() + + cursor.execute(""" + CREATE TABLE IF NOT EXISTS frames ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp TIMESTAMP NOT NULL, + app_name TEXT, + window_name TEXT, + focused BOOLEAN, + browser_url TEXT, + document_path TEXT, + device_name TEXT NOT NULL DEFAULT 'monitor_1' + ) + """) + + conn.commit() + conn.close() + + +# ============================================================ +# SAVE ACTIVITY +# ============================================================ + +def save_activity(timestamp, application, window_title): + + conn = sqlite3.connect(DB_PATH) + cursor = conn.cursor() + + cursor.execute(""" + INSERT INTO frames ( + timestamp, + app_name, + window_name, + focused, + browser_url, + document_path, + device_name + ) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, ( + timestamp, + application, + window_title, + 1, + None, + None, + "monitor_1" + )) + + conn.commit() + conn.close() + + +# ============================================================ +# GET ACTIVE WINDOW +# ============================================================ + +def get_active_window_info(): + + hwnd = user32.GetForegroundWindow() + + if not hwnd: + return "Unknown", "" + + # -------------------------------------------------------- + # Get window title + # -------------------------------------------------------- + + length = user32.GetWindowTextLengthW(hwnd) + + title_buffer = ctypes.create_unicode_buffer(length + 1) + + user32.GetWindowTextW( + hwnd, + title_buffer, + length + 1 + ) + + window_title = title_buffer.value + + + # -------------------------------------------------------- + # Get process ID + # -------------------------------------------------------- + + process_id = wintypes.DWORD() + + user32.GetWindowThreadProcessId( + hwnd, + ctypes.byref(process_id) + ) + + + # -------------------------------------------------------- + # Get application/process name + # -------------------------------------------------------- + + try: + + process = psutil.Process(process_id.value) + + app_name = process.name() + + except ( + psutil.NoSuchProcess, + psutil.AccessDenied, + psutil.ZombieProcess + ): + + app_name = "Unknown" + + + return app_name, window_title + + +# ============================================================ +# UTC TIMESTAMP +# ============================================================ + +def get_utc_timestamp(): + + """ + Activity Frames expects frame timestamps to be UTC. + + Example: + India 23:30 + becomes approximately + UTC 18:00 + """ + + return datetime.now(timezone.utc).strftime( + "%Y-%m-%d %H:%M:%S" + ) + + +# ============================================================ +# MAIN RECORDER +# ============================================================ + +def main(): + + setup_database() + + print("=" * 60) + print("Windows Activity Recorder Started") + print("=" * 60) + + print(f"Database: {DB_PATH}") + + print("Timestamp mode: UTC") + + print("Press Ctrl+C to stop.") + + print("=" * 60) + print() + + + last_activity = None + + + try: + + while True: + + app_name, window_title = get_active_window_info() + + current_activity = ( + app_name, + window_title + ) + + + # Save only when active window changes + if current_activity != last_activity: + + timestamp = get_utc_timestamp() + + + print(f"[{timestamp} UTC]") + + print( + f"Application: {app_name}" + ) + + print( + f"Window Title: {window_title}" + ) + + print("-" * 60) + + + save_activity( + timestamp, + app_name, + window_title + ) + + + last_activity = current_activity + + + time.sleep(1) + + + except KeyboardInterrupt: + + print() + print("=" * 60) + print("Recorder stopped.") + print("=" * 60) + + +# ============================================================ +# START PROGRAM +# ============================================================ + +if __name__ == "__main__": + main() \ No newline at end of file