Extract harness-independent hook protocol

CLAUDE.md

@@ -59,7 +59,7 @@ uv tool install . --reinstall
 
 - Session IDs are provided directly by Claude Code (no generated IDs)
 - Hook handler reads JSON from stdin (provided by Claude Code)
-- Tool name mapping is done via `TOOL_TYPE_MAP` in hook_handler.py
+- Tool name mapping is done via `get_tool_type` in core/models/hook.py
 - Database uses raw SQL with migration support for flexibility
 - All timestamps are stored as ISO format strings
 
@@ -127,9 +127,8 @@ echo '{"session_id": "test123", "transcript_path": "/tmp/transcript.jsonl", "too
 
 ## Adding New Tool Types
 
-1. Add to `ToolType` enum in models.py
-2. Update `TOOL_TYPE_MAP` in hook_handler.py
-3. No database migration needed (sqlite-utils handles schema)
+1. Add to `ToolType` enum in src/slopometry/core/models/hook.py
+2. Update `get_tool_type` in src/slopometry/core/models/hook.py
 
 ## Experiment Tracking
 

src/slopometry/cli.py

@@ -4,6 +4,7 @@
 import sys
 import warnings
 from importlib.metadata import version
+from pathlib import Path
 
 # REASON: analyzed repos may contain invalid escape sequences that emit SyntaxWarnings during AST parsing
 warnings.filterwarnings("ignore", category=SyntaxWarning)
@@ -53,45 +54,45 @@ def hook_handler() -> None:
 def hook_pre_tool_use() -> None:
     """Internal command for processing PreToolUse hook events."""
     from slopometry.core.hook_handler import handle_hook
-    from slopometry.core.models.hook import HookEventType
+    from slopometry.core.protocol.adapters.claude import CLAUDE_HOOK_KIND_MAP
 
-    sys.exit(handle_hook(event_type_override=HookEventType.PRE_TOOL_USE))
+    sys.exit(handle_hook(event_type_override=CLAUDE_HOOK_KIND_MAP["PreToolUse"]))
 
 
 @cli.command("hook-post-tool-use", hidden=True)
 def hook_post_tool_use() -> None:
     """Internal command for processing PostToolUse hook events."""
     from slopometry.core.hook_handler import handle_hook
-    from slopometry.core.models.hook import HookEventType
+    from slopometry.core.protocol.adapters.claude import CLAUDE_HOOK_KIND_MAP
 
-    sys.exit(handle_hook(event_type_override=HookEventType.POST_TOOL_USE))
+    sys.exit(handle_hook(event_type_override=CLAUDE_HOOK_KIND_MAP["PostToolUse"]))
 
 
 @cli.command("hook-notification", hidden=True)
 def hook_notification() -> None:
     """Internal command for processing Notification hook events."""
     from slopometry.core.hook_handler import handle_hook
-    from slopometry.core.models.hook import HookEventType
+    from slopometry.core.protocol.adapters.claude import CLAUDE_HOOK_KIND_MAP
 
-    sys.exit(handle_hook(event_type_override=HookEventType.NOTIFICATION))
+    sys.exit(handle_hook(event_type_override=CLAUDE_HOOK_KIND_MAP["Notification"]))
 
 
 @cli.command("hook-stop", hidden=True)
 def hook_stop() -> None:
     """Internal command for processing Stop hook events."""
     from slopometry.core.hook_handler import handle_hook
-    from slopometry.core.models.hook import HookEventType
+    from slopometry.core.protocol.adapters.claude import CLAUDE_HOOK_KIND_MAP
 
-    sys.exit(handle_hook(event_type_override=HookEventType.STOP))
+    sys.exit(handle_hook(event_type_override=CLAUDE_HOOK_KIND_MAP["Stop"]))
 
 
 @cli.command("hook-subagent-stop", hidden=True)
 def hook_subagent_stop() -> None:
     """Internal command for processing SubagentStop hook events."""
     from slopometry.core.hook_handler import handle_hook
-    from slopometry.core.models.hook import HookEventType
+    from slopometry.core.protocol.adapters.claude import CLAUDE_HOOK_KIND_MAP
 
-    sys.exit(handle_hook(event_type_override=HookEventType.SUBAGENT_STOP))
+    sys.exit(handle_hook(event_type_override=CLAUDE_HOOK_KIND_MAP["SubagentStop"]))
 
 
 @cli.command("hook-opencode", hidden=True)
@@ -113,6 +114,59 @@ def hook_opencode(event_type: str) -> None:
     sys.exit(handle_opencode_hook(event_type))
 
 
+@cli.command("ingest")
+@click.option("--source", required=True, help="Agent tool that produced the events, e.g. 'mmkr'.")
+@click.option(
+    "--file",
+    "input_file",
+    type=click.Path(exists=True, dir_okay=False, path_type=Path),
+    default=None,
+    help="JSONL file of event envelopes; reads from stdin when omitted.",
+)
+@click.option(
+    "--working-directory",
+    type=click.Path(exists=True, file_okay=False, path_type=Path),
+    default=None,
+    help="Working directory recorded on ingested events; defaults to the current directory.",
+)
+def ingest(source: str, input_file: Path | None, working_directory: Path | None) -> None:
+    """Ingest hook-protocol event envelopes (JSONL) from any agent tool.
+
+    Each line must be an EventEnvelope JSON object; see
+    slopometry.core.protocol.schema for the stable schema. Re-ingesting the
+    same (source, event_id) pairs is an idempotent no-op.
+    """
+    import json
+
+    from pydantic import ValidationError
+
+    from slopometry.core.protocol.ingest import ingest_envelopes
+    from slopometry.core.protocol.schema import EventEnvelope
+
+    content = input_file.read_text() if input_file else sys.stdin.read()
+    envelopes: list[EventEnvelope] = []
+    for line_number, line in enumerate(content.splitlines(), start=1):
+        if not line.strip():
+            continue
+        try:
+            envelope = EventEnvelope.model_validate(json.loads(line))
+        except (json.JSONDecodeError, ValidationError) as e:
+            console.print(f"[red]Invalid envelope at line {line_number}: {e}[/red]")
+            sys.exit(2)
+        if envelope.source != source:
+            console.print(
+                f"[red]Envelope source '{envelope.source}' at line {line_number} does not match --source '{source}'[/red]"
+            )
+            sys.exit(2)
+        envelopes.append(envelope)
+
+    report = ingest_envelopes(envelopes, working_directory=str(working_directory) if working_directory else None)
+    console.print(
+        f"[green]Ingested {report.inserted} events from source '{source}'"
+        f" (skipped {report.skipped_duplicates} duplicates)[/green]"
+    )
+
+
 @cli.command("shell-completion")
 @click.argument("shell", type=click.Choice(["bash", "zsh", "fish"]))
 def shell_completion(shell: str) -> None:

src/slopometry/core/database.py

@@ -15,10 +15,11 @@
 from slopometry.core.models.complexity import ComplexityDelta, ExtendedComplexityMetrics
 from slopometry.core.models.display import LeaderboardEntry, SessionDisplayData
 from slopometry.core.models.experiment import ExperimentProgress, ExperimentRun, ExperimentStatus, FeatureBoundary
-from slopometry.core.models.hook import GitState, HookEvent, HookEventType, Project, ProjectSource, ToolType
+from slopometry.core.models.hook import GitState, HookEvent, Project, ProjectSource, ToolType
 from slopometry.core.models.session import BehavioralPatterns, ContextCoverage, PlanEvolution, SessionStatistics
 from slopometry.core.models.user_story import NextFeaturePrediction, UserStory, UserStoryEntry
 from slopometry.core.plan_analyzer import PlanAnalyzer
+from slopometry.core.protocol.kinds import EventKind, KnownSource
 from slopometry.core.settings import settings
 
 
@@ -87,7 +88,8 @@ def _create_tables(self) -> None:
                     project_source TEXT,
                     transcript_path TEXT,
                     source TEXT DEFAULT 'claude_code',
-                    parent_session_id TEXT
+                    parent_session_id TEXT,
+                    event_id TEXT
                 )
             """)
             conn.execute("""
@@ -111,6 +113,10 @@ def _create_tables(self) -> None:
                 ON hook_events(session_id, event_type, source)
             """)
 
+            # NOTE: the (source, event_id) unique partial index is created by
+            # Migration019CanonicalEventKinds; it must not be created here since
+            # _create_tables runs before migrations on pre-existing databases.
+
             conn.execute("""
                 CREATE TABLE IF NOT EXISTS experiment_runs (
                     id TEXT PRIMARY KEY,
@@ -320,18 +326,38 @@ def _create_tables(self) -> None:
 
             conn.commit()
 
+    def has_event(self, source: str, event_id: str) -> bool:
+        """Check whether an event with the given (source, event_id) is already stored.
+
+        Used by envelope ingestion to make backfills idempotent."""
+        with self._get_db_connection() as conn:
+            row = conn.execute(
+                "SELECT 1 FROM hook_events WHERE source = ? AND event_id = ? LIMIT 1",
+                (source, event_id),
+            ).fetchone()
+            return row is not None
+
+    def get_max_sequence_number(self, session_id: str) -> int:
+        """Get the highest stored sequence number for a session (0 when the session has no events)."""
+        with self._get_db_connection() as conn:
+            row = conn.execute(
+                "SELECT COALESCE(MAX(sequence_number), 0) FROM hook_events WHERE session_id = ?",
+                (session_id,),
+            ).fetchone()
+            return row[0]
+
     def save_event(self, event: HookEvent) -> int:
         """Save a hook event to the database."""
         with self._get_db_connection() as conn:
             cursor = conn.execute(
                 """
                 INSERT INTO hook_events (
                     session_id, event_type, timestamp, sequence_number,
-                    tool_name, tool_type, metadata, duration_ms,
-                    exit_code, error_message, git_state, working_directory,
-                    project_name, project_source, transcript_path,
-                    source, parent_session_id
-                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                     tool_name, tool_type, metadata, duration_ms,
+                     exit_code, error_message, git_state, working_directory,
+                     project_name, project_source, transcript_path,
+                     source, parent_session_id, event_id
+                 ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
             """,
                 (
                     event.session_id,
@@ -348,10 +374,11 @@ def save_event(self, event: HookEvent) -> int:
                     event.working_directory,
                     event.project.name if event.project else None,
                     event.project.source.value if event.project else None,
-                    event.transcript_path,
-                    event.source.value,
-                    event.parent_session_id,
-                ),
+                     event.transcript_path,
+                     event.source,
+                     event.parent_session_id,
+                     event.event_id,
+                 ),
             )
             return cursor.lastrowid or 0
 
@@ -384,18 +411,13 @@ def get_session_events(self, session_id: str) -> list[HookEvent]:
                         source=ProjectSource(row["project_source"]),
                     )
 
-                # Handle source column (may be NULL for pre-migration rows)
-                from slopometry.core.models.hook import EventSource
-
-                source_val = row["source"] if "source" in row.keys() else None
-                source = EventSource(source_val) if source_val else EventSource.CLAUDE_CODE
                 parent_session_id = row["parent_session_id"] if "parent_session_id" in row.keys() else None
 
                 events.append(
                     HookEvent(
                         id=row["id"],
                         session_id=row["session_id"],
-                        event_type=HookEventType(row["event_type"]),
+                        event_type=EventKind(row["event_type"]),
                         timestamp=datetime.fromisoformat(row["timestamp"]),
                         sequence_number=row["sequence_number"],
                         tool_name=row["tool_name"],
@@ -407,17 +429,18 @@ def get_session_events(self, session_id: str) -> list[HookEvent]:
                         git_state=git_state,
                         working_directory=working_directory,
                         project=project,
-                        transcript_path=row["transcript_path"],
-                        source=source,
-                        parent_session_id=parent_session_id,
-                    )
+                         transcript_path=row["transcript_path"],
+                         source=row["source"],
+                         parent_session_id=parent_session_id,
+                         event_id=row["event_id"],
+                     )
                 )
             return events
 
     def get_session_source(self, session_id: str) -> str:
-        """Get the source agent tool for a session ('claude_code' or 'opencode').
+        """Get the source agent tool string for a session.
 
-        Returns 'claude_code' for pre-migration sessions or if the column is missing.
+        Returns 'claude_code' for pre-migration sessions stored before the source column existed.
         """
         with self._get_db_connection() as conn:
             conn.row_factory = sqlite3.Row
@@ -427,7 +450,7 @@ def get_session_source(self, session_id: str) -> str:
             ).fetchone()
             if row and row["source"]:
                 return row["source"]
-            return "claude_code"
+            return str(KnownSource.CLAUDE_CODE)
 
     def get_opencode_transcript(self, session_id: str) -> list[dict] | None:
         """Extract the transcript from an OpenCode session's Stop event metadata.
@@ -440,10 +463,10 @@ def get_opencode_transcript(self, session_id: str) -> list[dict] | None:
             row = conn.execute(
                 """
                 SELECT metadata FROM hook_events
-                WHERE session_id = ? AND event_type = 'Stop' AND source = 'opencode'
+                WHERE session_id = ? AND event_type = ? AND source = ?
                 ORDER BY sequence_number DESC LIMIT 1
                 """,
-                (session_id,),
+                (session_id, EventKind.STOP, KnownSource.OPENCODE),
             ).fetchone()
             if not row:
                 return None
@@ -563,7 +586,7 @@ def get_session_statistics(self, session_id: str) -> SessionStatistics | None:
                 (session_id,),
             ).fetchall()
 
-            events_by_type = {HookEventType(row["event_type"]): row["count"] for row in event_type_rows}
+            events_by_type = {EventKind(row["event_type"]): row["count"] for row in event_type_rows}
 
             tool_usage_rows = conn.execute(
                 """
@@ -834,12 +857,12 @@ def _calculate_plan_evolution(self, session_id: str) -> PlanEvolution:
                 WHERE session_id = ? AND event_type IN (?, ?)
                 ORDER BY sequence_number
                 """,
-                (session_id, HookEventType.POST_TOOL_USE.value, HookEventType.TODO_UPDATED.value),
+                (session_id, EventKind.TOOL_RESULT.value, EventKind.TODO_UPDATED.value),
             ).fetchall()
 
             # Check if this session has TODO_UPDATED events (OpenCode).
             # If so, skip POST_TOOL_USE todowrite events to avoid duplicate analysis.
-            has_todo_updated = any(row["event_type"] == HookEventType.TODO_UPDATED.value for row in rows)
+            has_todo_updated = any(row["event_type"] == EventKind.TODO_UPDATED.value for row in rows)
 
             for row in rows:
                 timestamp = datetime.fromisoformat(row["timestamp"])
@@ -849,7 +872,7 @@ def _calculate_plan_evolution(self, session_id: str) -> PlanEvolution:
 
                 metadata = json.loads(row["metadata"]) if row["metadata"] else {}
 
-                if event_type == HookEventType.TODO_UPDATED.value:
+                if event_type == EventKind.TODO_UPDATED.value:
                     # OpenCode todo.updated bus event — canonical source for OpenCode todos
                     todos = metadata.get("todos", [])
                     if todos: